| ... | @@ -1625,24 +1625,68 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void | ... | @@ -1625,24 +1625,68 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void |
| 1625 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 1625 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1626 | } | 1626 | } |
| 1627 | | 1627 | |
| | 1628 | fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 { |
| | 1629 | const air_tag = self.air.instructions.items(.tag); |
| | 1630 | const air_data = self.air.instructions.items(.data); |
| | 1631 | |
| | 1632 | const dst_ty = self.air.typeOf(dst_air); |
| | 1633 | const dst_info = dst_ty.intInfo(self.target.*); |
| | 1634 | if (Air.refToIndex(dst_air)) |inst| { |
| | 1635 | switch (air_tag[inst]) { |
| | 1636 | .constant => { |
| | 1637 | const src_val = self.air.values[air_data[inst].ty_pl.payload]; |
| | 1638 | var space: Value.BigIntSpace = undefined; |
| | 1639 | const src_int = src_val.toBigInt(&space, self.target.*); |
| | 1640 | return @intCast(u16, src_int.bitCountTwosComp()) + |
| | 1641 | @boolToInt(src_int.positive and dst_info.signedness == .signed); |
| | 1642 | }, |
| | 1643 | .intcast => { |
| | 1644 | const src_ty = self.air.typeOf(air_data[inst].ty_op.operand); |
| | 1645 | const src_info = src_ty.intInfo(self.target.*); |
| | 1646 | return @min(switch (src_info.signedness) { |
| | 1647 | .signed => switch (dst_info.signedness) { |
| | 1648 | .signed => src_info.bits, |
| | 1649 | .unsigned => src_info.bits - 1, |
| | 1650 | }, |
| | 1651 | .unsigned => switch (dst_info.signedness) { |
| | 1652 | .signed => src_info.bits + 1, |
| | 1653 | .unsigned => src_info.bits, |
| | 1654 | }, |
| | 1655 | }, dst_info.bits); |
| | 1656 | }, |
| | 1657 | else => {}, |
| | 1658 | } |
| | 1659 | } |
| | 1660 | return dst_info.bits; |
| | 1661 | } |
| | 1662 | |
| 1628 | fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { | 1663 | fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { |
| 1629 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; | 1664 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| 1630 | const result = result: { | 1665 | const result = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1631 | if (self.liveness.isUnused(inst)) break :result .dead; | | |
| 1632 | | | |
| 1633 | const tag = self.air.instructions.items(.tag)[inst]; | 1666 | const tag = self.air.instructions.items(.tag)[inst]; |
| 1634 | const ty = self.air.typeOfIndex(inst); | 1667 | const dst_ty = self.air.typeOfIndex(inst); |
| 1635 | | 1668 | if (dst_ty.zigTypeTag() == .Float) |
| 1636 | if (ty.zigTypeTag() == .Float) { | | |
| 1637 | break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); | 1669 | break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); |
| 1638 | } | | |
| 1639 | | 1670 | |
| 1640 | try self.spillRegisters(&.{ .rax, .rdx }); | 1671 | const dst_info = dst_ty.intInfo(self.target.*); |
| | 1672 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { |
| | 1673 | .signed => .int_signed, |
| | 1674 | .unsigned => .int_unsigned, |
| | 1675 | } }, .data = switch (tag) { |
| | 1676 | else => unreachable, |
| | 1677 | .mul, .mulwrap => std.math.max3( |
| | 1678 | self.activeIntBits(bin_op.lhs), |
| | 1679 | self.activeIntBits(bin_op.rhs), |
| | 1680 | dst_info.bits / 2, |
| | 1681 | ), |
| | 1682 | .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_info.bits, |
| | 1683 | } }; |
| | 1684 | const src_ty = Type.initPayload(&src_pl.base); |
| 1641 | | 1685 | |
| | 1686 | try self.spillRegisters(&.{ .rax, .rdx }); |
| 1642 | const lhs = try self.resolveInst(bin_op.lhs); | 1687 | const lhs = try self.resolveInst(bin_op.lhs); |
| 1643 | const rhs = try self.resolveInst(bin_op.rhs); | 1688 | const rhs = try self.resolveInst(bin_op.rhs); |
| 1644 | | 1689 | break :result try self.genMulDivBinOp(tag, inst, dst_ty, src_ty, lhs, rhs); |
| 1645 | break :result try self.genMulDivBinOp(tag, inst, ty, lhs, rhs); | | |
| 1646 | }; | 1690 | }; |
| 1647 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 1691 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1648 | } | 1692 | } |
| ... | @@ -1795,7 +1839,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1795,7 +1839,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { |
| 1795 | break :cc .c; | 1839 | break :cc .c; |
| 1796 | }; | 1840 | }; |
| 1797 | | 1841 | |
| 1798 | const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, lhs_mcv, rhs_mcv); | 1842 | const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv); |
| 1799 | const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2)); | 1843 | const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2)); |
| 1800 | try self.asmCmovccRegisterRegister( | 1844 | try self.asmCmovccRegisterRegister( |
| 1801 | registerAlias(dst_mcv.register, abi_size), | 1845 | registerAlias(dst_mcv.register, abi_size), |
| ... | @@ -1809,9 +1853,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1809,9 +1853,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { |
| 1809 | | 1853 | |
| 1810 | fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | 1854 | fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1811 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; | 1855 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1812 | const tag = self.air.instructions.items(.tag)[inst]; | | |
| 1813 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | 1856 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 1814 | const result = if (self.liveness.isUnused(inst)) .dead else result: { | 1857 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| | 1858 | const tag = self.air.instructions.items(.tag)[inst]; |
| 1815 | const ty = self.air.typeOf(bin_op.lhs); | 1859 | const ty = self.air.typeOf(bin_op.lhs); |
| 1816 | const abi_size = ty.abiSize(self.target.*); | 1860 | const abi_size = ty.abiSize(self.target.*); |
| 1817 | switch (ty.zigTypeTag()) { | 1861 | switch (ty.zigTypeTag()) { |
| ... | @@ -1842,21 +1886,17 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1842,21 +1886,17 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1842 | | 1886 | |
| 1843 | const int_info = ty.intInfo(self.target.*); | 1887 | const int_info = ty.intInfo(self.target.*); |
| 1844 | | 1888 | |
| 1845 | if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { | 1889 | if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { |
| 1846 | self.eflags_inst = inst; | 1890 | self.eflags_inst = inst; |
| 1847 | | 1891 | break :result .{ .register_overflow = .{ |
| 1848 | const cc: Condition = switch (int_info.signedness) { | | |
| 1849 | .unsigned => .c, | | |
| 1850 | .signed => .o, | | |
| 1851 | }; | | |
| 1852 | break :result MCValue{ .register_overflow = .{ | | |
| 1853 | .reg = partial.register, | 1892 | .reg = partial.register, |
| 1854 | .eflags = cc, | 1893 | .eflags = switch (int_info.signedness) { |
| | 1894 | .unsigned => .c, |
| | 1895 | .signed => .o, |
| | 1896 | }, |
| 1855 | } }; | 1897 | } }; |
| 1856 | } | 1898 | } |
| 1857 | | 1899 | |
| 1858 | self.eflags_inst = null; | | |
| 1859 | | | |
| 1860 | const tuple_ty = self.air.typeOfIndex(inst); | 1900 | const tuple_ty = self.air.typeOfIndex(inst); |
| 1861 | const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); | 1901 | const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); |
| 1862 | const tuple_align = tuple_ty.abiAlignment(self.target.*); | 1902 | const tuple_align = tuple_ty.abiAlignment(self.target.*); |
| ... | @@ -1865,12 +1905,11 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1865,12 +1905,11 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1865 | | 1905 | |
| 1866 | try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial.register); | 1906 | try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial.register); |
| 1867 | | 1907 | |
| 1868 | break :result MCValue{ .stack_offset = stack_offset }; | 1908 | break :result .{ .stack_offset = stack_offset }; |
| 1869 | }, | 1909 | }, |
| 1870 | else => unreachable, | 1910 | else => unreachable, |
| 1871 | } | 1911 | } |
| 1872 | }; | 1912 | }; |
| 1873 | | | |
| 1874 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 1913 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1875 | } | 1914 | } |
| 1876 | | 1915 | |
| ... | @@ -1931,48 +1970,56 @@ fn genSetStackTruncatedOverflowCompare( | ... | @@ -1931,48 +1970,56 @@ fn genSetStackTruncatedOverflowCompare( |
| 1931 | fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | 1970 | fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1932 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; | 1971 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1933 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | 1972 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 1934 | | 1973 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1935 | if (self.liveness.isUnused(inst)) { | 1974 | const dst_ty = self.air.typeOf(bin_op.lhs); |
| 1936 | return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); | 1975 | switch (dst_ty.zigTypeTag()) { |
| 1937 | } | | |
| 1938 | | | |
| 1939 | const ty = self.air.typeOf(bin_op.lhs); | | |
| 1940 | const abi_size = ty.abiSize(self.target.*); | | |
| 1941 | const result: MCValue = result: { | | |
| 1942 | switch (ty.zigTypeTag()) { | | |
| 1943 | .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), | 1976 | .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), |
| 1944 | .Int => { | 1977 | .Int => { |
| 1945 | if (abi_size > 8) { | 1978 | try self.spillEflagsIfOccupied(); |
| 1946 | return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{}); | | |
| 1947 | } | | |
| 1948 | | | |
| 1949 | const int_info = ty.intInfo(self.target.*); | | |
| 1950 | | 1979 | |
| 1951 | if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { | 1980 | const dst_info = dst_ty.intInfo(self.target.*); |
| 1952 | try self.spillEflagsIfOccupied(); | 1981 | if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) { |
| 1953 | self.eflags_inst = inst; | 1982 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { |
| | 1983 | .signed => .int_signed, |
| | 1984 | .unsigned => .int_unsigned, |
| | 1985 | } }, .data = std.math.max3( |
| | 1986 | self.activeIntBits(bin_op.lhs), |
| | 1987 | self.activeIntBits(bin_op.rhs), |
| | 1988 | dst_info.bits / 2, |
| | 1989 | ) }; |
| | 1990 | const src_ty = Type.initPayload(&src_pl.base); |
| 1954 | | 1991 | |
| 1955 | try self.spillRegisters(&.{ .rax, .rdx }); | 1992 | try self.spillRegisters(&.{ .rax, .rdx }); |
| 1956 | | | |
| 1957 | const lhs = try self.resolveInst(bin_op.lhs); | 1993 | const lhs = try self.resolveInst(bin_op.lhs); |
| 1958 | const rhs = try self.resolveInst(bin_op.rhs); | 1994 | const rhs = try self.resolveInst(bin_op.rhs); |
| 1959 | | 1995 | |
| 1960 | const partial = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); | 1996 | const partial = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs); |
| 1961 | const cc: Condition = switch (int_info.signedness) { | 1997 | const cc: Condition = switch (dst_info.signedness) { |
| 1962 | .unsigned => .c, | 1998 | .unsigned => .c, |
| 1963 | .signed => .o, | 1999 | .signed => .o, |
| 1964 | }; | 2000 | }; |
| 1965 | break :result MCValue{ .register_overflow = .{ | 2001 | switch (partial) { |
| 1966 | .reg = partial.register, | 2002 | .register => |reg| { |
| 1967 | .eflags = cc, | 2003 | self.eflags_inst = inst; |
| 1968 | } }; | 2004 | break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; |
| 1969 | } | 2005 | }, |
| | 2006 | else => {}, |
| | 2007 | } |
| 1970 | | 2008 | |
| 1971 | try self.spillEflagsIfOccupied(); | 2009 | const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*)); |
| 1972 | self.eflags_inst = null; | 2010 | const dst_mcv = try self.allocRegOrMem(inst, false); |
| | 2011 | try self.genSetStack( |
| | 2012 | Type.u1, |
| | 2013 | dst_mcv.stack_offset - dst_abi_size, |
| | 2014 | .{ .eflags = cc }, |
| | 2015 | .{}, |
| | 2016 | ); |
| | 2017 | try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial, .{}); |
| | 2018 | break :result dst_mcv; |
| | 2019 | } |
| 1973 | | 2020 | |
| 1974 | const dst_reg: Register = dst_reg: { | 2021 | const dst_reg: Register = dst_reg: { |
| 1975 | switch (int_info.signedness) { | 2022 | switch (dst_info.signedness) { |
| 1976 | .signed => { | 2023 | .signed => { |
| 1977 | const lhs = try self.resolveInst(bin_op.lhs); | 2024 | const lhs = try self.resolveInst(bin_op.lhs); |
| 1978 | const rhs = try self.resolveInst(bin_op.rhs); | 2025 | const rhs = try self.resolveInst(bin_op.rhs); |
| ... | @@ -1985,14 +2032,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1985,14 +2032,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 1985 | | 2032 | |
| 1986 | const dst_reg: Register = blk: { | 2033 | const dst_reg: Register = blk: { |
| 1987 | if (lhs.isRegister()) break :blk lhs.register; | 2034 | if (lhs.isRegister()) break :blk lhs.register; |
| 1988 | break :blk try self.copyToTmpRegister(ty, lhs); | 2035 | break :blk try self.copyToTmpRegister(dst_ty, lhs); |
| 1989 | }; | 2036 | }; |
| 1990 | const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); | 2037 | const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); |
| 1991 | defer self.register_manager.unlockReg(dst_reg_lock); | 2038 | defer self.register_manager.unlockReg(dst_reg_lock); |
| 1992 | | 2039 | |
| 1993 | const rhs_mcv: MCValue = blk: { | 2040 | const rhs_mcv: MCValue = blk: { |
| 1994 | if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; | 2041 | if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; |
| 1995 | break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; | 2042 | break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) }; |
| 1996 | }; | 2043 | }; |
| 1997 | const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { | 2044 | const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { |
| 1998 | .register => |reg| self.register_manager.lockReg(reg), | 2045 | .register => |reg| self.register_manager.lockReg(reg), |
| ... | @@ -2010,7 +2057,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2010,7 +2057,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2010 | const lhs = try self.resolveInst(bin_op.lhs); | 2057 | const lhs = try self.resolveInst(bin_op.lhs); |
| 2011 | const rhs = try self.resolveInst(bin_op.rhs); | 2058 | const rhs = try self.resolveInst(bin_op.rhs); |
| 2012 | | 2059 | |
| 2013 | const dst_mcv = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); | 2060 | const dst_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, dst_ty, lhs, rhs); |
| 2014 | break :dst_reg dst_mcv.register; | 2061 | break :dst_reg dst_mcv.register; |
| 2015 | }, | 2062 | }, |
| 2016 | } | 2063 | } |
| ... | @@ -2022,14 +2069,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2022,14 +2069,13 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2022 | const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); | 2069 | const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); |
| 2023 | const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); | 2070 | const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); |
| 2024 | | 2071 | |
| 2025 | try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, dst_reg); | 2072 | try self.genSetStackTruncatedOverflowCompare(dst_ty, stack_offset, overflow_bit_offset, dst_reg); |
| 2026 | | 2073 | |
| 2027 | break :result MCValue{ .stack_offset = stack_offset }; | 2074 | break :result .{ .stack_offset = stack_offset }; |
| 2028 | }, | 2075 | }, |
| 2029 | else => unreachable, | 2076 | else => unreachable, |
| 2030 | } | 2077 | } |
| 2031 | }; | 2078 | }; |
| 2032 | | | |
| 2033 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 2079 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 2034 | } | 2080 | } |
| 2035 | | 2081 | |
| ... | @@ -2040,7 +2086,6 @@ fn genIntMulDivOpMir( | ... | @@ -2040,7 +2086,6 @@ fn genIntMulDivOpMir( |
| 2040 | self: *Self, | 2086 | self: *Self, |
| 2041 | tag: Mir.Inst.Tag, | 2087 | tag: Mir.Inst.Tag, |
| 2042 | ty: Type, | 2088 | ty: Type, |
| 2043 | signedness: std.builtin.Signedness, | | |
| 2044 | lhs: MCValue, | 2089 | lhs: MCValue, |
| 2045 | rhs: MCValue, | 2090 | rhs: MCValue, |
| 2046 | ) !void { | 2091 | ) !void { |
| ... | @@ -2057,26 +2102,23 @@ fn genIntMulDivOpMir( | ... | @@ -2057,26 +2102,23 @@ fn genIntMulDivOpMir( |
| 2057 | try self.genSetReg(ty, .rax, lhs); | 2102 | try self.genSetReg(ty, .rax, lhs); |
| 2058 | } | 2103 | } |
| 2059 | | 2104 | |
| 2060 | switch (signedness) { | 2105 | switch (tag) { |
| 2061 | .signed => try self.asmOpOnly(.cqo), | 2106 | else => unreachable, |
| 2062 | .unsigned => try self.asmRegisterRegister(.xor, .rdx, .rdx), | 2107 | .mul, .imul => {}, |
| | 2108 | .div => try self.asmRegisterRegister(.xor, .edx, .edx), |
| | 2109 | .idiv => try self.asmOpOnly(.cqo), |
| 2063 | } | 2110 | } |
| 2064 | | 2111 | |
| 2065 | const factor = switch (rhs) { | 2112 | const factor: MCValue = switch (rhs) { |
| 2066 | .register => rhs, | 2113 | .register, .stack_offset => rhs, |
| 2067 | .stack_offset => rhs, | 2114 | else => .{ .register = try self.copyToTmpRegister(ty, rhs) }, |
| 2068 | else => blk: { | | |
| 2069 | const reg = try self.copyToTmpRegister(ty, rhs); | | |
| 2070 | break :blk MCValue{ .register = reg }; | | |
| 2071 | }, | | |
| 2072 | }; | 2115 | }; |
| 2073 | | | |
| 2074 | switch (factor) { | 2116 | switch (factor) { |
| 2075 | .register => |reg| try self.asmRegister(tag, reg), | 2117 | .register => |reg| try self.asmRegister(tag, reg), |
| 2076 | .stack_offset => |off| try self.asmMemory(tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ | 2118 | .stack_offset => |off| try self.asmMemory(tag, Memory.sib( |
| 2077 | .base = .rbp, | 2119 | Memory.PtrSize.fromSize(abi_size), |
| 2078 | .disp = -off, | 2120 | .{ .base = .rbp, .disp = -off }, |
| 2079 | })), | 2121 | )), |
| 2080 | else => unreachable, | 2122 | else => unreachable, |
| 2081 | } | 2123 | } |
| 2082 | } | 2124 | } |
| ... | @@ -2102,7 +2144,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa | ... | @@ -2102,7 +2144,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa |
| 2102 | try self.genIntMulDivOpMir(switch (signedness) { | 2144 | try self.genIntMulDivOpMir(switch (signedness) { |
| 2103 | .signed => .idiv, | 2145 | .signed => .idiv, |
| 2104 | .unsigned => .div, | 2146 | .unsigned => .div, |
| 2105 | }, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor }); | 2147 | }, Type.isize, .{ .register = dividend }, .{ .register = divisor }); |
| 2106 | | 2148 | |
| 2107 | try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64()); | 2149 | try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64()); |
| 2108 | try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63)); | 2150 | try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63)); |
| ... | @@ -3938,14 +3980,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3938,14 +3980,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 3938 | .register_overflow => |ro| { | 3980 | .register_overflow => |ro| { |
| 3939 | switch (index) { | 3981 | switch (index) { |
| 3940 | // Get wrapped value for overflow operation. | 3982 | // Get wrapped value for overflow operation. |
| 3941 | 0 => if (self.liveness.operandDies(inst, 0)) { | 3983 | 0 => break :result if (self.liveness.operandDies(inst, 0)) |
| 3942 | self.eflags_inst = null; | 3984 | .{ .register = ro.reg } |
| 3943 | break :result .{ .register = ro.reg }; | 3985 | else |
| 3944 | } else break :result try self.copyToRegisterWithInstTracking( | 3986 | try self.copyToRegisterWithInstTracking( |
| 3945 | inst, | 3987 | inst, |
| 3946 | Type.usize, | 3988 | Type.usize, |
| 3947 | .{ .register = ro.reg }, | 3989 | .{ .register = ro.reg }, |
| 3948 | ), | 3990 | ), |
| 3949 | // Get overflow bit. | 3991 | // Get overflow bit. |
| 3950 | 1 => if (self.liveness.operandDies(inst, 0)) { | 3992 | 1 => if (self.liveness.operandDies(inst, 0)) { |
| 3951 | self.eflags_inst = inst; | 3993 | self.eflags_inst = inst; |
| ... | @@ -4318,20 +4360,26 @@ fn genMulDivBinOp( | ... | @@ -4318,20 +4360,26 @@ fn genMulDivBinOp( |
| 4318 | self: *Self, | 4360 | self: *Self, |
| 4319 | tag: Air.Inst.Tag, | 4361 | tag: Air.Inst.Tag, |
| 4320 | maybe_inst: ?Air.Inst.Index, | 4362 | maybe_inst: ?Air.Inst.Index, |
| 4321 | ty: Type, | 4363 | dst_ty: Type, |
| | 4364 | src_ty: Type, |
| 4322 | lhs: MCValue, | 4365 | lhs: MCValue, |
| 4323 | rhs: MCValue, | 4366 | rhs: MCValue, |
| 4324 | ) !MCValue { | 4367 | ) !MCValue { |
| 4325 | if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) { | 4368 | if (dst_ty.zigTypeTag() == .Vector or dst_ty.zigTypeTag() == .Float) { |
| 4326 | return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()}); | 4369 | return self.fail("TODO implement genMulDivBinOp for {}", .{dst_ty.fmtDebug()}); |
| 4327 | } | | |
| 4328 | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); | | |
| 4329 | if (abi_size > 8) { | | |
| 4330 | return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()}); | | |
| 4331 | } | | |
| 4332 | if (tag == .div_float) { | | |
| 4333 | return self.fail("TODO implement genMulDivBinOp for div_float", .{}); | | |
| 4334 | } | 4370 | } |
| | 4371 | const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); |
| | 4372 | const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); |
| | 4373 | if (switch (tag) { |
| | 4374 | else => unreachable, |
| | 4375 | .mul, .mulwrap => dst_abi_size != src_abi_size and dst_abi_size != src_abi_size * 2, |
| | 4376 | .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_abi_size != src_abi_size, |
| | 4377 | } or src_abi_size > 8) return self.fail("TODO implement genMulDivBinOp from {} to {}", .{ |
| | 4378 | src_ty.fmt(self.bin_file.options.module.?), |
| | 4379 | dst_ty.fmt(self.bin_file.options.module.?), |
| | 4380 | }); |
| | 4381 | const ty = if (dst_abi_size <= 8) dst_ty else src_ty; |
| | 4382 | const abi_size = if (dst_abi_size <= 8) dst_abi_size else src_abi_size; |
| 4335 | | 4383 | |
| 4336 | assert(self.register_manager.isRegFree(.rax)); | 4384 | assert(self.register_manager.isRegFree(.rax)); |
| 4337 | assert(self.register_manager.isRegFree(.rdx)); | 4385 | assert(self.register_manager.isRegFree(.rdx)); |
| ... | @@ -4339,9 +4387,7 @@ fn genMulDivBinOp( | ... | @@ -4339,9 +4387,7 @@ fn genMulDivBinOp( |
| 4339 | const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx }); | 4387 | const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx }); |
| 4340 | defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); | 4388 | defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); |
| 4341 | | 4389 | |
| 4342 | const int_info = ty.intInfo(self.target.*); | 4390 | const signedness = ty.intInfo(self.target.*).signedness; |
| 4343 | const signedness = int_info.signedness; | | |
| 4344 | | | |
| 4345 | switch (tag) { | 4391 | switch (tag) { |
| 4346 | .mul, | 4392 | .mul, |
| 4347 | .mulwrap, | 4393 | .mulwrap, |
| ... | @@ -4349,11 +4395,12 @@ fn genMulDivBinOp( | ... | @@ -4349,11 +4395,12 @@ fn genMulDivBinOp( |
| 4349 | .div_trunc, | 4395 | .div_trunc, |
| 4350 | .div_exact, | 4396 | .div_exact, |
| 4351 | => { | 4397 | => { |
| 4352 | const track_inst_rax: ?Air.Inst.Index = switch (tag) { | 4398 | const track_inst_rax = switch (tag) { |
| 4353 | .mul, .mulwrap, .div_exact, .div_trunc => maybe_inst, | 4399 | .mul, .mulwrap => if (dst_abi_size <= 8) maybe_inst else null, |
| | 4400 | .div_exact, .div_trunc => maybe_inst, |
| 4354 | else => null, | 4401 | else => null, |
| 4355 | }; | 4402 | }; |
| 4356 | const track_inst_rdx: ?Air.Inst.Index = switch (tag) { | 4403 | const track_inst_rdx = switch (tag) { |
| 4357 | .rem => maybe_inst, | 4404 | .rem => maybe_inst, |
| 4358 | else => null, | 4405 | else => null, |
| 4359 | }; | 4406 | }; |
| ... | @@ -4373,13 +4420,24 @@ fn genMulDivBinOp( | ... | @@ -4373,13 +4420,24 @@ fn genMulDivBinOp( |
| 4373 | }, | 4420 | }, |
| 4374 | }; | 4421 | }; |
| 4375 | | 4422 | |
| 4376 | try self.genIntMulDivOpMir(mir_tag, ty, .signed, lhs, rhs); | 4423 | try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs); |
| 4377 | | 4424 | |
| 4378 | return .{ .register = registerAlias(switch (tag) { | 4425 | if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) { |
| 4379 | .mul, .mulwrap, .div_trunc, .div_exact => .rax, | 4426 | .mul, .mulwrap, .div_trunc, .div_exact => .rax, |
| 4380 | .rem => .rdx, | 4427 | .rem => .rdx, |
| 4381 | else => unreachable, | 4428 | else => unreachable, |
| 4382 | }, abi_size) }; | 4429 | }, dst_abi_size) }; |
| | 4430 | |
| | 4431 | const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false); |
| | 4432 | try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{ |
| | 4433 | .base = .rbp, |
| | 4434 | .disp = 0 - dst_mcv.stack_offset, |
| | 4435 | }), .rax); |
| | 4436 | try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{ |
| | 4437 | .base = .rbp, |
| | 4438 | .disp = 8 - dst_mcv.stack_offset, |
| | 4439 | }), .rdx); |
| | 4440 | return dst_mcv; |
| 4383 | }, | 4441 | }, |
| 4384 | | 4442 | |
| 4385 | .mod => { | 4443 | .mod => { |
| ... | @@ -4402,7 +4460,7 @@ fn genMulDivBinOp( | ... | @@ -4402,7 +4460,7 @@ fn genMulDivBinOp( |
| 4402 | return result; | 4460 | return result; |
| 4403 | }, | 4461 | }, |
| 4404 | .unsigned => { | 4462 | .unsigned => { |
| 4405 | try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs); | 4463 | try self.genIntMulDivOpMir(.div, ty, lhs, rhs); |
| 4406 | return .{ .register = registerAlias(.rdx, abi_size) }; | 4464 | return .{ .register = registerAlias(.rdx, abi_size) }; |
| 4407 | }, | 4465 | }, |
| 4408 | } | 4466 | } |
| ... | @@ -4441,18 +4499,13 @@ fn genMulDivBinOp( | ... | @@ -4441,18 +4499,13 @@ fn genMulDivBinOp( |
| 4441 | }; | 4499 | }; |
| 4442 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); | 4500 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4443 | | 4501 | |
| 4444 | const result: MCValue = result: { | 4502 | switch (signedness) { |
| 4445 | switch (signedness) { | 4503 | .signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs), |
| 4446 | .signed => break :result try self.genInlineIntDivFloor(ty, lhs, actual_rhs), | 4504 | .unsigned => { |
| 4447 | .unsigned => { | 4505 | try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs); |
| 4448 | try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, actual_rhs); | 4506 | return .{ .register = registerAlias(.rax, abi_size) }; |
| 4449 | break :result MCValue{ | 4507 | }, |
| 4450 | .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))), | 4508 | } |
| 4451 | }; | | |
| 4452 | }, | | |
| 4453 | } | | |
| 4454 | }; | | |
| 4455 | return result; | | |
| 4456 | }, | 4509 | }, |
| 4457 | | 4510 | |
| 4458 | else => unreachable, | 4511 | else => unreachable, |
| ... | @@ -6718,6 +6771,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE | ... | @@ -6718,6 +6771,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE |
| 6718 | } | 6771 | } |
| 6719 | | 6772 | |
| 6720 | fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void { | 6773 | fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void { |
| | 6774 | const base_reg = opts.dest_stack_base orelse .rbp; |
| 6721 | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); | 6775 | const abi_size = @intCast(u32, ty.abiSize(self.target.*)); |
| 6722 | switch (mcv) { | 6776 | switch (mcv) { |
| 6723 | .dead => unreachable, | 6777 | .dead => unreachable, |
| ... | @@ -6733,12 +6787,17 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -6733,12 +6787,17 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6733 | 4 => 0xaaaaaaaa, | 6787 | 4 => 0xaaaaaaaa, |
| 6734 | else => unreachable, | 6788 | else => unreachable, |
| 6735 | }; | 6789 | }; |
| 6736 | return self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ | 6790 | return self.asmMemoryImmediate(.mov, Memory.sib( |
| 6737 | .base = opts.dest_stack_base orelse .rbp, | 6791 | Memory.PtrSize.fromSize(abi_size), |
| 6738 | .disp = -stack_offset, | 6792 | .{ .base = base_reg, .disp = -stack_offset }, |
| 6739 | }), Immediate.u(value)); | 6793 | ), Immediate.u(value)); |
| 6740 | }, | 6794 | }, |
| 6741 | 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts), | 6795 | 8 => return self.genSetStack( |
| | 6796 | ty, |
| | 6797 | stack_offset, |
| | 6798 | .{ .immediate = 0xaaaaaaaaaaaaaaaa }, |
| | 6799 | opts, |
| | 6800 | ), |
| 6742 | else => |x| return self.genInlineMemset( | 6801 | else => |x| return self.genInlineMemset( |
| 6743 | .{ .stack_offset = stack_offset }, | 6802 | .{ .stack_offset = stack_offset }, |
| 6744 | .{ .immediate = 0xaa }, | 6803 | .{ .immediate = 0xaa }, |
| ... | @@ -6766,12 +6825,11 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -6766,12 +6825,11 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6766 | .{}, | 6825 | .{}, |
| 6767 | ); | 6826 | ); |
| 6768 | }, | 6827 | }, |
| 6769 | .eflags => { | 6828 | .eflags => |cc| try self.asmSetccMemory( |
| 6770 | const reg = try self.copyToTmpRegister(ty, mcv); | 6829 | Memory.sib(.byte, .{ .base = base_reg, .disp = -stack_offset }), |
| 6771 | return self.genSetStack(ty, stack_offset, .{ .register = reg }, opts); | 6830 | cc, |
| 6772 | }, | 6831 | ), |
| 6773 | .immediate => |x_big| { | 6832 | .immediate => |imm| { |
| 6774 | const base_reg = opts.dest_stack_base orelse .rbp; | | |
| 6775 | // TODO | 6833 | // TODO |
| 6776 | switch (abi_size) { | 6834 | switch (abi_size) { |
| 6777 | 0 => { | 6835 | 0 => { |
| ... | @@ -6779,13 +6837,13 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -6779,13 +6837,13 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6779 | try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{ | 6837 | try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{ |
| 6780 | .base = base_reg, | 6838 | .base = base_reg, |
| 6781 | .disp = -stack_offset, | 6839 | .disp = -stack_offset, |
| 6782 | }), Immediate.u(@truncate(u8, x_big))); | 6840 | }), Immediate.u(@truncate(u8, imm))); |
| 6783 | }, | 6841 | }, |
| 6784 | 1, 2, 4 => { | 6842 | 1, 2, 4 => { |
| 6785 | const immediate = if (ty.isSignedInt()) | 6843 | const immediate = if (ty.isSignedInt()) |
| 6786 | Immediate.s(@truncate(i32, @bitCast(i64, x_big))) | 6844 | Immediate.s(@truncate(i32, @bitCast(i64, imm))) |
| 6787 | else | 6845 | else |
| 6788 | Immediate.u(@intCast(u32, x_big)); | 6846 | Immediate.u(@intCast(u32, imm)); |
| 6789 | try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ | 6847 | try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ |
| 6790 | .base = base_reg, | 6848 | .base = base_reg, |
| 6791 | .disp = -stack_offset, | 6849 | .disp = -stack_offset, |
| ... | @@ -6795,27 +6853,32 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -6795,27 +6853,32 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 6795 | else => { | 6853 | else => { |
| 6796 | // 64 bit write to memory would take two mov's anyways so we | 6854 | // 64 bit write to memory would take two mov's anyways so we |
| 6797 | // insted just use two 32 bit writes to avoid register allocation | 6855 | // insted just use two 32 bit writes to avoid register allocation |
| 6798 | var offset: i32 = 0; | 6856 | if (std.math.cast(i32, @bitCast(i64, imm))) |small| { |
| 6799 | while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate( | 6857 | try self.asmMemoryImmediate(.mov, Memory.sib( |
| 6800 | .mov, | 6858 | Memory.PtrSize.fromSize(abi_size), |
| 6801 | Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }), | 6859 | .{ .base = base_reg, .disp = -stack_offset }, |
| 6802 | if (ty.isSignedInt()) | 6860 | ), Immediate.s(small)); |
| 6803 | Immediate.s(@truncate( | 6861 | } else { |
| 6804 | i32, | 6862 | var offset: i32 = 0; |
| 6805 | @bitCast(i64, x_big) >> (math.cast(u6, offset * 8) orelse 63), | 6863 | while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate( |
| 6806 | )) | 6864 | .mov, |
| 6807 | else | 6865 | Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }), |
| 6808 | Immediate.u(@truncate( | 6866 | if (ty.isSignedInt()) |
| 6809 | u32, | 6867 | Immediate.s(@truncate( |
| 6810 | if (math.cast(u6, offset * 8)) |shift| x_big >> shift else 0, | 6868 | i32, |
| 6811 | )), | 6869 | @bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63), |
| 6812 | ); | 6870 | )) |
| | 6871 | else |
| | 6872 | Immediate.u(@truncate( |
| | 6873 | u32, |
| | 6874 | if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0, |
| | 6875 | )), |
| | 6876 | ); |
| | 6877 | } |
| 6813 | }, | 6878 | }, |
| 6814 | } | 6879 | } |
| 6815 | }, | 6880 | }, |
| 6816 | .register => |reg| { | 6881 | .register => |reg| { |
| 6817 | const base_reg = opts.dest_stack_base orelse .rbp; | | |
| 6818 | | | |
| 6819 | switch (ty.zigTypeTag()) { | 6882 | switch (ty.zigTypeTag()) { |
| 6820 | .Float => { | 6883 | .Float => { |
| 6821 | if (intrinsicsAllowed(self.target.*, ty)) { | 6884 | if (intrinsicsAllowed(self.target.*, ty)) { |
Jacob Young <jacobly@ziglang.org> 2023-04-02 04:34:14-04:00