authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-05 22:50:38+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-05-05 22:50:38+02:00
log13d1798ea0c2665f09508ac505259e761f8b4e22
tree0c9378078313be90d1e6859ab5ca9f305cfa2a35
parent59905a62f9da9946a797cc35a2523c9929663600
parent90a8817f558bb5c5b4292c666b2ed61b4a415f8d
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11574 from ziglang/stage2-aarch64

stage2,aarch64: basic overflow arithmetic support

6 files changed, 1055 insertions(+), 73 deletions(-)

src/arch/aarch64/CodeGen.zig+584-66
......@@ -102,9 +102,12 @@ air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
102102const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
103103
104104const MCValue = union(enum) {
105 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
106 /// TODO Look into deleting this tag and using `dead` instead, since every use
107 /// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
105 /// No runtime bits. `void` types, empty structs, u0, enums with 1
106 /// tag, etc.
107 ///
108 /// TODO Look into deleting this tag and using `dead` instead,
109 /// since every use of MCValue.none should be instead looking at
110 /// the type and noticing it is 0 bits.
108111 none,
109112 /// Control flow will not allow this value to be observed.
110113 unreach,
......@@ -113,28 +116,56 @@ const MCValue = union(enum) {
113116 /// The value is undefined.
114117 undef,
115118 /// A pointer-sized integer that fits in a register.
116 /// If the type is a pointer, this is the pointer address in virtual address space.
119 ///
120 /// If the type is a pointer, this is the pointer address in
121 /// virtual address space.
117122 immediate: u64,
118123 /// The value is in a target-specific register.
119124 register: Register,
125 /// The value is a tuple { wrapped: u32, overflow: u1 } where
126 /// wrapped is stored in the register and the overflow bit is
127 /// stored in the C flag of the CPSR.
128 ///
129 /// This MCValue is only generated by a add_with_overflow or
130 /// sub_with_overflow instruction operating on u32.
131 register_c_flag: Register,
132 /// The value is a tuple { wrapped: i32, overflow: u1 } where
133 /// wrapped is stored in the register and the overflow bit is
134 /// stored in the V flag of the CPSR.
135 ///
136 /// This MCValue is only generated by a add_with_overflow or
137 /// sub_with_overflow instruction operating on i32.
138 register_v_flag: Register,
120139 /// The value is in memory at a hard-coded address.
121 /// If the type is a pointer, it means the pointer address is at this memory location.
140 ///
141 /// If the type is a pointer, it means the pointer address is at
142 /// this memory location.
122143 memory: u64,
123 /// The value is in memory referenced indirectly via a GOT entry index.
124 /// If the type is a pointer, it means the pointer is referenced indirectly via GOT.
125 /// When lowered, linker will emit relocations of type ARM64_RELOC_GOT_LOAD_PAGE21 and ARM64_RELOC_GOT_LOAD_PAGEOFF12.
144 /// The value is in memory referenced indirectly via a GOT entry
145 /// index.
146 ///
147 /// If the type is a pointer, it means the pointer is referenced
148 /// indirectly via GOT. When lowered, linker will emit
149 /// relocations of type ARM64_RELOC_GOT_LOAD_PAGE21 and
150 /// ARM64_RELOC_GOT_LOAD_PAGEOFF12.
126151 got_load: u32,
127152 /// The value is in memory referenced directly via symbol index.
128 /// If the type is a pointer, it means the pointer is referenced directly via symbol index.
129 /// When lowered, linker will emit a relocation of type ARM64_RELOC_PAGE21 and ARM64_RELOC_PAGEOFF12.
153 ///
154 /// If the type is a pointer, it means the pointer is referenced
155 /// directly via symbol index. When lowered, linker will emit a
156 /// relocation of type ARM64_RELOC_PAGE21 and
157 /// ARM64_RELOC_PAGEOFF12.
130158 direct_load: u32,
131159 /// The value is one of the stack variables.
132 /// If the type is a pointer, it means the pointer address is in the stack at this offset.
160 ///
161 /// If the type is a pointer, it means the pointer address is in
162 /// the stack at this offset.
133163 stack_offset: u32,
134 /// The value is a pointer to one of the stack variables (payload is stack offset).
164 /// The value is a pointer to one of the stack variables (payload
165 /// is stack offset).
135166 ptr_stack_offset: u32,
136 /// The value is in the compare flags assuming an unsigned operation,
137 /// with this operator applied on top of it.
167 /// The value is in the compare flags assuming an unsigned
168 /// operation, with this operator applied on top of it.
138169 compare_flags_unsigned: math.CompareOperator,
139170 /// The value is in the compare flags assuming a signed operation,
140171 /// with this operator applied on top of it.
......@@ -546,8 +577,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
546577 .trunc_float
547578 => try self.airUnaryMath(inst),
548579
549 .add_with_overflow => try self.airAddWithOverflow(inst),
550 .sub_with_overflow => try self.airSubWithOverflow(inst),
580 .add_with_overflow => try self.airOverflow(inst),
581 .sub_with_overflow => try self.airOverflow(inst),
551582 .mul_with_overflow => try self.airMulWithOverflow(inst),
552583 .shl_with_overflow => try self.airShlWithOverflow(inst),
553584
......@@ -716,8 +747,13 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
716747 branch.inst_table.putAssumeCapacity(inst, .dead);
717748 switch (prev_value) {
718749 .register => |reg| {
719 const canon_reg = toCanonicalReg(reg);
720 self.register_manager.freeReg(canon_reg);
750 self.register_manager.freeReg(reg);
751 },
752 .register_c_flag,
753 .register_v_flag,
754 => |reg| {
755 self.register_manager.freeReg(reg);
756 self.compare_flags_inst = null;
721757 },
722758 .compare_flags_signed, .compare_flags_unsigned => {
723759 self.compare_flags_inst = null;
......@@ -857,7 +893,13 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
857893 const stack_mcv = try self.allocRegOrMem(inst, false);
858894 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
859895 const reg_mcv = self.getResolvedInstValue(inst);
860 assert(reg == toCanonicalReg(reg_mcv.register));
896 switch (reg_mcv) {
897 .register,
898 .register_c_flag,
899 .register_v_flag,
900 => |r| assert(reg.id() == r.id()),
901 else => unreachable, // not a register
902 }
861903 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
862904 try branch.inst_table.put(self.gpa, inst, stack_mcv);
863905 try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
......@@ -868,7 +910,14 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
868910fn spillCompareFlagsIfOccupied(self: *Self) !void {
869911 if (self.compare_flags_inst) |inst_to_save| {
870912 const mcv = self.getResolvedInstValue(inst_to_save);
871 assert(mcv == .compare_flags_signed or mcv == .compare_flags_unsigned);
913 switch (mcv) {
914 .compare_flags_signed,
915 .compare_flags_unsigned,
916 .register_c_flag,
917 .register_v_flag,
918 => {},
919 else => unreachable, // mcv doesn't occupy the compare flags
920 }
872921
873922 const new_mcv = try self.allocRegOrMem(inst_to_save, true);
874923 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
......@@ -1245,25 +1294,33 @@ fn binOpRegister(
12451294 };
12461295 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
12471296
1248 const dest_reg = if (maybe_inst) |inst| blk: {
1249 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1297 const dest_reg = switch (mir_tag) {
1298 .cmp_shifted_register => undefined, // cmp has no destination register
1299 else => if (maybe_inst) |inst| blk: {
1300 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
12501301
1251 if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {
1252 break :blk lhs_reg;
1253 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
1254 break :blk rhs_reg;
1255 } else {
1256 const raw_reg = try self.register_manager.allocReg(inst);
1302 if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {
1303 break :blk lhs_reg;
1304 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
1305 break :blk rhs_reg;
1306 } else {
1307 const raw_reg = try self.register_manager.allocReg(inst);
1308 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1309 }
1310 } else blk: {
1311 const raw_reg = try self.register_manager.allocReg(null);
12571312 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1258 }
1259 } else try self.register_manager.allocReg(null);
1313 },
1314 };
12601315
12611316 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
12621317 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
12631318
12641319 const mir_data: Mir.Inst.Data = switch (mir_tag) {
12651320 .add_shifted_register,
1321 .adds_shifted_register,
12661322 .sub_shifted_register,
1323 .subs_shifted_register,
12671324 => .{ .rrr_imm6_shift = .{
12681325 .rd = dest_reg,
12691326 .rn = lhs_reg,
......@@ -1286,6 +1343,13 @@ fn binOpRegister(
12861343 .rn = lhs_reg,
12871344 .rm = rhs_reg,
12881345 } },
1346 .smull,
1347 .umull,
1348 => .{ .rrr = .{
1349 .rd = dest_reg.to64(),
1350 .rn = lhs_reg,
1351 .rm = rhs_reg,
1352 } },
12891353 .and_shifted_register,
12901354 .orr_shifted_register,
12911355 .eor_shifted_register,
......@@ -1368,14 +1432,19 @@ fn binOpImmediate(
13681432 const raw_reg = try self.register_manager.allocReg(inst);
13691433 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
13701434 }
1371 } else try self.register_manager.allocReg(null),
1435 } else blk: {
1436 const raw_reg = try self.register_manager.allocReg(null);
1437 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1438 },
13721439 };
13731440
13741441 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
13751442
13761443 const mir_data: Mir.Inst.Data = switch (mir_tag) {
13771444 .add_immediate,
1445 .adds_immediate,
13781446 .sub_immediate,
1447 .subs_immediate,
13791448 => .{ .rr_imm12_sh = .{
13801449 .rd = dest_reg,
13811450 .rn = lhs_reg,
......@@ -1711,24 +1780,421 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
17111780 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
17121781}
17131782
1714fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1715 _ = inst;
1716 return self.fail("TODO implement airAddWithOverflow for {}", .{self.target.cpu.arch});
1717}
1783fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
1784 const tag = self.air.instructions.items(.tag)[inst];
1785 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1786 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1787 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1788 const lhs = try self.resolveInst(extra.lhs);
1789 const rhs = try self.resolveInst(extra.rhs);
1790 const lhs_ty = self.air.typeOf(extra.lhs);
1791 const rhs_ty = self.air.typeOf(extra.rhs);
1792
1793 const tuple_ty = self.air.typeOfIndex(inst);
1794 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
1795 const tuple_align = tuple_ty.abiAlignment(self.target.*);
1796 const overflow_bit_offset = @intCast(u32, tuple_ty.structFieldOffset(1, self.target.*));
1797
1798 switch (lhs_ty.zigTypeTag()) {
1799 .Vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}),
1800 .Int => {
1801 const mod = self.bin_file.options.module.?;
1802 assert(lhs_ty.eql(rhs_ty, mod));
1803 const int_info = lhs_ty.intInfo(self.target.*);
1804 switch (int_info.bits) {
1805 1...31, 33...63 => {
1806 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
1807
1808 try self.spillCompareFlagsIfOccupied();
1809 self.compare_flags_inst = null;
1810
1811 const base_tag: Air.Inst.Tag = switch (tag) {
1812 .add_with_overflow => .add,
1813 .sub_with_overflow => .sub,
1814 else => unreachable,
1815 };
1816 const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
1817 const dest_reg = dest.register;
1818 self.register_manager.freezeRegs(&.{dest_reg});
1819 defer self.register_manager.unfreezeRegs(&.{dest_reg});
17181820
1719fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1720 _ = inst;
1721 return self.fail("TODO implement airSubWithOverflow for {}", .{self.target.cpu.arch});
1821 const raw_truncated_reg = try self.register_manager.allocReg(null);
1822 const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*));
1823 self.register_manager.freezeRegs(&.{truncated_reg});
1824 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1825
1826 // sbfx/ubfx truncated, dest, #0, #bits
1827 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
1828
1829 // cmp dest, truncated
1830 _ = try self.binOp(.cmp_eq, null, dest, .{ .register = truncated_reg }, Type.usize, Type.usize);
1831
1832 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1833 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags_unsigned = .neq });
1834
1835 break :result MCValue{ .stack_offset = stack_offset };
1836 },
1837 32, 64 => {
1838 // Only say yes if the operation is
1839 // commutative, i.e. we can swap both of the
1840 // operands
1841 const lhs_immediate_ok = switch (tag) {
1842 .add_with_overflow => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
1843 .sub_with_overflow => false,
1844 else => unreachable,
1845 };
1846 const rhs_immediate_ok = switch (tag) {
1847 .add_with_overflow,
1848 .sub_with_overflow,
1849 => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),
1850 else => unreachable,
1851 };
1852
1853 const mir_tag_register: Mir.Inst.Tag = switch (tag) {
1854 .add_with_overflow => .adds_shifted_register,
1855 .sub_with_overflow => .subs_shifted_register,
1856 else => unreachable,
1857 };
1858 const mir_tag_immediate: Mir.Inst.Tag = switch (tag) {
1859 .add_with_overflow => .adds_immediate,
1860 .sub_with_overflow => .subs_immediate,
1861 else => unreachable,
1862 };
1863
1864 try self.spillCompareFlagsIfOccupied();
1865 self.compare_flags_inst = inst;
1866
1867 const dest = blk: {
1868 if (rhs_immediate_ok) {
1869 break :blk try self.binOpImmediate(mir_tag_immediate, null, lhs, rhs, lhs_ty, false);
1870 } else if (lhs_immediate_ok) {
1871 // swap lhs and rhs
1872 break :blk try self.binOpImmediate(mir_tag_immediate, null, rhs, lhs, rhs_ty, true);
1873 } else {
1874 break :blk try self.binOpRegister(mir_tag_register, null, lhs, rhs, lhs_ty, rhs_ty);
1875 }
1876 };
1877
1878 switch (int_info.signedness) {
1879 .unsigned => break :result MCValue{ .register_c_flag = dest.register },
1880 .signed => break :result MCValue{ .register_v_flag = dest.register },
1881 }
1882 },
1883 else => return self.fail("TODO overflow operations on integers > u32/i32", .{}),
1884 }
1885 },
1886 else => unreachable,
1887 }
1888 };
1889 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
17221890}
17231891
17241892fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1725 _ = inst;
1726 return self.fail("TODO implement airMulWithOverflow for {}", .{self.target.cpu.arch});
1893 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1894 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1895 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
1896 const result: MCValue = result: {
1897 const lhs = try self.resolveInst(extra.lhs);
1898 const rhs = try self.resolveInst(extra.rhs);
1899 const lhs_ty = self.air.typeOf(extra.lhs);
1900 const rhs_ty = self.air.typeOf(extra.rhs);
1901
1902 const tuple_ty = self.air.typeOfIndex(inst);
1903 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
1904 const tuple_align = tuple_ty.abiAlignment(self.target.*);
1905 const overflow_bit_offset = @intCast(u32, tuple_ty.structFieldOffset(1, self.target.*));
1906
1907 switch (lhs_ty.zigTypeTag()) {
1908 .Vector => return self.fail("TODO implement mul_with_overflow for vectors", .{}),
1909 .Int => {
1910 const int_info = lhs_ty.intInfo(self.target.*);
1911
1912 if (int_info.bits <= 32) {
1913 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
1914
1915 try self.spillCompareFlagsIfOccupied();
1916 self.compare_flags_inst = null;
1917
1918 const base_tag: Mir.Inst.Tag = switch (int_info.signedness) {
1919 .signed => .smull,
1920 .unsigned => .umull,
1921 };
1922
1923 const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
1924 const dest_reg = dest.register;
1925 self.register_manager.freezeRegs(&.{dest_reg});
1926 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1927
1928 const truncated_reg = try self.register_manager.allocReg(null);
1929 self.register_manager.freezeRegs(&.{truncated_reg});
1930 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1931
1932 try self.truncRegister(
1933 dest_reg.to32(),
1934 truncated_reg.to32(),
1935 int_info.signedness,
1936 int_info.bits,
1937 );
1938
1939 switch (int_info.signedness) {
1940 .signed => {
1941 _ = try self.addInst(.{
1942 .tag = .cmp_extended_register,
1943 .data = .{ .rr_extend_shift = .{
1944 .rn = dest_reg.to64(),
1945 .rm = truncated_reg.to32(),
1946 .ext_type = .sxtw,
1947 .imm3 = 0,
1948 } },
1949 });
1950 },
1951 .unsigned => {
1952 _ = try self.addInst(.{
1953 .tag = .cmp_extended_register,
1954 .data = .{ .rr_extend_shift = .{
1955 .rn = dest_reg.to64(),
1956 .rm = truncated_reg.to32(),
1957 .ext_type = .uxtw,
1958 .imm3 = 0,
1959 } },
1960 });
1961 },
1962 }
1963
1964 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1965 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
1966 .compare_flags_unsigned = .neq,
1967 });
1968
1969 break :result MCValue{ .stack_offset = stack_offset };
1970 } else if (int_info.bits <= 64) {
1971 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
1972
1973 try self.spillCompareFlagsIfOccupied();
1974 self.compare_flags_inst = null;
1975
1976 // TODO this should really be put in a helper similar to `binOpRegister`
1977 const lhs_is_register = lhs == .register;
1978 const rhs_is_register = rhs == .register;
1979
1980 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1981 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
1982
1983 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
1984 const raw_reg = try self.register_manager.allocReg(null);
1985 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1986 self.register_manager.freezeRegs(&.{reg});
1987 break :blk reg;
1988 };
1989 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1990
1991 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
1992 const raw_reg = try self.register_manager.allocReg(null);
1993 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1994 self.register_manager.freezeRegs(&.{reg});
1995 break :blk reg;
1996 };
1997 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1998
1999 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
2000 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
2001
2002 // TODO reuse operands
2003 const dest_reg = blk: {
2004 const raw_reg = try self.register_manager.allocReg(null);
2005 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
2006 self.register_manager.freezeRegs(&.{reg});
2007 break :blk reg;
2008 };
2009 defer self.register_manager.unfreezeRegs(&.{dest_reg});
2010
2011 switch (int_info.signedness) {
2012 .signed => {
2013 // mul dest, lhs, rhs
2014 _ = try self.addInst(.{
2015 .tag = .mul,
2016 .data = .{ .rrr = .{
2017 .rd = dest_reg,
2018 .rn = lhs_reg,
2019 .rm = rhs_reg,
2020 } },
2021 });
2022
2023 const dest_high_reg = try self.register_manager.allocReg(null);
2024 self.register_manager.freezeRegs(&.{dest_high_reg});
2025 defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
2026
2027 // smulh dest_high, lhs, rhs
2028 _ = try self.addInst(.{
2029 .tag = .smulh,
2030 .data = .{ .rrr = .{
2031 .rd = dest_high_reg,
2032 .rn = lhs_reg,
2033 .rm = rhs_reg,
2034 } },
2035 });
2036
2037 // cmp dest_high, dest, asr #63
2038 _ = try self.addInst(.{
2039 .tag = .cmp_shifted_register,
2040 .data = .{ .rr_imm6_shift = .{
2041 .rn = dest_high_reg,
2042 .rm = dest_reg,
2043 .imm6 = 63,
2044 .shift = .asr,
2045 } },
2046 });
2047
2048 const shift: u6 = @intCast(u6, @as(u7, 64) - @intCast(u7, int_info.bits));
2049 if (shift > 0) {
2050 // lsl dest_high, dest, #shift
2051 _ = try self.addInst(.{
2052 .tag = .lsl_immediate,
2053 .data = .{ .rr_shift = .{
2054 .rd = dest_high_reg,
2055 .rn = dest_reg,
2056 .shift = shift,
2057 } },
2058 });
2059
2060 // cmp dest, dest_high, #shift
2061 _ = try self.addInst(.{
2062 .tag = .cmp_shifted_register,
2063 .data = .{ .rr_imm6_shift = .{
2064 .rn = dest_reg,
2065 .rm = dest_high_reg,
2066 .imm6 = shift,
2067 .shift = .asr,
2068 } },
2069 });
2070 }
2071 },
2072 .unsigned => {
2073 const dest_high_reg = try self.register_manager.allocReg(null);
2074 self.register_manager.freezeRegs(&.{dest_high_reg});
2075 defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
2076
2077 // umulh dest_high, lhs, rhs
2078 _ = try self.addInst(.{
2079 .tag = .umulh,
2080 .data = .{ .rrr = .{
2081 .rd = dest_high_reg,
2082 .rn = lhs_reg,
2083 .rm = rhs_reg,
2084 } },
2085 });
2086
2087 // mul dest, lhs, rhs
2088 _ = try self.addInst(.{
2089 .tag = .mul,
2090 .data = .{ .rrr = .{
2091 .rd = dest_reg,
2092 .rn = lhs_reg,
2093 .rm = rhs_reg,
2094 } },
2095 });
2096
2097 _ = try self.binOp(
2098 .cmp_eq,
2099 null,
2100 .{ .register = dest_high_reg },
2101 .{ .immediate = 0 },
2102 Type.usize,
2103 Type.usize,
2104 );
2105
2106 if (int_info.bits < 64) {
2107 // lsr dest_high, dest, #shift
2108 _ = try self.addInst(.{
2109 .tag = .lsr_immediate,
2110 .data = .{ .rr_shift = .{
2111 .rd = dest_high_reg,
2112 .rn = dest_reg,
2113 .shift = @intCast(u6, int_info.bits),
2114 } },
2115 });
2116
2117 _ = try self.binOp(
2118 .cmp_eq,
2119 null,
2120 .{ .register = dest_high_reg },
2121 .{ .immediate = 0 },
2122 Type.usize,
2123 Type.usize,
2124 );
2125 }
2126 },
2127 }
2128
2129 const truncated_reg = try self.register_manager.allocReg(null);
2130 self.register_manager.freezeRegs(&.{truncated_reg});
2131 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
2132
2133 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
2134
2135 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
2136 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
2137 .compare_flags_unsigned = .neq,
2138 });
2139
2140 break :result MCValue{ .stack_offset = stack_offset };
2141 } else return self.fail("TODO implement mul_with_overflow for integers > u64/i64", .{});
2142 },
2143 else => unreachable,
2144 }
2145 };
2146 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
17272147}
17282148
17292149fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1730 _ = inst;
1731 return self.fail("TODO implement airShlWithOverflow for {}", .{self.target.cpu.arch});
2150 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2151 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2152 if (self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2153 const result: MCValue = result: {
2154 const lhs = try self.resolveInst(extra.lhs);
2155 const rhs = try self.resolveInst(extra.rhs);
2156 const lhs_ty = self.air.typeOf(extra.lhs);
2157 const rhs_ty = self.air.typeOf(extra.rhs);
2158
2159 const tuple_ty = self.air.typeOfIndex(inst);
2160 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
2161 const tuple_align = tuple_ty.abiAlignment(self.target.*);
2162 const overflow_bit_offset = @intCast(u32, tuple_ty.structFieldOffset(1, self.target.*));
2163
2164 switch (lhs_ty.zigTypeTag()) {
2165 .Vector => return self.fail("TODO implement shl_with_overflow for vectors", .{}),
2166 .Int => {
2167 const int_info = lhs_ty.intInfo(self.target.*);
2168 if (int_info.bits <= 64) {
2169 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
2170
2171 if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});
2172 defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});
2173
2174 try self.spillCompareFlagsIfOccupied();
2175 self.compare_flags_inst = null;
2176
2177 // lsl dest, lhs, rhs
2178 const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
2179
2180 // asr/lsr reconstructed, dest, rhs
2181 const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);
2182
2183 // cmp lhs, reconstructed
2184 _ = try self.binOp(.cmp_eq, null, lhs, reconstructed, lhs_ty, lhs_ty);
2185
2186 try self.genSetStack(lhs_ty, stack_offset, dest);
2187 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags_unsigned = .neq });
2188
2189 break :result MCValue{ .stack_offset = stack_offset };
2190 } else {
2191 return self.fail("TODO overflow operations on integers > u64/i64", .{});
2192 }
2193 },
2194 else => unreachable,
2195 }
2196 };
2197 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
17322198}
17332199
17342200fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
......@@ -1957,7 +2423,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
19572423 switch (elem_size) {
19582424 else => {
19592425 const dest = try self.allocRegOrMem(inst, true);
1960 const addr = try self.binOp(.ptr_add, null, base_mcv, index_mcv, slice_ty, Type.usize);
2426 const addr = try self.binOp(.ptr_add, null, base_mcv, index_mcv, slice_ptr_field_type, Type.usize);
19612427 try self.load(dest, addr, slice_ptr_field_type);
19622428
19632429 break :result dest;
......@@ -2085,8 +2551,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
20852551 .undef => unreachable,
20862552 .unreach => unreachable,
20872553 .dead => unreachable,
2088 .compare_flags_unsigned => unreachable,
2089 .compare_flags_signed => unreachable,
2554 .compare_flags_unsigned,
2555 .compare_flags_signed,
2556 .register_c_flag,
2557 .register_v_flag,
2558 => unreachable, // cannot hold an address
20902559 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
20912560 .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
20922561 .register => |addr_reg| {
......@@ -2303,8 +2772,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
23032772 .undef => unreachable,
23042773 .unreach => unreachable,
23052774 .dead => unreachable,
2306 .compare_flags_unsigned => unreachable,
2307 .compare_flags_signed => unreachable,
2775 .compare_flags_unsigned,
2776 .compare_flags_signed,
2777 .register_c_flag,
2778 .register_v_flag,
2779 => unreachable, // cannot hold an address
23082780 .immediate => |imm| {
23092781 try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
23102782 },
......@@ -2409,9 +2881,60 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
24092881fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
24102882 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
24112883 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2412 _ = extra;
2413 return self.fail("TODO implement codegen struct_field_val", .{});
2414 //return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
2884 const operand = extra.struct_operand;
2885 const index = extra.field_index;
2886 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2887 const mcv = try self.resolveInst(operand);
2888 const struct_ty = self.air.typeOf(operand);
2889 const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));
2890
2891 switch (mcv) {
2892 .dead, .unreach => unreachable,
2893 .stack_offset => |off| {
2894 break :result MCValue{ .stack_offset = off - struct_field_offset };
2895 },
2896 .memory => |addr| {
2897 break :result MCValue{ .memory = addr + struct_field_offset };
2898 },
2899 .register_c_flag,
2900 .register_v_flag,
2901 => |reg| {
2902 switch (index) {
2903 0 => {
2904 // get wrapped value: return register
2905 break :result MCValue{ .register = reg };
2906 },
2907 1 => {
2908 // TODO return special MCValue condition flags
2909 // get overflow bit: set register to C flag
2910 // resp. V flag
2911 const raw_dest_reg = try self.register_manager.allocReg(null);
2912 const dest_reg = raw_dest_reg.to32();
2913
2914 // C flag: cset reg, cs
2915 // V flag: cset reg, vs
2916 _ = try self.addInst(.{
2917 .tag = .cset,
2918 .data = .{ .r_cond = .{
2919 .rd = dest_reg,
2920 .cond = switch (mcv) {
2921 .register_c_flag => .cs,
2922 .register_v_flag => .vs,
2923 else => unreachable,
2924 },
2925 } },
2926 });
2927
2928 break :result MCValue{ .register = dest_reg };
2929 },
2930 else => unreachable,
2931 }
2932 },
2933 else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}),
2934 }
2935 };
2936
2937 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
24152938}
24162939
24172940fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
......@@ -2451,7 +2974,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
24512974
24522975 switch (mcv) {
24532976 .register => |reg| {
2454 self.register_manager.getRegAssumeFree(toCanonicalReg(reg), inst);
2977 self.register_manager.getRegAssumeFree(reg, inst);
24552978 },
24562979 else => {},
24572980 }
......@@ -2516,15 +3039,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
25163039
25173040 switch (mc_arg) {
25183041 .none => continue,
2519 .undef => unreachable,
2520 .immediate => unreachable,
2521 .unreach => unreachable,
2522 .dead => unreachable,
2523 .memory => unreachable,
2524 .compare_flags_signed => unreachable,
2525 .compare_flags_unsigned => unreachable,
2526 .got_load => unreachable,
2527 .direct_load => unreachable,
25283042 .register => |reg| {
25293043 try self.register_manager.getReg(reg, null);
25303044 try self.genSetReg(arg_ty, reg, arg_mcv);
......@@ -2535,6 +3049,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
25353049 .ptr_stack_offset => {
25363050 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
25373051 },
3052 else => unreachable,
25383053 }
25393054 }
25403055
......@@ -3438,6 +3953,11 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
34383953 else => return self.fail("TODO implement storing other types abi_size={}", .{abi_size}),
34393954 }
34403955 },
3956 .register_c_flag,
3957 .register_v_flag,
3958 => {
3959 return self.fail("TODO implement genSetStack {}", .{mcv});
3960 },
34413961 .got_load,
34423962 .direct_load,
34433963 .memory,
......@@ -3555,7 +4075,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
35554075 .tag = .cset,
35564076 .data = .{ .r_cond = .{
35574077 .rd = reg,
3558 .cond = condition.negate(),
4078 .cond = condition,
35594079 } },
35604080 });
35614081 },
......@@ -3598,6 +4118,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
35984118 .data = .{ .rr = .{ .rd = reg, .rn = src_reg } },
35994119 });
36004120 },
4121 .register_c_flag,
4122 .register_v_flag,
4123 => unreachable, // doesn't fit into a register
36014124 .got_load,
36024125 .direct_load,
36034126 => |sym_index| {
......@@ -4199,8 +4722,3 @@ fn registerAlias(reg: Register, size_bytes: u64) Register {
41994722 unreachable; // TODO handle floating-point registers
42004723 }
42014724}
4202
4203/// Resolves any aliased registers to the 64-bit wide ones.
4204fn toCanonicalReg(reg: Register) Register {
4205 return reg.to64();
4206}
src/arch/aarch64/Emit.zig+77-1
......@@ -77,8 +77,10 @@ pub fn emitMir(
7777 const inst = @intCast(u32, index);
7878 switch (tag) {
7979 .add_immediate => try emit.mirAddSubtractImmediate(inst),
80 .adds_immediate => try emit.mirAddSubtractImmediate(inst),
8081 .cmp_immediate => try emit.mirAddSubtractImmediate(inst),
8182 .sub_immediate => try emit.mirAddSubtractImmediate(inst),
83 .subs_immediate => try emit.mirAddSubtractImmediate(inst),
8284
8385 .asr_register => try emit.mirShiftRegister(inst),
8486 .lsl_register => try emit.mirShiftRegister(inst),
......@@ -104,10 +106,19 @@ pub fn emitMir(
104106 .call_extern => try emit.mirCallExtern(inst),
105107
106108 .eor_immediate => try emit.mirLogicalImmediate(inst),
109 .tst_immediate => try emit.mirLogicalImmediate(inst),
107110
108111 .add_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
112 .adds_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
109113 .cmp_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
110114 .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
115 .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
116
117 .add_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
118 .adds_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
119 .sub_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
120 .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
121 .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
111122
112123 .cset => try emit.mirConditionalSelect(inst),
113124
......@@ -162,6 +173,10 @@ pub fn emitMir(
162173 .movz => try emit.mirMoveWideImmediate(inst),
163174
164175 .mul => try emit.mirDataProcessing3Source(inst),
176 .smulh => try emit.mirDataProcessing3Source(inst),
177 .smull => try emit.mirDataProcessing3Source(inst),
178 .umulh => try emit.mirDataProcessing3Source(inst),
179 .umull => try emit.mirDataProcessing3Source(inst),
165180
166181 .nop => try emit.mirNop(),
167182
......@@ -454,7 +469,9 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
454469 const tag = emit.mir.instructions.items(.tag)[inst];
455470 switch (tag) {
456471 .add_immediate,
472 .adds_immediate,
457473 .sub_immediate,
474 .subs_immediate,
458475 => {
459476 const rr_imm12_sh = emit.mir.instructions.items(.data)[inst].rr_imm12_sh;
460477 const rd = rr_imm12_sh.rd;
......@@ -464,7 +481,9 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
464481
465482 switch (tag) {
466483 .add_immediate => try emit.writeInstruction(Instruction.add(rd, rn, imm12, sh)),
484 .adds_immediate => try emit.writeInstruction(Instruction.adds(rd, rn, imm12, sh)),
467485 .sub_immediate => try emit.writeInstruction(Instruction.sub(rd, rn, imm12, sh)),
486 .subs_immediate => try emit.writeInstruction(Instruction.subs(rd, rn, imm12, sh)),
468487 else => unreachable,
469488 }
470489 },
......@@ -666,6 +685,14 @@ fn mirLogicalImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
666685
667686 switch (tag) {
668687 .eor_immediate => try emit.writeInstruction(Instruction.eorImmediate(rd, rn, imms, immr, n)),
688 .tst_immediate => {
689 const zr: Register = switch (rd.size()) {
690 32 => .wzr,
691 64 => .xzr,
692 else => unreachable,
693 };
694 try emit.writeInstruction(Instruction.andsImmediate(zr, rn, imms, immr, n));
695 },
669696 else => unreachable,
670697 }
671698}
......@@ -674,7 +701,9 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
674701 const tag = emit.mir.instructions.items(.tag)[inst];
675702 switch (tag) {
676703 .add_shifted_register,
704 .adds_shifted_register,
677705 .sub_shifted_register,
706 .subs_shifted_register,
678707 => {
679708 const rrr_imm6_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_shift;
680709 const rd = rrr_imm6_shift.rd;
......@@ -685,7 +714,9 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
685714
686715 switch (tag) {
687716 .add_shifted_register => try emit.writeInstruction(Instruction.addShiftedRegister(rd, rn, rm, shift, imm6)),
717 .adds_shifted_register => try emit.writeInstruction(Instruction.addsShiftedRegister(rd, rn, rm, shift, imm6)),
688718 .sub_shifted_register => try emit.writeInstruction(Instruction.subShiftedRegister(rd, rn, rm, shift, imm6)),
719 .subs_shifted_register => try emit.writeInstruction(Instruction.subsShiftedRegister(rd, rn, rm, shift, imm6)),
689720 else => unreachable,
690721 }
691722 },
......@@ -707,6 +738,47 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
707738 }
708739}
709740
741fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
742 const tag = emit.mir.instructions.items(.tag)[inst];
743 switch (tag) {
744 .add_extended_register,
745 .adds_extended_register,
746 .sub_extended_register,
747 .subs_extended_register,
748 => {
749 const rrr_extend_shift = emit.mir.instructions.items(.data)[inst].rrr_extend_shift;
750 const rd = rrr_extend_shift.rd;
751 const rn = rrr_extend_shift.rn;
752 const rm = rrr_extend_shift.rm;
753 const ext_type = rrr_extend_shift.ext_type;
754 const imm3 = rrr_extend_shift.imm3;
755
756 switch (tag) {
757 .add_extended_register => try emit.writeInstruction(Instruction.addExtendedRegister(rd, rn, rm, ext_type, imm3)),
758 .adds_extended_register => try emit.writeInstruction(Instruction.addsExtendedRegister(rd, rn, rm, ext_type, imm3)),
759 .sub_extended_register => try emit.writeInstruction(Instruction.subExtendedRegister(rd, rn, rm, ext_type, imm3)),
760 .subs_extended_register => try emit.writeInstruction(Instruction.subsExtendedRegister(rd, rn, rm, ext_type, imm3)),
761 else => unreachable,
762 }
763 },
764 .cmp_extended_register => {
765 const rr_extend_shift = emit.mir.instructions.items(.data)[inst].rr_extend_shift;
766 const rn = rr_extend_shift.rn;
767 const rm = rr_extend_shift.rm;
768 const ext_type = rr_extend_shift.ext_type;
769 const imm3 = rr_extend_shift.imm3;
770 const zr: Register = switch (rn.size()) {
771 32 => .wzr,
772 64 => .xzr,
773 else => unreachable,
774 };
775
776 try emit.writeInstruction(Instruction.subsExtendedRegister(zr, rn, rm, ext_type, imm3));
777 },
778 else => unreachable,
779 }
780}
781
710782fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
711783 const tag = emit.mir.instructions.items(.tag)[inst];
712784 switch (tag) {
......@@ -717,7 +789,7 @@ fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
717789 64 => .xzr,
718790 else => unreachable,
719791 };
720 try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond));
792 try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond.negate()));
721793 },
722794 else => unreachable,
723795 }
......@@ -988,6 +1060,10 @@ fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
9881060
9891061 switch (tag) {
9901062 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1063 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
1064 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
1065 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
1066 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
9911067 else => unreachable,
9921068 }
9931069}
src/arch/aarch64/Mir.zig+47
......@@ -26,8 +26,16 @@ pub const Inst = struct {
2626 pub const Tag = enum(u16) {
2727 /// Add (immediate)
2828 add_immediate,
29 /// Add, update condition flags (immediate)
30 adds_immediate,
2931 /// Add (shifted register)
3032 add_shifted_register,
33 /// Add, update condition flags (shifted register)
34 adds_shifted_register,
35 /// Add (extended register)
36 add_extended_register,
37 /// Add, update condition flags (extended register)
38 adds_extended_register,
3139 /// Bitwise AND (shifted register)
3240 and_shifted_register,
3341 /// Arithmetic Shift Right (immediate)
......@@ -52,6 +60,8 @@ pub const Inst = struct {
5260 cmp_immediate,
5361 /// Compare (shifted register)
5462 cmp_shifted_register,
63 /// Compare (extended register)
64 cmp_extended_register,
5565 /// Conditional set
5666 cset,
5767 /// Pseudo-instruction: End of prologue
......@@ -142,6 +152,10 @@ pub const Inst = struct {
142152 ret,
143153 /// Signed bitfield extract
144154 sbfx,
155 /// Signed multiply high
156 smulh,
157 /// Signed multiply long
158 smull,
145159 /// Signed extend byte
146160 sxtb,
147161 /// Signed extend halfword
......@@ -170,12 +184,26 @@ pub const Inst = struct {
170184 strh_register,
171185 /// Subtract (immediate)
172186 sub_immediate,
187 /// Subtract, update condition flags (immediate)
188 subs_immediate,
173189 /// Subtract (shifted register)
174190 sub_shifted_register,
191 /// Subtract, update condition flags (shifted register)
192 subs_shifted_register,
193 /// Subtract (extended register)
194 sub_extended_register,
195 /// Subtract, update condition flags (extended register)
196 subs_extended_register,
175197 /// Supervisor Call
176198 svc,
199 /// Test bits (immediate)
200 tst_immediate,
177201 /// Unsigned bitfield extract
178202 ubfx,
203 /// Unsigned multiply high
204 umulh,
205 /// Unsigned multiply long
206 umull,
179207 /// Unsigned extend byte
180208 uxtb,
181209 /// Unsigned extend halfword
......@@ -282,6 +310,15 @@ pub const Inst = struct {
282310 imm6: u6,
283311 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
284312 },
313 /// Two registers with sign-extension (extension type and 3-bit shift amount)
314 ///
315 /// Used by e.g. cmp_extended_register
316 rr_extend_shift: struct {
317 rn: Register,
318 rm: Register,
319 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
320 imm3: u3,
321 },
285322 /// Two registers and a shift (logical instruction version)
286323 /// (shift type and 6-bit amount)
287324 ///
......@@ -338,6 +375,16 @@ pub const Inst = struct {
338375 imm6: u6,
339376 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
340377 },
378 /// Three registers with sign-extension (extension type and 3-bit shift amount)
379 ///
380 /// Used by e.g. add_extended_register
381 rrr_extend_shift: struct {
382 rd: Register,
383 rn: Register,
384 rm: Register,
385 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
386 imm3: u3,
387 },
341388 /// Three registers and a shift (logical instruction version)
342389 /// (shift type and 6-bit amount)
343390 ///
src/arch/aarch64/bits.zig+144
......@@ -330,6 +330,17 @@ pub const Instruction = union(enum) {
330330 op: u1,
331331 sf: u1,
332332 },
333 add_subtract_extended_register: packed struct {
334 rd: u5,
335 rn: u5,
336 imm3: u3,
337 option: u3,
338 rm: u5,
339 fixed: u8 = 0b01011_00_1,
340 s: u1,
341 op: u1,
342 sf: u1,
343 },
333344 conditional_branch: struct {
334345 cond: u4,
335346 o0: u1,
......@@ -495,6 +506,7 @@ pub const Instruction = union(enum) {
495506 .logical_immediate => |v| @bitCast(u32, v),
496507 .bitfield => |v| @bitCast(u32, v),
497508 .add_subtract_shifted_register => |v| @bitCast(u32, v),
509 .add_subtract_extended_register => |v| @bitCast(u32, v),
498510 // TODO once packed structs work, this can be refactored
499511 .conditional_branch => |v| @as(u32, v.cond) | (@as(u32, v.o0) << 4) | (@as(u32, v.imm19) << 5) | (@as(u32, v.o1) << 24) | (@as(u32, v.fixed) << 25),
500512 .compare_and_branch => |v| @as(u32, v.rt) | (@as(u32, v.imm19) << 5) | (@as(u32, v.op) << 24) | (@as(u32, v.fixed) << 25) | (@as(u32, v.sf) << 31),
......@@ -1006,6 +1018,44 @@ pub const Instruction = union(enum) {
10061018 };
10071019 }
10081020
1021 pub const AddSubtractExtendedRegisterOption = enum(u3) {
1022 uxtb,
1023 uxth,
1024 uxtw,
1025 uxtx, // serves also as lsl
1026 sxtb,
1027 sxth,
1028 sxtw,
1029 sxtx,
1030 };
1031
1032 fn addSubtractExtendedRegister(
1033 op: u1,
1034 s: u1,
1035 rd: Register,
1036 rn: Register,
1037 rm: Register,
1038 extend: AddSubtractExtendedRegisterOption,
1039 imm3: u3,
1040 ) Instruction {
1041 return Instruction{
1042 .add_subtract_extended_register = .{
1043 .rd = rd.enc(),
1044 .rn = rn.enc(),
1045 .imm3 = imm3,
1046 .option = @enumToInt(extend),
1047 .rm = rm.enc(),
1048 .s = s,
1049 .op = op,
1050 .sf = switch (rd.size()) {
1051 32 => 0b0,
1052 64 => 0b1,
1053 else => unreachable, // unexpected register size
1054 },
1055 },
1056 };
1057 }
1058
10091059 fn conditionalBranch(
10101060 o0: u1,
10111061 o1: u1,
......@@ -1524,6 +1574,48 @@ pub const Instruction = union(enum) {
15241574 return addSubtractShiftedRegister(0b1, 0b1, shift, rd, rn, rm, imm6);
15251575 }
15261576
1577 // Add/subtract (extended register)
1578
1579 pub fn addExtendedRegister(
1580 rd: Register,
1581 rn: Register,
1582 rm: Register,
1583 extend: AddSubtractExtendedRegisterOption,
1584 imm3: u3,
1585 ) Instruction {
1586 return addSubtractExtendedRegister(0b0, 0b0, rd, rn, rm, extend, imm3);
1587 }
1588
1589 pub fn addsExtendedRegister(
1590 rd: Register,
1591 rn: Register,
1592 rm: Register,
1593 extend: AddSubtractExtendedRegisterOption,
1594 imm3: u3,
1595 ) Instruction {
1596 return addSubtractExtendedRegister(0b0, 0b1, rd, rn, rm, extend, imm3);
1597 }
1598
1599 pub fn subExtendedRegister(
1600 rd: Register,
1601 rn: Register,
1602 rm: Register,
1603 extend: AddSubtractExtendedRegisterOption,
1604 imm3: u3,
1605 ) Instruction {
1606 return addSubtractExtendedRegister(0b1, 0b0, rd, rn, rm, extend, imm3);
1607 }
1608
1609 pub fn subsExtendedRegister(
1610 rd: Register,
1611 rn: Register,
1612 rm: Register,
1613 extend: AddSubtractExtendedRegisterOption,
1614 imm3: u3,
1615 ) Instruction {
1616 return addSubtractExtendedRegister(0b1, 0b1, rd, rn, rm, extend, imm3);
1617 }
1618
15271619 // Conditional branch
15281620
15291621 pub fn bCond(cond: Condition, offset: i21) Instruction {
......@@ -1564,6 +1656,16 @@ pub const Instruction = union(enum) {
15641656 return dataProcessing3Source(0b00, 0b000, 0b0, rd, rn, rm, ra);
15651657 }
15661658
1659 pub fn smaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1660 assert(rd.size() == 64 and rn.size() == 32 and rm.size() == 32 and ra.size() == 64);
1661 return dataProcessing3Source(0b00, 0b001, 0b0, rd, rn, rm, ra);
1662 }
1663
1664 pub fn umaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1665 assert(rd.size() == 64 and rn.size() == 32 and rm.size() == 32 and ra.size() == 64);
1666 return dataProcessing3Source(0b00, 0b101, 0b0, rd, rn, rm, ra);
1667 }
1668
15671669 pub fn msub(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
15681670 return dataProcessing3Source(0b00, 0b000, 0b1, rd, rn, rm, ra);
15691671 }
......@@ -1572,6 +1674,24 @@ pub const Instruction = union(enum) {
15721674 return madd(rd, rn, rm, .xzr);
15731675 }
15741676
1677 pub fn smull(rd: Register, rn: Register, rm: Register) Instruction {
1678 return smaddl(rd, rn, rm, .xzr);
1679 }
1680
1681 pub fn smulh(rd: Register, rn: Register, rm: Register) Instruction {
1682 assert(rd.size() == 64);
1683 return dataProcessing3Source(0b00, 0b010, 0b0, rd, rn, rm, .xzr);
1684 }
1685
1686 pub fn umull(rd: Register, rn: Register, rm: Register) Instruction {
1687 return umaddl(rd, rn, rm, .xzr);
1688 }
1689
1690 pub fn umulh(rd: Register, rn: Register, rm: Register) Instruction {
1691 assert(rd.size() == 64);
1692 return dataProcessing3Source(0b00, 0b110, 0b0, rd, rn, rm, .xzr);
1693 }
1694
15751695 pub fn mneg(rd: Register, rn: Register, rm: Register) Instruction {
15761696 return msub(rd, rn, rm, .xzr);
15771697 }
......@@ -1790,6 +1910,30 @@ test "serialize instructions" {
17901910 .inst = Instruction.lsrImmediate(.x4, .x2, 63),
17911911 .expected = 0b1_10_100110_1_111111_111111_00010_00100,
17921912 },
1913 .{ // umull x0, w0, w1
1914 .inst = Instruction.umull(.x0, .w0, .w1),
1915 .expected = 0b1_00_11011_1_01_00001_0_11111_00000_00000,
1916 },
1917 .{ // smull x0, w0, w1
1918 .inst = Instruction.smull(.x0, .w0, .w1),
1919 .expected = 0b1_00_11011_0_01_00001_0_11111_00000_00000,
1920 },
1921 .{ // tst x0, #0xffffffff00000000
1922 .inst = Instruction.andsImmediate(.xzr, .x0, 0b011111, 0b100000, 0b1),
1923 .expected = 0b1_11_100100_1_100000_011111_00000_11111,
1924 },
1925 .{ // umulh x0, x1, x2
1926 .inst = Instruction.umulh(.x0, .x1, .x2),
1927 .expected = 0b1_00_11011_1_10_00010_0_11111_00001_00000,
1928 },
1929 .{ // smulh x0, x1, x2
1930 .inst = Instruction.smulh(.x0, .x1, .x2),
1931 .expected = 0b1_00_11011_0_10_00010_0_11111_00001_00000,
1932 },
1933 .{ // adds x0, x1, x2, sxtx
1934 .inst = Instruction.addsExtendedRegister(.x0, .x1, .x2, .sxtx, 0),
1935 .expected = 0b1_0_1_01011_00_1_00010_111_000_00001_00000,
1936 },
17931937 };
17941938
17951939 for (testcases) |case| {
src/arch/arm/CodeGen.zig+1-1
......@@ -1989,7 +1989,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
19891989 },
19901990 else => {
19911991 const dest = try self.allocRegOrMem(inst, true);
1992 const addr = try self.binOp(.ptr_add, null, base_mcv, index_mcv, slice_ty, Type.usize);
1992 const addr = try self.binOp(.ptr_add, null, base_mcv, index_mcv, slice_ptr_field_type, Type.usize);
19931993 try self.load(dest, addr, slice_ptr_field_type);
19941994
19951995 break :result dest;
test/behavior/math.zig+202-5
......@@ -624,7 +624,6 @@ test "128-bit multiplication" {
624624
625625test "@addWithOverflow" {
626626 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
627 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
628627
629628 var result: u8 = undefined;
630629 try expect(@addWithOverflow(u8, 250, 100, &result));
......@@ -665,9 +664,8 @@ test "small int addition" {
665664 try expect(result == 0);
666665}
667666
668test "@mulWithOverflow" {
667test "basic @mulWithOverflow" {
669668 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
670 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
671669
672670 var result: u8 = undefined;
673671 try expect(@mulWithOverflow(u8, 86, 3, &result));
......@@ -685,9 +683,209 @@ test "@mulWithOverflow" {
685683 try expect(result == 236);
686684}
687685
686// TODO migrate to this for all backends once they handle more cases
687test "extensive @mulWithOverflow" {
688 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
689 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
690 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
691 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
692
693 {
694 var a: u5 = 3;
695 var b: u5 = 10;
696 var res: u5 = undefined;
697 try expect(!@mulWithOverflow(u5, a, b, &res));
698 try expect(res == 30);
699
700 b = 11;
701 try expect(@mulWithOverflow(u5, a, b, &res));
702 try expect(res == 1);
703 }
704
705 {
706 var a: i5 = 3;
707 var b: i5 = -5;
708 var res: i5 = undefined;
709 try expect(!@mulWithOverflow(i5, a, b, &res));
710 try expect(res == -15);
711
712 b = -6;
713 try expect(@mulWithOverflow(i5, a, b, &res));
714 try expect(res == 14);
715 }
716
717 {
718 var a: u8 = 3;
719 var b: u8 = 85;
720 var res: u8 = undefined;
721
722 try expect(!@mulWithOverflow(u8, a, b, &res));
723 try expect(res == 255);
724
725 b = 86;
726 try expect(@mulWithOverflow(u8, a, b, &res));
727 try expect(res == 2);
728 }
729
730 {
731 var a: i8 = 3;
732 var b: i8 = -42;
733 var res: i8 = undefined;
734 try expect(!@mulWithOverflow(i8, a, b, &res));
735 try expect(res == -126);
736
737 b = -43;
738 try expect(@mulWithOverflow(i8, a, b, &res));
739 try expect(res == 127);
740 }
741
742 {
743 var a: u14 = 3;
744 var b: u14 = 0x1555;
745 var res: u14 = undefined;
746 try expect(!@mulWithOverflow(u14, a, b, &res));
747 try expect(res == 0x3fff);
748
749 b = 0x1556;
750 try expect(@mulWithOverflow(u14, a, b, &res));
751 try expect(res == 2);
752 }
753
754 {
755 var a: i14 = 3;
756 var b: i14 = -0xaaa;
757 var res: i14 = undefined;
758 try expect(!@mulWithOverflow(i14, a, b, &res));
759 try expect(res == -0x1ffe);
760
761 b = -0xaab;
762 try expect(@mulWithOverflow(i14, a, b, &res));
763 try expect(res == 0x1fff);
764 }
765
766 {
767 var a: u16 = 3;
768 var b: u16 = 0x5555;
769 var res: u16 = undefined;
770 try expect(!@mulWithOverflow(u16, a, b, &res));
771 try expect(res == 0xffff);
772
773 b = 0x5556;
774 try expect(@mulWithOverflow(u16, a, b, &res));
775 try expect(res == 2);
776 }
777
778 {
779 var a: i16 = 3;
780 var b: i16 = -0x2aaa;
781 var res: i16 = undefined;
782 try expect(!@mulWithOverflow(i16, a, b, &res));
783 try expect(res == -0x7ffe);
784
785 b = -0x2aab;
786 try expect(@mulWithOverflow(i16, a, b, &res));
787 try expect(res == 0x7fff);
788 }
789
790 {
791 var a: u30 = 3;
792 var b: u30 = 0x15555555;
793 var res: u30 = undefined;
794 try expect(!@mulWithOverflow(u30, a, b, &res));
795 try expect(res == 0x3fffffff);
796
797 b = 0x15555556;
798 try expect(@mulWithOverflow(u30, a, b, &res));
799 try expect(res == 2);
800 }
801
802 {
803 var a: i30 = 3;
804 var b: i30 = -0xaaaaaaa;
805 var res: i30 = undefined;
806 try expect(!@mulWithOverflow(i30, a, b, &res));
807 try expect(res == -0x1ffffffe);
808
809 b = -0xaaaaaab;
810 try expect(@mulWithOverflow(i30, a, b, &res));
811 try expect(res == 0x1fffffff);
812 }
813
814 {
815 var a: u32 = 3;
816 var b: u32 = 0x55555555;
817 var res: u32 = undefined;
818 try expect(!@mulWithOverflow(u32, a, b, &res));
819 try expect(res == 0xffffffff);
820
821 b = 0x55555556;
822 try expect(@mulWithOverflow(u32, a, b, &res));
823 try expect(res == 2);
824 }
825
826 {
827 var a: i32 = 3;
828 var b: i32 = -0x2aaaaaaa;
829 var res: i32 = undefined;
830 try expect(!@mulWithOverflow(i32, a, b, &res));
831 try expect(res == -0x7ffffffe);
832
833 b = -0x2aaaaaab;
834 try expect(@mulWithOverflow(i32, a, b, &res));
835 try expect(res == 0x7fffffff);
836 }
837
838 {
839 var a: u62 = 3;
840 var b: u62 = 0x1555555555555555;
841 var res: u62 = undefined;
842 try expect(!@mulWithOverflow(u62, a, b, &res));
843 try expect(res == 0x3fffffffffffffff);
844
845 b = 0x1555555555555556;
846 try expect(@mulWithOverflow(u62, a, b, &res));
847 try expect(res == 2);
848 }
849
850 {
851 var a: i62 = 3;
852 var b: i62 = -0xaaaaaaaaaaaaaaa;
853 var res: i62 = undefined;
854 try expect(!@mulWithOverflow(i62, a, b, &res));
855 try expect(res == -0x1ffffffffffffffe);
856
857 b = -0xaaaaaaaaaaaaaab;
858 try expect(@mulWithOverflow(i62, a, b, &res));
859 try expect(res == 0x1fffffffffffffff);
860 }
861
862 {
863 var a: u64 = 3;
864 var b: u64 = 0x5555555555555555;
865 var res: u64 = undefined;
866 try expect(!@mulWithOverflow(u64, a, b, &res));
867 try expect(res == 0xffffffffffffffff);
868
869 b = 0x5555555555555556;
870 try expect(@mulWithOverflow(u64, a, b, &res));
871 try expect(res == 2);
872 }
873
874 {
875 var a: i64 = 3;
876 var b: i64 = -0x2aaaaaaaaaaaaaaa;
877 var res: i64 = undefined;
878 try expect(!@mulWithOverflow(i64, a, b, &res));
879 try expect(res == -0x7ffffffffffffffe);
880
881 b = -0x2aaaaaaaaaaaaaab;
882 try expect(@mulWithOverflow(i64, a, b, &res));
883 try expect(res == 0x7fffffffffffffff);
884 }
885}
886
688887test "@subWithOverflow" {
689888 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
690 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
691889
692890 var result: u8 = undefined;
693891 try expect(@subWithOverflow(u8, 1, 2, &result));
......@@ -707,7 +905,6 @@ test "@subWithOverflow" {
707905test "@shlWithOverflow" {
708906 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
709907 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
710 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
711908
712909 var result: u16 = undefined;
713910 try expect(@shlWithOverflow(u16, 0b0010111111111111, 3, &result));