authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-08-05 16:56:27-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-08-05 16:56:27-07:00
log40babaa537329c8425579a47ba4767f298f5d869
treefdd4b5bd0a464867d3ae934d65421cf9ae4b870c
parentab6b53a259180aab9c9fc4c57d981ba9ecd6e2fa
parentf46c80b267396d02b5008bf8c426e0eb886a05d2
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12338 from joachimschmidt557/stage2-aarch64

stage2 AArch64: fixes for removing builtin.stage2_arch

4 files changed, 844 insertions(+), 171 deletions(-)

src/arch/aarch64/CodeGen.zig+700-161
...@@ -166,10 +166,12 @@ const MCValue = union(enum) {...@@ -166,10 +166,12 @@ const MCValue = union(enum) {
166 /// the type is u1) or true (if the type in bool) iff the166 /// the type is u1) or true (if the type in bool) iff the
167 /// specified condition is true.167 /// specified condition is true.
168 condition_flags: Condition,168 condition_flags: Condition,
169 /// The value is a function argument passed via the stack.
170 stack_argument_offset: u32,
169171
170 fn isMemory(mcv: MCValue) bool {172 fn isMemory(mcv: MCValue) bool {
171 return switch (mcv) {173 return switch (mcv) {
172 .memory, .stack_offset => true,174 .memory, .stack_offset, .stack_argument_offset => true,
173 else => false,175 else => false,
174 };176 };
175 }177 }
...@@ -192,6 +194,7 @@ const MCValue = union(enum) {...@@ -192,6 +194,7 @@ const MCValue = union(enum) {
192 .condition_flags,194 .condition_flags,
193 .ptr_stack_offset,195 .ptr_stack_offset,
194 .undef,196 .undef,
197 .stack_argument_offset,
195 => false,198 => false,
196199
197 .register,200 .register,
...@@ -337,6 +340,7 @@ pub fn generate(...@@ -337,6 +340,7 @@ pub fn generate(
337 .prev_di_line = module_fn.lbrace_line,340 .prev_di_line = module_fn.lbrace_line,
338 .prev_di_column = module_fn.lbrace_column,341 .prev_di_column = module_fn.lbrace_column,
339 .stack_size = mem.alignForwardGeneric(u32, function.max_end_stack, function.stack_align),342 .stack_size = mem.alignForwardGeneric(u32, function.max_end_stack, function.stack_align),
343 .saved_regs_stack_space = function.saved_regs_stack_space,
340 };344 };
341 defer emit.deinit();345 defer emit.deinit();
342346
...@@ -414,6 +418,23 @@ fn gen(self: *Self) !void {...@@ -414,6 +418,23 @@ fn gen(self: *Self) !void {
414 // sub sp, sp, #reloc418 // sub sp, sp, #reloc
415 const backpatch_reloc = try self.addNop();419 const backpatch_reloc = try self.addNop();
416420
421 if (self.ret_mcv == .stack_offset) {
422 // The address of where to store the return value is in x0
423 // (or w0 when pointer size is 32 bits). As this register
424 // might get overwritten along the way, save the address
425 // to the stack.
426 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
427 const ptr_bytes = @divExact(ptr_bits, 8);
428 const ret_ptr_reg = registerAlias(.x0, ptr_bytes);
429
430 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset, ptr_bytes) + ptr_bytes;
431 self.next_stack_offset = stack_offset;
432 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
433
434 try self.genSetStack(Type.usize, stack_offset, MCValue{ .register = ret_ptr_reg });
435 self.ret_mcv = MCValue{ .stack_offset = stack_offset };
436 }
437
417 _ = try self.addInst(.{438 _ = try self.addInst(.{
418 .tag = .dbg_prologue_end,439 .tag = .dbg_prologue_end,
419 .data = .{ .nop = {} },440 .data = .{ .nop = {} },
...@@ -540,33 +561,38 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -540,33 +561,38 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
540561
541 switch (air_tags[inst]) {562 switch (air_tags[inst]) {
542 // zig fmt: off563 // zig fmt: off
543 .add => try self.airBinOp(inst, .add),564 .add => try self.airBinOp(inst, .add),
544 .addwrap => try self.airBinOp(inst, .addwrap),565 .addwrap => try self.airBinOp(inst, .addwrap),
545 .sub => try self.airBinOp(inst, .sub),566 .sub => try self.airBinOp(inst, .sub),
546 .subwrap => try self.airBinOp(inst, .subwrap),567 .subwrap => try self.airBinOp(inst, .subwrap),
547 .mul => try self.airBinOp(inst, .mul),568 .mul => try self.airBinOp(inst, .mul),
548 .mulwrap => try self.airBinOp(inst, .mulwrap),569 .mulwrap => try self.airBinOp(inst, .mulwrap),
549 .shl => try self.airBinOp(inst, .shl),570 .shl => try self.airBinOp(inst, .shl),
550 .shl_exact => try self.airBinOp(inst, .shl_exact),571 .shl_exact => try self.airBinOp(inst, .shl_exact),
551 .bool_and => try self.airBinOp(inst, .bool_and),572 .bool_and => try self.airBinOp(inst, .bool_and),
552 .bool_or => try self.airBinOp(inst, .bool_or),573 .bool_or => try self.airBinOp(inst, .bool_or),
553 .bit_and => try self.airBinOp(inst, .bit_and),574 .bit_and => try self.airBinOp(inst, .bit_and),
554 .bit_or => try self.airBinOp(inst, .bit_or),575 .bit_or => try self.airBinOp(inst, .bit_or),
555 .xor => try self.airBinOp(inst, .xor),576 .xor => try self.airBinOp(inst, .xor),
556 .shr => try self.airBinOp(inst, .shr),577 .shr => try self.airBinOp(inst, .shr),
557 .shr_exact => try self.airBinOp(inst, .shr_exact),578 .shr_exact => try self.airBinOp(inst, .shr_exact),
558579 .div_float => try self.airBinOp(inst, .div_float),
559 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),580 .div_trunc => try self.airBinOp(inst, .div_trunc),
560 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),581 .div_floor => try self.airBinOp(inst, .div_floor),
582 .div_exact => try self.airBinOp(inst, .div_exact),
583 .rem => try self.airBinOp(inst, .rem),
584 .mod => try self.airBinOp(inst, .mod),
585
586 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
587 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
588
589 .min => try self.airMin(inst),
590 .max => try self.airMax(inst),
561591
562 .add_sat => try self.airAddSat(inst),592 .add_sat => try self.airAddSat(inst),
563 .sub_sat => try self.airSubSat(inst),593 .sub_sat => try self.airSubSat(inst),
564 .mul_sat => try self.airMulSat(inst),594 .mul_sat => try self.airMulSat(inst),
565 .rem => try self.airRem(inst),
566 .mod => try self.airMod(inst),
567 .shl_sat => try self.airShlSat(inst),595 .shl_sat => try self.airShlSat(inst),
568 .min => try self.airMin(inst),
569 .max => try self.airMax(inst),
570 .slice => try self.airSlice(inst),596 .slice => try self.airSlice(inst),
571597
572 .sqrt,598 .sqrt,
...@@ -591,8 +617,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -591,8 +617,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
591 .mul_with_overflow => try self.airMulWithOverflow(inst),617 .mul_with_overflow => try self.airMulWithOverflow(inst),
592 .shl_with_overflow => try self.airShlWithOverflow(inst),618 .shl_with_overflow => try self.airShlWithOverflow(inst),
593619
594 .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
595
596 .cmp_lt => try self.airCmp(inst, .lt),620 .cmp_lt => try self.airCmp(inst, .lt),
597 .cmp_lte => try self.airCmp(inst, .lte),621 .cmp_lte => try self.airCmp(inst, .lte),
598 .cmp_eq => try self.airCmp(inst, .eq),622 .cmp_eq => try self.airCmp(inst, .eq),
...@@ -1008,17 +1032,43 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1008,17 +1032,43 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1008 if (self.liveness.isUnused(inst))1032 if (self.liveness.isUnused(inst))
1009 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });1033 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
10101034
1011 const operand_ty = self.air.typeOf(ty_op.operand);1035 const operand = ty_op.operand;
1012 const operand = try self.resolveInst(ty_op.operand);1036 const operand_mcv = try self.resolveInst(operand);
1013 const info_a = operand_ty.intInfo(self.target.*);1037 const operand_ty = self.air.typeOf(operand);
1014 const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);1038 const operand_info = operand_ty.intInfo(self.target.*);
1015 if (info_a.signedness != info_b.signedness)1039
1016 return self.fail("TODO gen intcast sign safety in semantic analysis", .{});1040 const dest_ty = self.air.typeOfIndex(inst);
1041 const dest_abi_size = dest_ty.abiSize(self.target.*);
1042 const dest_info = dest_ty.intInfo(self.target.*);
1043
1044 const result: MCValue = result: {
1045 const operand_lock: ?RegisterLock = switch (operand_mcv) {
1046 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1047 else => null,
1048 };
1049 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
1050
1051 const truncated: MCValue = switch (operand_mcv) {
1052 .register => |r| MCValue{ .register = registerAlias(r, dest_abi_size) },
1053 else => operand_mcv,
1054 };
10171055
1018 if (info_a.bits == info_b.bits)1056 if (dest_info.bits > operand_info.bits) {
1019 return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });1057 const dest_mcv = try self.allocRegOrMem(inst, true);
1058 try self.setRegOrMem(self.air.typeOfIndex(inst), dest_mcv, truncated);
1059 break :result dest_mcv;
1060 } else {
1061 if (self.reuseOperand(inst, operand, 0, truncated)) {
1062 break :result truncated;
1063 } else {
1064 const dest_mcv = try self.allocRegOrMem(inst, true);
1065 try self.setRegOrMem(self.air.typeOfIndex(inst), dest_mcv, truncated);
1066 break :result dest_mcv;
1067 }
1068 }
1069 };
10201070
1021 return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch});1071 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1022}1072}
10231073
1024fn truncRegister(1074fn truncRegister(
...@@ -1044,6 +1094,8 @@ fn truncRegister(...@@ -1044,6 +1094,8 @@ fn truncRegister(
1044 });1094 });
1045 },1095 },
1046 32, 64 => {1096 32, 64 => {
1097 assert(dest_reg.size() == operand_reg.size());
1098
1047 _ = try self.addInst(.{1099 _ = try self.addInst(.{
1048 .tag = .mov_register,1100 .tag = .mov_register,
1049 .data = .{ .rr = .{1101 .data = .{ .rr = .{
...@@ -1099,7 +1151,7 @@ fn trunc(...@@ -1099,7 +1151,7 @@ fn trunc(
10991151
1100 return MCValue{ .register = dest_reg };1152 return MCValue{ .register = dest_reg };
1101 } else {1153 } else {
1102 return self.fail("TODO: truncate to ints > 32 bits", .{});1154 return self.fail("TODO: truncate to ints > 64 bits", .{});
1103 }1155 }
1104}1156}
11051157
...@@ -1262,6 +1314,9 @@ fn binOpRegister(...@@ -1262,6 +1314,9 @@ fn binOpRegister(
1262 const lhs_is_register = lhs == .register;1314 const lhs_is_register = lhs == .register;
1263 const rhs_is_register = rhs == .register;1315 const rhs_is_register = rhs == .register;
12641316
1317 if (lhs_is_register) assert(lhs.register == registerAlias(lhs.register, lhs_ty.abiSize(self.target.*)));
1318 if (rhs_is_register) assert(rhs.register == registerAlias(rhs.register, rhs_ty.abiSize(self.target.*)));
1319
1265 const lhs_lock: ?RegisterLock = if (lhs_is_register)1320 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1266 self.register_manager.lockReg(lhs.register)1321 self.register_manager.lockReg(lhs.register)
1267 else1322 else
...@@ -1291,13 +1346,22 @@ fn binOpRegister(...@@ -1291,13 +1346,22 @@ fn binOpRegister(
1291 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);1346 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
1292 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);1347 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
12931348
1294 const rhs_reg = if (rhs_is_register) rhs.register else blk: {1349 const rhs_reg = if (rhs_is_register)
1350 // lhs is almost always equal to rhs, except in shifts. In
1351 // order to guarantee that registers will have equal sizes, we
1352 // use the register alias of rhs corresponding to the size of
1353 // lhs.
1354 registerAlias(rhs.register, lhs_ty.abiSize(self.target.*))
1355 else blk: {
1295 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {1356 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
1296 break :inst Air.refToIndex(md.rhs).?;1357 break :inst Air.refToIndex(md.rhs).?;
1297 } else null;1358 } else null;
12981359
1299 const raw_reg = try self.register_manager.allocReg(track_inst, gp);1360 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1300 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));1361
1362 // Here, we deliberately use lhs as lhs and rhs may differ in
1363 // the case of shifts. See comment above.
1364 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
13011365
1302 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });1366 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
13031367
...@@ -1348,6 +1412,8 @@ fn binOpRegister(...@@ -1348,6 +1412,8 @@ fn binOpRegister(
1348 .lsl_register,1412 .lsl_register,
1349 .asr_register,1413 .asr_register,
1350 .lsr_register,1414 .lsr_register,
1415 .sdiv,
1416 .udiv,
1351 => .{ .rrr = .{1417 => .{ .rrr = .{
1352 .rd = dest_reg,1418 .rd = dest_reg,
1353 .rn = lhs_reg,1419 .rn = lhs_reg,
...@@ -1404,6 +1470,8 @@ fn binOpImmediate(...@@ -1404,6 +1470,8 @@ fn binOpImmediate(
1404) !MCValue {1470) !MCValue {
1405 const lhs_is_register = lhs == .register;1471 const lhs_is_register = lhs == .register;
14061472
1473 if (lhs_is_register) assert(lhs.register == registerAlias(lhs.register, lhs_ty.abiSize(self.target.*)));
1474
1407 const lhs_lock: ?RegisterLock = if (lhs_is_register)1475 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1408 self.register_manager.lockReg(lhs.register)1476 self.register_manager.lockReg(lhs.register)
1409 else1477 else
...@@ -1586,6 +1654,151 @@ fn binOp(...@@ -1586,6 +1654,151 @@ fn binOp(
1586 else => unreachable,1654 else => unreachable,
1587 }1655 }
1588 },1656 },
1657 .div_float => {
1658 switch (lhs_ty.zigTypeTag()) {
1659 .Float => return self.fail("TODO div_float", .{}),
1660 .Vector => return self.fail("TODO div_float on vectors", .{}),
1661 else => unreachable,
1662 }
1663 },
1664 .div_trunc, .div_floor, .div_exact => {
1665 switch (lhs_ty.zigTypeTag()) {
1666 .Float => return self.fail("TODO div on floats", .{}),
1667 .Vector => return self.fail("TODO div on vectors", .{}),
1668 .Int => {
1669 assert(lhs_ty.eql(rhs_ty, mod));
1670 const int_info = lhs_ty.intInfo(self.target.*);
1671 if (int_info.bits <= 64) {
1672 switch (int_info.signedness) {
1673 .signed => {
1674 switch (tag) {
1675 .div_trunc, .div_exact => {
1676 // TODO optimize integer division by constants
1677 return try self.binOpRegister(.sdiv, lhs, rhs, lhs_ty, rhs_ty, metadata);
1678 },
1679 .div_floor => return self.fail("TODO div_floor on signed integers", .{}),
1680 else => unreachable,
1681 }
1682 },
1683 .unsigned => {
1684 // TODO optimize integer division by constants
1685 return try self.binOpRegister(.udiv, lhs, rhs, lhs_ty, rhs_ty, metadata);
1686 },
1687 }
1688 } else {
1689 return self.fail("TODO integer division for ints with bits > 64", .{});
1690 }
1691 },
1692 else => unreachable,
1693 }
1694 },
1695 .rem, .mod => {
1696 switch (lhs_ty.zigTypeTag()) {
1697 .Float => return self.fail("TODO rem/mod on floats", .{}),
1698 .Vector => return self.fail("TODO rem/mod on vectors", .{}),
1699 .Int => {
1700 assert(lhs_ty.eql(rhs_ty, mod));
1701 const int_info = lhs_ty.intInfo(self.target.*);
1702 if (int_info.bits <= 64) {
1703 if (int_info.signedness == .signed and tag == .mod) {
1704 return self.fail("TODO mod on signed integers", .{});
1705 } else {
1706 const lhs_is_register = lhs == .register;
1707 const rhs_is_register = rhs == .register;
1708
1709 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1710 self.register_manager.lockReg(lhs.register)
1711 else
1712 null;
1713 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
1714
1715 const rhs_lock: ?RegisterLock = if (rhs_is_register)
1716 self.register_manager.lockReg(rhs.register)
1717 else
1718 null;
1719 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
1720
1721 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1722
1723 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
1724 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
1725 break :inst Air.refToIndex(md.lhs).?;
1726 } else null;
1727
1728 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1729 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1730
1731 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1732
1733 break :blk reg;
1734 };
1735 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
1736 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
1737
1738 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
1739 const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
1740 break :inst Air.refToIndex(md.rhs).?;
1741 } else null;
1742
1743 const raw_reg = try self.register_manager.allocReg(track_inst, gp);
1744 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1745
1746 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
1747
1748 break :blk reg;
1749 };
1750 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
1751 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
1752
1753 const dest_regs: [2]Register = blk: {
1754 const raw_regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
1755 const abi_size = lhs_ty.abiSize(self.target.*);
1756 break :blk .{
1757 registerAlias(raw_regs[0], abi_size),
1758 registerAlias(raw_regs[1], abi_size),
1759 };
1760 };
1761 const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs);
1762 defer for (dest_regs_locks) |reg| {
1763 self.register_manager.unlockReg(reg);
1764 };
1765 const quotient_reg = dest_regs[0];
1766 const remainder_reg = dest_regs[1];
1767
1768 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
1769 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
1770
1771 _ = try self.addInst(.{
1772 .tag = switch (int_info.signedness) {
1773 .signed => .sdiv,
1774 .unsigned => .udiv,
1775 },
1776 .data = .{ .rrr = .{
1777 .rd = quotient_reg,
1778 .rn = lhs_reg,
1779 .rm = rhs_reg,
1780 } },
1781 });
1782
1783 _ = try self.addInst(.{
1784 .tag = .msub,
1785 .data = .{ .rrrr = .{
1786 .rd = remainder_reg,
1787 .rn = quotient_reg,
1788 .rm = rhs_reg,
1789 .ra = lhs_reg,
1790 } },
1791 });
1792
1793 return MCValue{ .register = remainder_reg };
1794 }
1795 } else {
1796 return self.fail("TODO rem/mod for integers with bits > 64", .{});
1797 }
1798 },
1799 else => unreachable,
1800 }
1801 },
1589 .addwrap,1802 .addwrap,
1590 .subwrap,1803 .subwrap,
1591 .mulwrap,1804 .mulwrap,
...@@ -1869,7 +2082,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1869,7 +2082,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
1869 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);2082 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
18702083
1871 // cmp dest, truncated2084 // cmp dest, truncated
1872 _ = try self.binOp(.cmp_eq, dest, .{ .register = truncated_reg }, Type.usize, Type.usize, null);2085 _ = try self.binOp(.cmp_eq, dest, .{ .register = truncated_reg }, lhs_ty, lhs_ty, null);
18732086
1874 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });2087 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1875 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });2088 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .condition_flags = .ne });
...@@ -2257,24 +2470,6 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2257,24 +2470,6 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2257 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });2470 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2258}2471}
22592472
2260fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
2261 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2262 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
2263 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2264}
2265
2266fn airRem(self: *Self, inst: Air.Inst.Index) !void {
2267 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2268 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});
2269 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2270}
2271
2272fn airMod(self: *Self, inst: Air.Inst.Index) !void {
2273 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2274 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mod for {}", .{self.target.cpu.arch});
2275 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2276}
2277
2278fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {2473fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
2279 const bin_op = self.air.instructions.items(.data)[inst].bin_op;2474 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2280 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});2475 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
...@@ -2313,6 +2508,9 @@ fn errUnionErr(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) !MCV...@@ -2313,6 +2508,9 @@ fn errUnionErr(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) !MCV
2313 const err_offset = @intCast(u32, errUnionErrorOffset(payload_ty, self.target.*));2508 const err_offset = @intCast(u32, errUnionErrorOffset(payload_ty, self.target.*));
2314 switch (error_union_mcv) {2509 switch (error_union_mcv) {
2315 .register => return self.fail("TODO errUnionErr for registers", .{}),2510 .register => return self.fail("TODO errUnionErr for registers", .{}),
2511 .stack_argument_offset => |off| {
2512 return MCValue{ .stack_argument_offset = off + err_offset };
2513 },
2316 .stack_offset => |off| {2514 .stack_offset => |off| {
2317 return MCValue{ .stack_offset = off - err_offset };2515 return MCValue{ .stack_offset = off - err_offset };
2318 },2516 },
...@@ -2347,6 +2545,9 @@ fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type)...@@ -2347,6 +2545,9 @@ fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type)
2347 const payload_offset = @intCast(u32, errUnionPayloadOffset(payload_ty, self.target.*));2545 const payload_offset = @intCast(u32, errUnionPayloadOffset(payload_ty, self.target.*));
2348 switch (error_union_mcv) {2546 switch (error_union_mcv) {
2349 .register => return self.fail("TODO errUnionPayload for registers", .{}),2547 .register => return self.fail("TODO errUnionPayload for registers", .{}),
2548 .stack_argument_offset => |off| {
2549 return MCValue{ .stack_argument_offset = off + payload_offset };
2550 },
2350 .stack_offset => |off| {2551 .stack_offset => |off| {
2351 return MCValue{ .stack_offset = off - payload_offset };2552 return MCValue{ .stack_offset = off - payload_offset };
2352 },2553 },
...@@ -2436,21 +2637,28 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2436,21 +2637,28 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2436 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2637 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2437}2638}
24382639
2640fn slicePtr(mcv: MCValue) MCValue {
2641 switch (mcv) {
2642 .dead, .unreach, .none => unreachable,
2643 .register => unreachable, // a slice doesn't fit in one register
2644 .stack_argument_offset => |off| {
2645 return MCValue{ .stack_argument_offset = off };
2646 },
2647 .stack_offset => |off| {
2648 return MCValue{ .stack_offset = off };
2649 },
2650 .memory => |addr| {
2651 return MCValue{ .memory = addr };
2652 },
2653 else => unreachable, // invalid MCValue for a slice
2654 }
2655}
2656
2439fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {2657fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2440 const ty_op = self.air.instructions.items(.data)[inst].ty_op;2658 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2441 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {2659 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2442 const mcv = try self.resolveInst(ty_op.operand);2660 const mcv = try self.resolveInst(ty_op.operand);
2443 switch (mcv) {2661 break :result slicePtr(mcv);
2444 .dead, .unreach, .none => unreachable,
2445 .register => unreachable, // a slice doesn't fit in one register
2446 .stack_offset => |off| {
2447 break :result MCValue{ .stack_offset = off };
2448 },
2449 .memory => |addr| {
2450 break :result MCValue{ .memory = addr };
2451 },
2452 else => return self.fail("TODO implement slice_len for {}", .{mcv}),
2453 }
2454 };2662 };
2455 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2663 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2456}2664}
...@@ -2464,6 +2672,9 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -2464,6 +2672,9 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2464 switch (mcv) {2672 switch (mcv) {
2465 .dead, .unreach, .none => unreachable,2673 .dead, .unreach, .none => unreachable,
2466 .register => unreachable, // a slice doesn't fit in one register2674 .register => unreachable, // a slice doesn't fit in one register
2675 .stack_argument_offset => |off| {
2676 break :result MCValue{ .stack_argument_offset = off + ptr_bytes };
2677 },
2467 .stack_offset => |off| {2678 .stack_offset => |off| {
2468 break :result MCValue{ .stack_offset = off - ptr_bytes };2679 break :result MCValue{ .stack_offset = off - ptr_bytes };
2469 },2680 },
...@@ -2514,6 +2725,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2514,6 +2725,9 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
25142725
2515 if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });2726 if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2516 const result: MCValue = result: {2727 const result: MCValue = result: {
2728 const slice_ty = self.air.typeOf(bin_op.lhs);
2729 const elem_ty = slice_ty.childType();
2730 const elem_size = elem_ty.abiSize(self.target.*);
2517 const slice_mcv = try self.resolveInst(bin_op.lhs);2731 const slice_mcv = try self.resolveInst(bin_op.lhs);
25182732
2519 // TODO optimize for the case where the index is a constant,2733 // TODO optimize for the case where the index is a constant,
...@@ -2521,10 +2735,6 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2521,10 +2735,6 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2521 const index_mcv = try self.resolveInst(bin_op.rhs);2735 const index_mcv = try self.resolveInst(bin_op.rhs);
2522 const index_is_register = index_mcv == .register;2736 const index_is_register = index_mcv == .register;
25232737
2524 const slice_ty = self.air.typeOf(bin_op.lhs);
2525 const elem_ty = slice_ty.childType();
2526 const elem_size = elem_ty.abiSize(self.target.*);
2527
2528 var buf: Type.SlicePtrFieldTypeBuffer = undefined;2738 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
2529 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);2739 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
25302740
...@@ -2534,15 +2744,17 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2534,15 +2744,17 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2534 null;2744 null;
2535 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);2745 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
25362746
2537 const base_mcv: MCValue = switch (slice_mcv) {2747 const base_mcv = slicePtr(slice_mcv);
2538 .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
2539 else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
2540 };
2541 const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register);
2542 defer self.register_manager.unlockReg(base_lock);
25432748
2544 switch (elem_size) {2749 switch (elem_size) {
2545 else => {2750 else => {
2751 const base_reg = switch (base_mcv) {
2752 .register => |r| r,
2753 else => try self.copyToTmpRegister(slice_ptr_field_type, base_mcv),
2754 };
2755 const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg);
2756 defer self.register_manager.unlockReg(base_reg_lock);
2757
2546 const dest = try self.allocRegOrMem(inst, true);2758 const dest = try self.allocRegOrMem(inst, true);
2547 const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ptr_field_type, Type.usize, null);2759 const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ptr_field_type, Type.usize, null);
2548 try self.load(dest, addr, slice_ptr_field_type);2760 try self.load(dest, addr, slice_ptr_field_type);
...@@ -2557,7 +2769,16 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2557,7 +2769,16 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2557fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {2769fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2558 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;2770 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2559 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;2771 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2560 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});2772 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2773 const slice_mcv = try self.resolveInst(extra.lhs);
2774 const index_mcv = try self.resolveInst(extra.rhs);
2775 const base_mcv = slicePtr(slice_mcv);
2776
2777 const slice_ty = self.air.typeOf(extra.lhs);
2778
2779 const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ty, Type.usize, null);
2780 break :result addr;
2781 };
2561 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });2782 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2562}2783}
25632784
...@@ -2577,7 +2798,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2577,7 +2798,15 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2577fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {2798fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2578 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;2799 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2579 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;2800 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2580 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});2801 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2802 const ptr_mcv = try self.resolveInst(extra.lhs);
2803 const index_mcv = try self.resolveInst(extra.rhs);
2804
2805 const ptr_ty = self.air.typeOf(extra.lhs);
2806
2807 const addr = try self.binOp(.ptr_add, ptr_mcv, index_mcv, ptr_ty, Type.usize, null);
2808 break :result addr;
2809 };
2581 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });2810 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2582}2811}
25832812
...@@ -2726,6 +2955,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -2726,6 +2955,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2726 },2955 },
2727 .memory,2956 .memory,
2728 .stack_offset,2957 .stack_offset,
2958 .stack_argument_offset,
2729 .got_load,2959 .got_load,
2730 .direct_load,2960 .direct_load,
2731 => {2961 => {
...@@ -2907,6 +3137,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2907,6 +3137,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2907 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);3137 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
29083138
2909 switch (value) {3139 switch (value) {
3140 .dead => unreachable,
3141 .undef => unreachable,
2910 .register => |value_reg| {3142 .register => |value_reg| {
2911 try self.genStrRegister(value_reg, addr_reg, value_ty);3143 try self.genStrRegister(value_reg, addr_reg, value_ty);
2912 },3144 },
...@@ -2920,13 +3152,48 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2920,13 +3152,48 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2920 try self.genSetReg(value_ty, tmp_reg, value);3152 try self.genSetReg(value_ty, tmp_reg, value);
2921 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);3153 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2922 } else {3154 } else {
2923 return self.fail("TODO implement memcpy", .{});3155 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
3156 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
3157 defer for (regs_locks) |reg| {
3158 self.register_manager.unlockReg(reg);
3159 };
3160
3161 const src_reg = addr_reg;
3162 const dst_reg = regs[0];
3163 const len_reg = regs[1];
3164 const count_reg = regs[2];
3165 const tmp_reg = regs[3];
3166
3167 switch (value) {
3168 .stack_offset => |off| {
3169 // sub src_reg, fp, #off
3170 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
3171 },
3172 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = @intCast(u32, addr) }),
3173 .stack_argument_offset => |off| {
3174 _ = try self.addInst(.{
3175 .tag = .ldr_ptr_stack_argument,
3176 .data = .{ .load_store_stack = .{
3177 .rt = src_reg,
3178 .offset = off,
3179 } },
3180 });
3181 },
3182 else => return self.fail("TODO store {} to register", .{value}),
3183 }
3184
3185 // mov len, #abi_size
3186 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
3187
3188 // memcpy(src, dst, len)
3189 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
2924 }3190 }
2925 },3191 },
2926 }3192 }
2927 },3193 },
2928 .memory,3194 .memory,
2929 .stack_offset,3195 .stack_offset,
3196 .stack_argument_offset,
2930 .got_load,3197 .got_load,
2931 .direct_load,3198 .direct_load,
2932 => {3199 => {
...@@ -3005,10 +3272,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3005,10 +3272,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3005 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {3272 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3006 const mcv = try self.resolveInst(operand);3273 const mcv = try self.resolveInst(operand);
3007 const struct_ty = self.air.typeOf(operand);3274 const struct_ty = self.air.typeOf(operand);
3275 const struct_field_ty = struct_ty.structFieldType(index);
3008 const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));3276 const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));
30093277
3010 switch (mcv) {3278 switch (mcv) {
3011 .dead, .unreach => unreachable,3279 .dead, .unreach => unreachable,
3280 .stack_argument_offset => |off| {
3281 break :result MCValue{ .stack_argument_offset = off + struct_field_offset };
3282 },
3012 .stack_offset => |off| {3283 .stack_offset => |off| {
3013 break :result MCValue{ .stack_offset = off - struct_field_offset };3284 break :result MCValue{ .stack_offset = off - struct_field_offset };
3014 },3285 },
...@@ -3016,29 +3287,28 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3016,29 +3287,28 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3016 break :result MCValue{ .memory = addr + struct_field_offset };3287 break :result MCValue{ .memory = addr + struct_field_offset };
3017 },3288 },
3018 .register_with_overflow => |rwo| {3289 .register_with_overflow => |rwo| {
3019 switch (index) {3290 const reg_lock = self.register_manager.lockRegAssumeUnused(rwo.reg);
3020 0 => {3291 defer self.register_manager.unlockReg(reg_lock);
3021 // get wrapped value: return register
3022 break :result MCValue{ .register = rwo.reg };
3023 },
3024 1 => {
3025 // TODO return special MCValue condition flags
3026 // get overflow bit: set register to C flag
3027 // resp. V flag
3028 const raw_dest_reg = try self.register_manager.allocReg(null, gp);
3029 const dest_reg = raw_dest_reg.to32();
30303292
3031 _ = try self.addInst(.{3293 const field: MCValue = switch (index) {
3032 .tag = .cset,3294 // get wrapped value: return register
3033 .data = .{ .r_cond = .{3295 0 => MCValue{ .register = rwo.reg },
3034 .rd = dest_reg,3296
3035 .cond = rwo.flag,3297 // get overflow bit: return C or V flag
3036 } },3298 1 => MCValue{ .condition_flags = rwo.flag },
3037 });
30383299
3039 break :result MCValue{ .register = dest_reg };
3040 },
3041 else => unreachable,3300 else => unreachable,
3301 };
3302
3303 if (self.reuseOperand(inst, operand, 0, field)) {
3304 break :result field;
3305 } else {
3306 // Copy to new register
3307 const raw_dest_reg = try self.register_manager.allocReg(null, gp);
3308 const dest_reg = registerAlias(raw_dest_reg, struct_field_ty.abiSize(self.target.*));
3309 try self.genSetReg(struct_field_ty, dest_reg, field);
3310
3311 break :result MCValue{ .register = dest_reg };
3042 }3312 }
3043 },3313 },
3044 else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}),3314 else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}),
...@@ -3143,6 +3413,31 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions....@@ -3143,6 +3413,31 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3143 // saving compare flags may require a new caller-saved register3413 // saving compare flags may require a new caller-saved register
3144 try self.spillCompareFlagsIfOccupied();3414 try self.spillCompareFlagsIfOccupied();
31453415
3416 if (info.return_value == .stack_offset) {
3417 log.debug("airCall: return by reference", .{});
3418 const ret_ty = fn_ty.fnReturnType();
3419 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3420 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
3421 const stack_offset = try self.allocMem(inst, ret_abi_size, ret_abi_align);
3422
3423 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
3424 const ptr_bytes = @divExact(ptr_bits, 8);
3425 const ret_ptr_reg = registerAlias(.x0, ptr_bytes);
3426
3427 var ptr_ty_payload: Type.Payload.ElemType = .{
3428 .base = .{ .tag = .single_mut_pointer },
3429 .data = ret_ty,
3430 };
3431 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
3432 try self.register_manager.getReg(ret_ptr_reg, null);
3433 try self.genSetReg(ptr_ty, ret_ptr_reg, .{ .ptr_stack_offset = stack_offset });
3434
3435 info.return_value = .{ .stack_offset = stack_offset };
3436 }
3437
3438 // Make space for the arguments passed via the stack
3439 self.max_end_stack += info.stack_byte_count;
3440
3146 for (info.args) |mc_arg, arg_i| {3441 for (info.args) |mc_arg, arg_i| {
3147 const arg = args[arg_i];3442 const arg = args[arg_i];
3148 const arg_ty = self.air.typeOf(arg);3443 const arg_ty = self.air.typeOf(arg);
...@@ -3154,12 +3449,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions....@@ -3154,12 +3449,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3154 try self.register_manager.getReg(reg, null);3449 try self.register_manager.getReg(reg, null);
3155 try self.genSetReg(arg_ty, reg, arg_mcv);3450 try self.genSetReg(arg_ty, reg, arg_mcv);
3156 },3451 },
3157 .stack_offset => {3452 .stack_offset => unreachable,
3158 return self.fail("TODO implement calling with parameters in memory", .{});3453 .stack_argument_offset => |offset| try self.genSetStackArgument(
3159 },3454 arg_ty,
3160 .ptr_stack_offset => {3455 offset,
3161 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});3456 arg_mcv,
3162 },3457 ),
3163 else => unreachable,3458 else => unreachable,
3164 }3459 }
3165 }3460 }
...@@ -3303,8 +3598,15 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {...@@ -3303,8 +3598,15 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
3303 },3598 },
3304 .stack_offset => {3599 .stack_offset => {
3305 // Return result by reference3600 // Return result by reference
3306 // TODO3601 //
3307 return self.fail("TODO implement airRet for {}", .{self.ret_mcv});3602 // self.ret_mcv is an address to where this function
3603 // should store its result into
3604 var ptr_ty_payload: Type.Payload.ElemType = .{
3605 .base = .{ .tag = .single_mut_pointer },
3606 .data = ret_ty,
3607 };
3608 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
3609 try self.store(self.ret_mcv, operand, ptr_ty, ret_ty);
3308 },3610 },
3309 else => unreachable,3611 else => unreachable,
3310 }3612 }
...@@ -3330,10 +3632,34 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -3330,10 +3632,34 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
3330 },3632 },
3331 .stack_offset => {3633 .stack_offset => {
3332 // Return result by reference3634 // Return result by reference
3333 // TODO3635 //
3334 return self.fail("TODO implement airRetLoad for {}", .{self.ret_mcv});3636 // self.ret_mcv is an address to where this function
3637 // should store its result into
3638 //
3639 // If the operand is a ret_ptr instruction, we are done
3640 // here. Else we need to load the result from the location
3641 // pointed to by the operand and store it to the result
3642 // location.
3643 const op_inst = Air.refToIndex(un_op).?;
3644 if (self.air.instructions.items(.tag)[op_inst] != .ret_ptr) {
3645 const abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3646 const abi_align = ret_ty.abiAlignment(self.target.*);
3647
3648 // This is essentially allocMem without the
3649 // instruction tracking
3650 if (abi_align > self.stack_align)
3651 self.stack_align = abi_align;
3652 // TODO find a free slot instead of always appending
3653 const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align) + abi_size;
3654 self.next_stack_offset = offset;
3655 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
3656
3657 const tmp_mcv = MCValue{ .stack_offset = offset };
3658 try self.load(tmp_mcv, ptr, ptr_ty);
3659 try self.store(self.ret_mcv, tmp_mcv, ptr_ty, ret_ty);
3660 }
3335 },3661 },
3336 else => unreachable,3662 else => unreachable, // invalid return result
3337 }3663 }
33383664
3339 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());3665 try self.exitlude_jump_relocs.append(self.gpa, try self.addNop());
...@@ -3635,40 +3961,14 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {...@@ -3635,40 +3961,14 @@ fn isNonNull(self: *Self, operand: MCValue) !MCValue {
36353961
3636fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {3962fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
3637 const error_type = ty.errorUnionSet();3963 const error_type = ty.errorUnionSet();
3638 const payload_type = ty.errorUnionPayload();3964 const error_int_type = Type.initTag(.u16);
36393965
3640 if (error_type.errorSetIsEmpty()) {3966 if (error_type.errorSetIsEmpty()) {
3641 return MCValue{ .immediate = 0 }; // always false3967 return MCValue{ .immediate = 0 }; // always false
3642 }3968 }
36433969
3644 const err_off = errUnionErrorOffset(payload_type, self.target.*);3970 const error_mcv = try self.errUnionErr(operand, ty);
3645 switch (operand) {3971 _ = try self.binOp(.cmp_eq, error_mcv, .{ .immediate = 0 }, error_int_type, error_int_type, null);
3646 .stack_offset => |off| {
3647 const offset = off - @intCast(u32, err_off);
3648 const tmp_reg = try self.copyToTmpRegister(Type.anyerror, .{ .stack_offset = offset });
3649 _ = try self.addInst(.{
3650 .tag = .cmp_immediate,
3651 .data = .{ .r_imm12_sh = .{
3652 .rn = tmp_reg,
3653 .imm12 = 0,
3654 } },
3655 });
3656 },
3657 .register => |reg| {
3658 if (err_off > 0 or payload_type.hasRuntimeBitsIgnoreComptime()) {
3659 return self.fail("TODO implement isErr for register operand with payload bits", .{});
3660 }
3661 _ = try self.addInst(.{
3662 .tag = .cmp_immediate,
3663 .data = .{ .r_imm12_sh = .{
3664 .rn = reg,
3665 .imm12 = 0,
3666 } },
3667 });
3668 },
3669 else => return self.fail("TODO implement isErr for {}", .{operand}),
3670 }
3671
3672 return MCValue{ .condition_flags = .hi };3972 return MCValue{ .condition_flags = .hi };
3673}3973}
36743974
...@@ -3886,7 +4186,7 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {...@@ -3886,7 +4186,7 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
3886 block_data.mcv = switch (operand_mcv) {4186 block_data.mcv = switch (operand_mcv) {
3887 .none, .dead, .unreach => unreachable,4187 .none, .dead, .unreach => unreachable,
3888 .register, .stack_offset, .memory => operand_mcv,4188 .register, .stack_offset, .memory => operand_mcv,
3889 .immediate, .condition_flags => blk: {4189 .immediate, .stack_argument_offset, .condition_flags => blk: {
3890 const new_mcv = try self.allocRegOrMem(block, true);4190 const new_mcv = try self.allocRegOrMem(block, true);
3891 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);4191 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
3892 break :blk new_mcv;4192 break :blk new_mcv;
...@@ -4128,6 +4428,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -4128,6 +4428,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
4128 .got_load,4428 .got_load,
4129 .direct_load,4429 .direct_load,
4130 .memory,4430 .memory,
4431 .stack_argument_offset,
4131 .stack_offset,4432 .stack_offset,
4132 => {4433 => {
4133 switch (mcv) {4434 switch (mcv) {
...@@ -4166,6 +4467,15 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -4166,6 +4467,15 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
4166 // sub src_reg, fp, #off4467 // sub src_reg, fp, #off
4167 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });4468 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
4168 },4469 },
4470 .stack_argument_offset => |off| {
4471 _ = try self.addInst(.{
4472 .tag = .ldr_ptr_stack_argument,
4473 .data = .{ .load_store_stack = .{
4474 .rt = src_reg,
4475 .offset = off,
4476 } },
4477 });
4478 },
4169 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }),4479 .memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }),
4170 .got_load,4480 .got_load,
4171 .direct_load,4481 .direct_load,
...@@ -4269,6 +4579,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -4269,6 +4579,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
4269 }4579 }
4270 },4580 },
4271 .register => |src_reg| {4581 .register => |src_reg| {
4582 assert(src_reg.size() == reg.size());
4583
4272 // If the registers are the same, nothing to do.4584 // If the registers are the same, nothing to do.
4273 if (src_reg.id() == reg.id())4585 if (src_reg.id() == reg.id())
4274 return;4586 return;
...@@ -4330,6 +4642,196 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -4330,6 +4642,196 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
4330 else => unreachable,4642 else => unreachable,
4331 }4643 }
4332 },4644 },
4645 .stack_argument_offset => |off| {
4646 const abi_size = ty.abiSize(self.target.*);
4647
4648 switch (abi_size) {
4649 1, 2, 4, 8 => {
4650 const tag: Mir.Inst.Tag = switch (abi_size) {
4651 1 => if (ty.isSignedInt()) Mir.Inst.Tag.ldrsb_stack_argument else .ldrb_stack_argument,
4652 2 => if (ty.isSignedInt()) Mir.Inst.Tag.ldrsh_stack_argument else .ldrh_stack_argument,
4653 4, 8 => .ldr_stack_argument,
4654 else => unreachable, // unexpected abi size
4655 };
4656
4657 _ = try self.addInst(.{
4658 .tag = tag,
4659 .data = .{ .load_store_stack = .{
4660 .rt = reg,
4661 .offset = @intCast(u32, off),
4662 } },
4663 });
4664 },
4665 3, 5, 6, 7 => return self.fail("TODO implement genSetReg types size {}", .{abi_size}),
4666 else => unreachable,
4667 }
4668 },
4669 }
4670}
4671
4672fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
4673 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4674 switch (mcv) {
4675 .dead => unreachable,
4676 .none, .unreach => return,
4677 .undef => {
4678 if (!self.wantSafety())
4679 return; // The already existing value will do just fine.
4680 // TODO Upgrade this to a memset call when we have that available.
4681 switch (ty.abiSize(self.target.*)) {
4682 1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
4683 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
4684 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
4685 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
4686 else => return self.fail("TODO implement memset", .{}),
4687 }
4688 },
4689 .register => |reg| {
4690 switch (abi_size) {
4691 1, 2, 4, 8 => {
4692 const tag: Mir.Inst.Tag = switch (abi_size) {
4693 1 => .strb_immediate,
4694 2 => .strh_immediate,
4695 4, 8 => .str_immediate,
4696 else => unreachable, // unexpected abi size
4697 };
4698 const rt = registerAlias(reg, abi_size);
4699 const offset = switch (abi_size) {
4700 1 => blk: {
4701 if (math.cast(u12, stack_offset)) |imm| {
4702 break :blk Instruction.LoadStoreOffset.imm(imm);
4703 } else {
4704 return self.fail("TODO genSetStackArgument byte with larger offset", .{});
4705 }
4706 },
4707 2 => blk: {
4708 assert(std.mem.isAlignedGeneric(u32, stack_offset, 2)); // misaligned stack entry
4709 if (math.cast(u12, @divExact(stack_offset, 2))) |imm| {
4710 break :blk Instruction.LoadStoreOffset.imm(imm);
4711 } else {
4712 return self.fail("TODO getSetStackArgument halfword with larger offset", .{});
4713 }
4714 },
4715 4, 8 => blk: {
4716 const alignment = abi_size;
4717 assert(std.mem.isAlignedGeneric(u32, stack_offset, alignment)); // misaligned stack entry
4718 if (math.cast(u12, @divExact(stack_offset, alignment))) |imm| {
4719 break :blk Instruction.LoadStoreOffset.imm(imm);
4720 } else {
4721 return self.fail("TODO genSetStackArgument with larger offset", .{});
4722 }
4723 },
4724 else => unreachable,
4725 };
4726
4727 _ = try self.addInst(.{
4728 .tag = tag,
4729 .data = .{ .load_store_register_immediate = .{
4730 .rt = rt,
4731 .rn = .sp,
4732 .offset = offset.immediate,
4733 } },
4734 });
4735 },
4736 else => return self.fail("TODO genSetStackArgument other types abi_size={}", .{abi_size}),
4737 }
4738 },
4739 .register_with_overflow => {
4740 return self.fail("TODO implement genSetStackArgument {}", .{mcv});
4741 },
4742 .got_load,
4743 .direct_load,
4744 .memory,
4745 .stack_argument_offset,
4746 .stack_offset,
4747 => {
4748 if (abi_size <= 4) {
4749 const reg = try self.copyToTmpRegister(ty, mcv);
4750 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
4751 } else {
4752 var ptr_ty_payload: Type.Payload.ElemType = .{
4753 .base = .{ .tag = .single_mut_pointer },
4754 .data = ty,
4755 };
4756 const ptr_ty = Type.initPayload(&ptr_ty_payload.base);
4757
4758 // TODO call extern memcpy
4759 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
4760 const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
4761 defer for (regs_locks) |reg| {
4762 self.register_manager.unlockReg(reg);
4763 };
4764
4765 const src_reg = regs[0];
4766 const dst_reg = regs[1];
4767 const len_reg = regs[2];
4768 const count_reg = regs[3];
4769 const tmp_reg = regs[4];
4770
4771 switch (mcv) {
4772 .stack_offset => |off| {
4773 // sub src_reg, fp, #off
4774 try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off });
4775 },
4776 .stack_argument_offset => |off| {
4777 _ = try self.addInst(.{
4778 .tag = .ldr_ptr_stack_argument,
4779 .data = .{ .load_store_stack = .{
4780 .rt = src_reg,
4781 .offset = off,
4782 } },
4783 });
4784 },
4785 .memory => |addr| try self.genSetReg(ptr_ty, src_reg, .{ .immediate = @intCast(u32, addr) }),
4786 .got_load,
4787 .direct_load,
4788 => |sym_index| {
4789 const tag: Mir.Inst.Tag = switch (mcv) {
4790 .got_load => .load_memory_ptr_got,
4791 .direct_load => .load_memory_ptr_direct,
4792 else => unreachable,
4793 };
4794 const mod = self.bin_file.options.module.?;
4795 _ = try self.addInst(.{
4796 .tag = tag,
4797 .data = .{
4798 .payload = try self.addExtra(Mir.LoadMemoryPie{
4799 .register = @enumToInt(src_reg),
4800 .atom_index = mod.declPtr(self.mod_fn.owner_decl).link.macho.sym_index,
4801 .sym_index = sym_index,
4802 }),
4803 },
4804 });
4805 },
4806 else => unreachable,
4807 }
4808
4809 // add dst_reg, sp, #stack_offset
4810 _ = try self.addInst(.{
4811 .tag = .add_immediate,
4812 .data = .{ .rr_imm12_sh = .{
4813 .rd = dst_reg,
4814 .rn = .sp,
4815 .imm12 = math.cast(u12, stack_offset) orelse {
4816 return self.fail("TODO load: set reg to stack offset with all possible offsets", .{});
4817 },
4818 } },
4819 });
4820
4821 // mov len, #abi_size
4822 try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
4823
4824 // memcpy(src, dst, len)
4825 try self.genInlineMemcpy(src_reg, dst_reg, len_reg, count_reg, tmp_reg);
4826 }
4827 },
4828 .condition_flags,
4829 .immediate,
4830 .ptr_stack_offset,
4831 => {
4832 const reg = try self.copyToTmpRegister(ty, mcv);
4833 return self.genSetStackArgument(ty, stack_offset, MCValue{ .register = reg });
4834 },
4333 }4835 }
4334}4836}
43354837
...@@ -4799,11 +5301,27 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {...@@ -4799,11 +5301,27 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
4799 result.stack_align = 1;5301 result.stack_align = 1;
4800 return result;5302 return result;
4801 },5303 },
4802 .Unspecified, .C => {5304 .C => {
4803 // ARM64 Procedure Call Standard5305 // ARM64 Procedure Call Standard
4804 var ncrn: usize = 0; // Next Core Register Number5306 var ncrn: usize = 0; // Next Core Register Number
4805 var nsaa: u32 = 0; // Next stacked argument address5307 var nsaa: u32 = 0; // Next stacked argument address
48065308
5309 if (ret_ty.zigTypeTag() == .NoReturn) {
5310 result.return_value = .{ .unreach = {} };
5311 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
5312 result.return_value = .{ .none = {} };
5313 } else {
5314 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
5315 if (ret_ty_size == 0) {
5316 assert(ret_ty.isError());
5317 result.return_value = .{ .immediate = 0 };
5318 } else if (ret_ty_size <= 8) {
5319 result.return_value = .{ .register = registerAlias(c_abi_int_return_regs[0], ret_ty_size) };
5320 } else {
5321 return self.fail("TODO support more return types for ARM backend", .{});
5322 }
5323 }
5324
4807 for (param_types) |ty, i| {5325 for (param_types) |ty, i| {
4808 const param_size = @intCast(u32, ty.abiSize(self.target.*));5326 const param_size = @intCast(u32, ty.abiSize(self.target.*));
4809 if (param_size == 0) {5327 if (param_size == 0) {
...@@ -4837,7 +5355,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {...@@ -4837,7 +5355,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
4837 }5355 }
4838 }5356 }
48395357
4840 result.args[i] = .{ .stack_offset = nsaa };5358 result.args[i] = .{ .stack_argument_offset = nsaa };
4841 nsaa += param_size;5359 nsaa += param_size;
4842 }5360 }
4843 }5361 }
...@@ -4845,28 +5363,49 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {...@@ -4845,28 +5363,49 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
4845 result.stack_byte_count = nsaa;5363 result.stack_byte_count = nsaa;
4846 result.stack_align = 16;5364 result.stack_align = 16;
4847 },5365 },
4848 else => return self.fail("TODO implement function parameters for {} on aarch64", .{cc}),5366 .Unspecified => {
4849 }5367 if (ret_ty.zigTypeTag() == .NoReturn) {
48505368 result.return_value = .{ .unreach = {} };
4851 if (ret_ty.zigTypeTag() == .NoReturn) {5369 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
4852 result.return_value = .{ .unreach = {} };5370 result.return_value = .{ .none = {} };
4853 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
4854 result.return_value = .{ .none = {} };
4855 } else switch (cc) {
4856 .Naked => unreachable,
4857 .Unspecified, .C => {
4858 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
4859 if (ret_ty_size == 0) {
4860 assert(ret_ty.isError());
4861 result.return_value = .{ .immediate = 0 };
4862 } else if (ret_ty_size <= 8) {
4863 result.return_value = .{ .register = registerAlias(c_abi_int_return_regs[0], ret_ty_size) };
4864 } else {5371 } else {
4865 return self.fail("TODO support more return types for ARM backend", .{});5372 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
5373 if (ret_ty_size == 0) {
5374 assert(ret_ty.isError());
5375 result.return_value = .{ .immediate = 0 };
5376 } else if (ret_ty_size <= 8) {
5377 result.return_value = .{ .register = registerAlias(.x0, ret_ty_size) };
5378 } else {
5379 // The result is returned by reference, not by
5380 // value. This means that x0 (or w0 when pointer
5381 // size is 32 bits) will contain the address of
5382 // where this function should write the result
5383 // into.
5384 result.return_value = .{ .stack_offset = 0 };
5385 }
4866 }5386 }
5387
5388 var stack_offset: u32 = 0;
5389
5390 for (param_types) |ty, i| {
5391 if (ty.abiSize(self.target.*) > 0) {
5392 const param_size = @intCast(u32, ty.abiSize(self.target.*));
5393 const param_alignment = ty.abiAlignment(self.target.*);
5394
5395 stack_offset = std.mem.alignForwardGeneric(u32, stack_offset, param_alignment);
5396 result.args[i] = .{ .stack_argument_offset = stack_offset };
5397 stack_offset += param_size;
5398 } else {
5399 result.args[i] = .{ .none = {} };
5400 }
5401 }
5402
5403 result.stack_byte_count = stack_offset;
5404 result.stack_align = 16;
4867 },5405 },
4868 else => return self.fail("TODO implement function return values for {}", .{cc}),5406 else => return self.fail("TODO implement function parameters for {} on aarch64", .{cc}),
4869 }5407 }
5408
4870 return result;5409 return result;
4871}5410}
48725411
src/arch/aarch64/Emit.zig+109-10
...@@ -27,14 +27,21 @@ code: *std.ArrayList(u8),...@@ -27,14 +27,21 @@ code: *std.ArrayList(u8),
2727
28prev_di_line: u32,28prev_di_line: u32,
29prev_di_column: u32,29prev_di_column: u32,
30
30/// Relative to the beginning of `code`.31/// Relative to the beginning of `code`.
31prev_di_pc: usize,32prev_di_pc: usize,
3233
34/// The amount of stack space consumed by the saved callee-saved
35/// registers in bytes
36saved_regs_stack_space: u32,
37
33/// The branch type of every branch38/// The branch type of every branch
34branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .{},39branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .{},
40
35/// For every forward branch, maps the target instruction to a list of41/// For every forward branch, maps the target instruction to a list of
36/// branches which branch to this target instruction42/// branches which branch to this target instruction
37branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .{},43branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .{},
44
38/// For backward branches: stores the code offset of the target45/// For backward branches: stores the code offset of the target
39/// instruction46/// instruction
40///47///
...@@ -42,6 +49,8 @@ branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUn...@@ -42,6 +49,8 @@ branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUn
42/// instruction49/// instruction
43code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},50code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
4451
52/// The final stack frame size of the function (already aligned to the
53/// respective stack alignment). Does not include prologue stack space.
45stack_size: u32,54stack_size: u32,
4655
47const InnerError = error{56const InnerError = error{
...@@ -82,9 +91,11 @@ pub fn emitMir(...@@ -82,9 +91,11 @@ pub fn emitMir(
82 .sub_immediate => try emit.mirAddSubtractImmediate(inst),91 .sub_immediate => try emit.mirAddSubtractImmediate(inst),
83 .subs_immediate => try emit.mirAddSubtractImmediate(inst),92 .subs_immediate => try emit.mirAddSubtractImmediate(inst),
8493
85 .asr_register => try emit.mirShiftRegister(inst),94 .asr_register => try emit.mirDataProcessing2Source(inst),
86 .lsl_register => try emit.mirShiftRegister(inst),95 .lsl_register => try emit.mirDataProcessing2Source(inst),
87 .lsr_register => try emit.mirShiftRegister(inst),96 .lsr_register => try emit.mirDataProcessing2Source(inst),
97 .sdiv => try emit.mirDataProcessing2Source(inst),
98 .udiv => try emit.mirDataProcessing2Source(inst),
8899
89 .asr_immediate => try emit.mirShiftImmediate(inst),100 .asr_immediate => try emit.mirShiftImmediate(inst),
90 .lsl_immediate => try emit.mirShiftImmediate(inst),101 .lsl_immediate => try emit.mirShiftImmediate(inst),
...@@ -148,6 +159,13 @@ pub fn emitMir(...@@ -148,6 +159,13 @@ pub fn emitMir(
148 .strb_stack => try emit.mirLoadStoreStack(inst),159 .strb_stack => try emit.mirLoadStoreStack(inst),
149 .strh_stack => try emit.mirLoadStoreStack(inst),160 .strh_stack => try emit.mirLoadStoreStack(inst),
150161
162 .ldr_stack_argument => try emit.mirLoadStackArgument(inst),
163 .ldr_ptr_stack_argument => try emit.mirLoadStackArgument(inst),
164 .ldrb_stack_argument => try emit.mirLoadStackArgument(inst),
165 .ldrh_stack_argument => try emit.mirLoadStackArgument(inst),
166 .ldrsb_stack_argument => try emit.mirLoadStackArgument(inst),
167 .ldrsh_stack_argument => try emit.mirLoadStackArgument(inst),
168
151 .ldr_register => try emit.mirLoadStoreRegisterRegister(inst),169 .ldr_register => try emit.mirLoadStoreRegisterRegister(inst),
152 .ldrb_register => try emit.mirLoadStoreRegisterRegister(inst),170 .ldrb_register => try emit.mirLoadStoreRegisterRegister(inst),
153 .ldrh_register => try emit.mirLoadStoreRegisterRegister(inst),171 .ldrh_register => try emit.mirLoadStoreRegisterRegister(inst),
...@@ -172,6 +190,7 @@ pub fn emitMir(...@@ -172,6 +190,7 @@ pub fn emitMir(
172 .movk => try emit.mirMoveWideImmediate(inst),190 .movk => try emit.mirMoveWideImmediate(inst),
173 .movz => try emit.mirMoveWideImmediate(inst),191 .movz => try emit.mirMoveWideImmediate(inst),
174192
193 .msub => try emit.mirDataProcessing3Source(inst),
175 .mul => try emit.mirDataProcessing3Source(inst),194 .mul => try emit.mirDataProcessing3Source(inst),
176 .smulh => try emit.mirDataProcessing3Source(inst),195 .smulh => try emit.mirDataProcessing3Source(inst),
177 .smull => try emit.mirDataProcessing3Source(inst),196 .smull => try emit.mirDataProcessing3Source(inst),
...@@ -504,7 +523,7 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -504,7 +523,7 @@ fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
504 }523 }
505}524}
506525
507fn mirShiftRegister(emit: *Emit, inst: Mir.Inst.Index) !void {526fn mirDataProcessing2Source(emit: *Emit, inst: Mir.Inst.Index) !void {
508 const tag = emit.mir.instructions.items(.tag)[inst];527 const tag = emit.mir.instructions.items(.tag)[inst];
509 const rrr = emit.mir.instructions.items(.data)[inst].rrr;528 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
510 const rd = rrr.rd;529 const rd = rrr.rd;
...@@ -515,6 +534,8 @@ fn mirShiftRegister(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -515,6 +534,8 @@ fn mirShiftRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
515 .asr_register => try emit.writeInstruction(Instruction.asrRegister(rd, rn, rm)),534 .asr_register => try emit.writeInstruction(Instruction.asrRegister(rd, rn, rm)),
516 .lsl_register => try emit.writeInstruction(Instruction.lslRegister(rd, rn, rm)),535 .lsl_register => try emit.writeInstruction(Instruction.lslRegister(rd, rn, rm)),
517 .lsr_register => try emit.writeInstruction(Instruction.lsrRegister(rd, rn, rm)),536 .lsr_register => try emit.writeInstruction(Instruction.lsrRegister(rd, rn, rm)),
537 .sdiv => try emit.writeInstruction(Instruction.sdiv(rd, rn, rm)),
538 .udiv => try emit.writeInstruction(Instruction.udiv(rd, rn, rm)),
518 else => unreachable,539 else => unreachable,
519 }540 }
520}541}
...@@ -920,6 +941,67 @@ fn mirLoadStoreRegisterPair(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -920,6 +941,67 @@ fn mirLoadStoreRegisterPair(emit: *Emit, inst: Mir.Inst.Index) !void {
920 }941 }
921}942}
922943
944fn mirLoadStackArgument(emit: *Emit, inst: Mir.Inst.Index) !void {
945 const tag = emit.mir.instructions.items(.tag)[inst];
946 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
947 const rt = load_store_stack.rt;
948
949 const raw_offset = emit.stack_size + emit.saved_regs_stack_space + load_store_stack.offset;
950 switch (tag) {
951 .ldr_ptr_stack_argument => {
952 const offset = if (math.cast(u12, raw_offset)) |imm| imm else {
953 return emit.fail("TODO load stack argument ptr with larger offset", .{});
954 };
955
956 switch (tag) {
957 .ldr_ptr_stack_argument => try emit.writeInstruction(Instruction.add(rt, .sp, offset, false)),
958 else => unreachable,
959 }
960 },
961 .ldrb_stack_argument, .ldrsb_stack_argument => {
962 const offset = if (math.cast(u12, raw_offset)) |imm| Instruction.LoadStoreOffset.imm(imm) else {
963 return emit.fail("TODO load stack argument byte with larger offset", .{});
964 };
965
966 switch (tag) {
967 .ldrb_stack_argument => try emit.writeInstruction(Instruction.ldrb(rt, .sp, offset)),
968 .ldrsb_stack_argument => try emit.writeInstruction(Instruction.ldrsb(rt, .sp, offset)),
969 else => unreachable,
970 }
971 },
972 .ldrh_stack_argument, .ldrsh_stack_argument => {
973 assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry
974 const offset = if (math.cast(u12, @divExact(raw_offset, 2))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
975 return emit.fail("TODO load stack argument halfword with larger offset", .{});
976 };
977
978 switch (tag) {
979 .ldrh_stack_argument => try emit.writeInstruction(Instruction.ldrh(rt, .sp, offset)),
980 .ldrsh_stack_argument => try emit.writeInstruction(Instruction.ldrsh(rt, .sp, offset)),
981 else => unreachable,
982 }
983 },
984 .ldr_stack_argument => {
985 const alignment: u32 = switch (rt.size()) {
986 32 => 4,
987 64 => 8,
988 else => unreachable,
989 };
990
991 assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry
992 const offset = if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
993 return emit.fail("TODO load stack argument with larger offset", .{});
994 };
995
996 switch (tag) {
997 .ldr_stack_argument => try emit.writeInstruction(Instruction.ldr(rt, .sp, offset)),
998 else => unreachable,
999 }
1000 },
1001 else => unreachable,
1002 }
1003}
1004
923fn mirLoadStoreStack(emit: *Emit, inst: Mir.Inst.Index) !void {1005fn mirLoadStoreStack(emit: *Emit, inst: Mir.Inst.Index) !void {
924 const tag = emit.mir.instructions.items(.tag)[inst];1006 const tag = emit.mir.instructions.items(.tag)[inst];
925 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;1007 const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
...@@ -1059,14 +1141,31 @@ fn mirMoveWideImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -1059,14 +1141,31 @@ fn mirMoveWideImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
10591141
1060fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {1142fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
1061 const tag = emit.mir.instructions.items(.tag)[inst];1143 const tag = emit.mir.instructions.items(.tag)[inst];
1062 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
10631144
1064 switch (tag) {1145 switch (tag) {
1065 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),1146 .mul,
1066 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),1147 .smulh,
1067 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),1148 .smull,
1068 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),1149 .umulh,
1069 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),1150 .umull,
1151 => {
1152 const rrr = emit.mir.instructions.items(.data)[inst].rrr;
1153 switch (tag) {
1154 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1155 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
1156 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
1157 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
1158 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
1159 else => unreachable,
1160 }
1161 },
1162 .msub => {
1163 const rrrr = emit.mir.instructions.items(.data)[inst].rrrr;
1164 switch (tag) {
1165 .msub => try emit.writeInstruction(Instruction.msub(rrrr.rd, rrrr.rn, rrrr.rm, rrrr.ra)),
1166 else => unreachable,
1167 }
1168 },
1070 else => unreachable,1169 else => unreachable,
1071 }1170 }
1072}1171}
src/arch/aarch64/Mir.zig+27
...@@ -92,20 +92,28 @@ pub const Inst = struct {...@@ -92,20 +92,28 @@ pub const Inst = struct {
92 load_memory_ptr_direct,92 load_memory_ptr_direct,
93 /// Load Pair of Registers93 /// Load Pair of Registers
94 ldp,94 ldp,
95 /// Pseudo-instruction: Load pointer to stack argument
96 ldr_ptr_stack_argument,
95 /// Pseudo-instruction: Load from stack97 /// Pseudo-instruction: Load from stack
96 ldr_stack,98 ldr_stack,
99 /// Pseudo-instruction: Load from stack argument
100 ldr_stack_argument,
97 /// Load Register (immediate)101 /// Load Register (immediate)
98 ldr_immediate,102 ldr_immediate,
99 /// Load Register (register)103 /// Load Register (register)
100 ldr_register,104 ldr_register,
101 /// Pseudo-instruction: Load byte from stack105 /// Pseudo-instruction: Load byte from stack
102 ldrb_stack,106 ldrb_stack,
107 /// Pseudo-instruction: Load byte from stack argument
108 ldrb_stack_argument,
103 /// Load Register Byte (immediate)109 /// Load Register Byte (immediate)
104 ldrb_immediate,110 ldrb_immediate,
105 /// Load Register Byte (register)111 /// Load Register Byte (register)
106 ldrb_register,112 ldrb_register,
107 /// Pseudo-instruction: Load halfword from stack113 /// Pseudo-instruction: Load halfword from stack
108 ldrh_stack,114 ldrh_stack,
115 /// Pseudo-instruction: Load halfword from stack argument
116 ldrh_stack_argument,
109 /// Load Register Halfword (immediate)117 /// Load Register Halfword (immediate)
110 ldrh_immediate,118 ldrh_immediate,
111 /// Load Register Halfword (register)119 /// Load Register Halfword (register)
...@@ -114,10 +122,14 @@ pub const Inst = struct {...@@ -114,10 +122,14 @@ pub const Inst = struct {
114 ldrsb_immediate,122 ldrsb_immediate,
115 /// Pseudo-instruction: Load signed byte from stack123 /// Pseudo-instruction: Load signed byte from stack
116 ldrsb_stack,124 ldrsb_stack,
125 /// Pseudo-instruction: Load signed byte from stack argument
126 ldrsb_stack_argument,
117 /// Load Register Signed Halfword (immediate)127 /// Load Register Signed Halfword (immediate)
118 ldrsh_immediate,128 ldrsh_immediate,
119 /// Pseudo-instruction: Load signed halfword from stack129 /// Pseudo-instruction: Load signed halfword from stack
120 ldrsh_stack,130 ldrsh_stack,
131 /// Pseudo-instruction: Load signed halfword from stack argument
132 ldrsh_stack_argument,
121 /// Load Register Signed Word (immediate)133 /// Load Register Signed Word (immediate)
122 ldrsw_immediate,134 ldrsw_immediate,
123 /// Logical Shift Left (immediate)135 /// Logical Shift Left (immediate)
...@@ -136,6 +148,8 @@ pub const Inst = struct {...@@ -136,6 +148,8 @@ pub const Inst = struct {
136 movk,148 movk,
137 /// Move wide with zero149 /// Move wide with zero
138 movz,150 movz,
151 /// Multiply-subtract
152 msub,
139 /// Multiply153 /// Multiply
140 mul,154 mul,
141 /// Bitwise NOT155 /// Bitwise NOT
...@@ -152,6 +166,8 @@ pub const Inst = struct {...@@ -152,6 +166,8 @@ pub const Inst = struct {
152 ret,166 ret,
153 /// Signed bitfield extract167 /// Signed bitfield extract
154 sbfx,168 sbfx,
169 /// Signed divide
170 sdiv,
155 /// Signed multiply high171 /// Signed multiply high
156 smulh,172 smulh,
157 /// Signed multiply long173 /// Signed multiply long
...@@ -200,6 +216,8 @@ pub const Inst = struct {...@@ -200,6 +216,8 @@ pub const Inst = struct {
200 tst_immediate,216 tst_immediate,
201 /// Unsigned bitfield extract217 /// Unsigned bitfield extract
202 ubfx,218 ubfx,
219 /// Unsigned divide
220 udiv,
203 /// Unsigned multiply high221 /// Unsigned multiply high
204 umulh,222 umulh,
205 /// Unsigned multiply long223 /// Unsigned multiply long
...@@ -430,6 +448,15 @@ pub const Inst = struct {...@@ -430,6 +448,15 @@ pub const Inst = struct {
430 rn: Register,448 rn: Register,
431 offset: bits.Instruction.LoadStorePairOffset,449 offset: bits.Instruction.LoadStorePairOffset,
432 },450 },
451 /// Four registers
452 ///
453 /// Used by e.g. msub
454 rrrr: struct {
455 rd: Register,
456 rn: Register,
457 rm: Register,
458 ra: Register,
459 },
433 /// Debug info: line and column460 /// Debug info: line and column
434 ///461 ///
435 /// Used by e.g. dbg_line462 /// Used by e.g. dbg_line
src/arch/aarch64/bits.zig+8
...@@ -1698,6 +1698,14 @@ pub const Instruction = union(enum) {...@@ -1698,6 +1698,14 @@ pub const Instruction = union(enum) {
16981698
1699 // Data processing (2 source)1699 // Data processing (2 source)
17001700
1701 pub fn udiv(rd: Register, rn: Register, rm: Register) Instruction {
1702 return dataProcessing2Source(0b0, 0b000010, rd, rn, rm);
1703 }
1704
1705 pub fn sdiv(rd: Register, rn: Register, rm: Register) Instruction {
1706 return dataProcessing2Source(0b0, 0b000011, rd, rn, rm);
1707 }
1708
1701 pub fn lslv(rd: Register, rn: Register, rm: Register) Instruction {1709 pub fn lslv(rd: Register, rn: Register, rm: Register) Instruction {
1702 return dataProcessing2Source(0b0, 0b001000, rd, rn, rm);1710 return dataProcessing2Source(0b0, 0b001000, rd, rn, rm);
1703 }1711 }