authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-07 11:28:57+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-07 13:19:53+02:00
logbf11cdc9d880555aee087a9bc0ecd1424428b63b
tree910d4ef4450555b104ba1aefa535b69d9318e85b
parent197c2a465f51b61bd3d2e58c5e6fecadacb20894

x64: refactor code to avoid stage1 sema limitations

2 files changed, 448 insertions(+), 395 deletions(-)

src/arch/x86_64/CodeGen.zig+447-394
...@@ -191,36 +191,12 @@ pub const MCValue = union(enum) {...@@ -191,36 +191,12 @@ pub const MCValue = union(enum) {
191 };191 };
192 }192 }
193193
194 fn usesCompareFlags(mcv: MCValue) bool {
195 return switch (mcv) {
196 .compare_flags_unsigned,
197 .compare_flags_signed,
198 .register_overflow_unsigned,
199 .register_overflow_signed,
200 => true,
201 else => false,
202 };
203 }
204
205 fn isRegister(mcv: MCValue) bool {194 fn isRegister(mcv: MCValue) bool {
206 return switch (mcv) {195 return switch (mcv) {
207 .register,196 .register => true,
208 .register_overflow_unsigned,
209 .register_overflow_signed,
210 => true,
211 else => false,197 else => false,
212 };198 };
213 }199 }
214
215 fn asRegister(mcv: MCValue) ?Register {
216 return switch (mcv) {
217 .register,
218 .register_overflow_unsigned,
219 .register_overflow_signed,
220 => |reg| reg,
221 else => null,
222 };
223 }
224};200};
225201
226const Branch = struct {202const Branch = struct {
...@@ -852,15 +828,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live...@@ -852,15 +828,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
852 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];828 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
853 branch.inst_table.putAssumeCapacityNoClobber(inst, result);829 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
854830
855 if (result.asRegister()) |reg| {831 switch (result) {
856 // In some cases (such as bitcast), an operand832 .register,
857 // may be the same MCValue as the result. If833 .register_overflow_signed,
858 // that operand died and was a register, it834 .register_overflow_unsigned,
859 // was freed by processDeath. We have to835 => |reg| {
860 // "re-allocate" the register.836 // In some cases (such as bitcast), an operand
861 if (self.register_manager.isRegFree(reg)) {837 // may be the same MCValue as the result. If
862 self.register_manager.getRegAssumeFree(reg, inst);838 // that operand died and was a register, it
863 }839 // was freed by processDeath. We have to
840 // "re-allocate" the register.
841 if (self.register_manager.isRegFree(reg)) {
842 self.register_manager.getRegAssumeFree(reg, inst);
843 }
844 },
845 else => {},
864 }846 }
865 }847 }
866 self.finishAirBookkeeping();848 self.finishAirBookkeeping();
...@@ -948,18 +930,32 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void...@@ -948,18 +930,32 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
948pub fn spillCompareFlagsIfOccupied(self: *Self) !void {930pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
949 if (self.compare_flags_inst) |inst_to_save| {931 if (self.compare_flags_inst) |inst_to_save| {
950 const mcv = self.getResolvedInstValue(inst_to_save);932 const mcv = self.getResolvedInstValue(inst_to_save);
951 assert(mcv.usesCompareFlags());933 const new_mcv = switch (mcv) {
934 .register_overflow_signed,
935 .register_overflow_unsigned,
936 => try self.allocRegOrMem(inst_to_save, false),
937 .compare_flags_signed,
938 .compare_flags_unsigned,
939 => try self.allocRegOrMem(inst_to_save, true),
940 else => unreachable,
941 };
952942
953 const new_mcv = try self.allocRegOrMem(inst_to_save, !mcv.isRegister());
954 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);943 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
955 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });944 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
945
956 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];946 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
957 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);947 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
958948
959 self.compare_flags_inst = null;949 self.compare_flags_inst = null;
950
960 // TODO consolidate with register manager and spillInstruction951 // TODO consolidate with register manager and spillInstruction
961 // this call should really belong in the register manager!952 // this call should really belong in the register manager!
962 if (mcv.isRegister()) self.register_manager.freeReg(mcv.asRegister().?);953 switch (mcv) {
954 .register_overflow_signed,
955 .register_overflow_unsigned,
956 => |reg| self.register_manager.freeReg(reg),
957 else => {},
958 }
963 }959 }
964}960}
965961
...@@ -1031,7 +1027,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1031,7 +1027,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1031 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1027 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1032 else => null,1028 else => null,
1033 };1029 };
1034 defer if (operand_lock) |reg| self.register_manager.unlockReg(reg);1030 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
10351031
1036 const reg = try self.register_manager.allocReg(inst);1032 const reg = try self.register_manager.allocReg(inst);
1037 try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });1033 try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
...@@ -1062,7 +1058,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -1062,7 +1058,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1062 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1058 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1063 else => null,1059 else => null,
1064 };1060 };
1065 defer if (operand_lock) |reg| self.register_manager.unlockReg(reg);1061 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
10661062
1067 const reg: Register = blk: {1063 const reg: Register = blk: {
1068 if (operand.isRegister()) {1064 if (operand.isRegister()) {
...@@ -1150,7 +1146,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {...@@ -1150,7 +1146,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1150 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1146 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1151 else => null,1147 else => null,
1152 };1148 };
1153 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);1149 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
11541150
1155 const lhs_reg = try self.copyToTmpRegister(ty, lhs);1151 const lhs_reg = try self.copyToTmpRegister(ty, lhs);
1156 const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);1152 const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);
...@@ -1161,7 +1157,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {...@@ -1161,7 +1157,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1161 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1157 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1162 else => null,1158 else => null,
1163 };1159 };
1164 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);1160 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
11651161
1166 try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);1162 try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);
11671163
...@@ -1200,9 +1196,9 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r...@@ -1200,9 +1196,9 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
1200 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1196 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1201 else => null,1197 else => null,
1202 };1198 };
1203 defer if (offset_lock) |reg| self.register_manager.unlockReg(reg);1199 defer if (offset_lock) |lock| self.register_manager.unlockReg(lock);
12041200
1205 const dst_mcv = blk: {1201 const dst_mcv: MCValue = blk: {
1206 if (self.reuseOperand(inst, op_lhs, 0, ptr)) {1202 if (self.reuseOperand(inst, op_lhs, 0, ptr)) {
1207 if (ptr.isMemory() or ptr.isRegister()) break :blk ptr;1203 if (ptr.isMemory() or ptr.isRegister()) break :blk ptr;
1208 }1204 }
...@@ -1213,9 +1209,9 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r...@@ -1213,9 +1209,9 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
1213 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1209 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1214 else => null,1210 else => null,
1215 };1211 };
1216 defer if (dst_mcv_lock) |reg| self.register_manager.unlockReg(reg);1212 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
12171213
1218 const offset_mcv = blk: {1214 const offset_mcv: MCValue = blk: {
1219 if (self.reuseOperand(inst, op_rhs, 1, offset)) {1215 if (self.reuseOperand(inst, op_rhs, 1, offset)) {
1220 if (offset.isRegister()) break :blk offset;1216 if (offset.isRegister()) break :blk offset;
1221 }1217 }
...@@ -1226,7 +1222,7 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r...@@ -1226,7 +1222,7 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
1226 .register => |reg| self.register_manager.lockReg(reg),1222 .register => |reg| self.register_manager.lockReg(reg),
1227 else => null,1223 else => null,
1228 };1224 };
1229 defer if (offset_mcv_lock) |reg| self.register_manager.unlockReg(reg);1225 defer if (offset_mcv_lock) |lock| self.register_manager.unlockReg(lock);
12301226
1231 try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });1227 try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });
12321228
...@@ -1315,16 +1311,16 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air...@@ -1315,16 +1311,16 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1315 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1311 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1316 else => null,1312 else => null,
1317 };1313 };
1318 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);1314 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
13191315
1320 const rhs = try self.resolveInst(op_rhs);1316 const rhs = try self.resolveInst(op_rhs);
1321 const rhs_lock: ?RegisterLock = switch (rhs) {1317 const rhs_lock: ?RegisterLock = switch (rhs) {
1322 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1318 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1323 else => null,1319 else => null,
1324 };1320 };
1325 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);1321 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
13261322
1327 const dst_mcv = blk: {1323 const dst_mcv: MCValue = blk: {
1328 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {1324 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
1329 break :blk lhs;1325 break :blk lhs;
1330 }1326 }
...@@ -1334,9 +1330,9 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air...@@ -1334,9 +1330,9 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1334 .register => |reg| self.register_manager.lockReg(reg),1330 .register => |reg| self.register_manager.lockReg(reg),
1335 else => null,1331 else => null,
1336 };1332 };
1337 defer if (dst_mcv_lock) |reg| self.register_manager.unlockReg(reg);1333 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
13381334
1339 const rhs_mcv = blk: {1335 const rhs_mcv: MCValue = blk: {
1340 if (rhs.isMemory() or rhs.isRegister()) break :blk rhs;1336 if (rhs.isMemory() or rhs.isRegister()) break :blk rhs;
1341 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };1337 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
1342 };1338 };
...@@ -1344,7 +1340,7 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air...@@ -1344,7 +1340,7 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1344 .register => |reg| self.register_manager.lockReg(reg),1340 .register => |reg| self.register_manager.lockReg(reg),
1345 else => null,1341 else => null,
1346 };1342 };
1347 defer if (rhs_mcv_lock) |reg| self.register_manager.unlockReg(reg);1343 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
13481344
1349 try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv);1345 try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv);
13501346
...@@ -1476,9 +1472,13 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1476,9 +1472,13 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1476fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {1472fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1477 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;1473 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1478 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;1474 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1479 const result = if (self.liveness.isUnused(inst)) .dead else result: {
1480 const ty = self.air.typeOf(bin_op.lhs);
14811475
1476 if (self.liveness.isUnused(inst)) {
1477 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1478 }
1479
1480 const ty = self.air.typeOf(bin_op.lhs);
1481 const result: MCValue = result: {
1482 switch (ty.zigTypeTag()) {1482 switch (ty.zigTypeTag()) {
1483 .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),1483 .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
1484 .Int => {1484 .Int => {
...@@ -1529,7 +1529,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1529,7 +1529,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1529 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),1529 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1530 else => null,1530 else => null,
1531 };1531 };
1532 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);1532 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
15331533
1534 const dst_reg: Register = blk: {1534 const dst_reg: Register = blk: {
1535 if (lhs.isRegister()) break :blk lhs.register;1535 if (lhs.isRegister()) break :blk lhs.register;
...@@ -1538,7 +1538,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1538,7 +1538,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1538 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);1538 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1539 defer self.register_manager.unlockReg(dst_reg_lock);1539 defer self.register_manager.unlockReg(dst_reg_lock);
15401540
1541 const rhs_mcv = blk: {1541 const rhs_mcv: MCValue = blk: {
1542 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;1542 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
1543 break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };1543 break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };
1544 };1544 };
...@@ -1546,7 +1546,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1546,7 +1546,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1546 .register => |reg| self.register_manager.lockReg(reg),1546 .register => |reg| self.register_manager.lockReg(reg),
1547 else => null,1547 else => null,
1548 };1548 };
1549 defer if (rhs_mcv_lock) |reg| self.register_manager.unlockReg(reg);1549 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
15501550
1551 try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);1551 try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
15521552
...@@ -1734,19 +1734,19 @@ fn genIntMulDivOpMir(...@@ -1734,19 +1734,19 @@ fn genIntMulDivOpMir(
1734/// Clobbers .rax and .rdx registers.1734/// Clobbers .rax and .rdx registers.
1735fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {1735fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
1736 const signedness = ty.intInfo(self.target.*).signedness;1736 const signedness = ty.intInfo(self.target.*).signedness;
1737 const dividend = switch (lhs) {1737 const dividend: Register = switch (lhs) {
1738 .register => |reg| reg,1738 .register => |reg| reg,
1739 else => try self.copyToTmpRegister(ty, lhs),1739 else => try self.copyToTmpRegister(ty, lhs),
1740 };1740 };
1741 const dividend_lock = self.register_manager.lockReg(dividend);1741 const dividend_lock = self.register_manager.lockReg(dividend);
1742 defer if (dividend_lock) |reg| self.register_manager.unlockReg(reg);1742 defer if (dividend_lock) |lock| self.register_manager.unlockReg(lock);
17431743
1744 const divisor = switch (rhs) {1744 const divisor: Register = switch (rhs) {
1745 .register => |reg| reg,1745 .register => |reg| reg,
1746 else => try self.copyToTmpRegister(ty, rhs),1746 else => try self.copyToTmpRegister(ty, rhs),
1747 };1747 };
1748 const divisor_lock = self.register_manager.lockReg(divisor);1748 const divisor_lock = self.register_manager.lockReg(divisor);
1749 defer if (divisor_lock) |reg| self.register_manager.unlockReg(reg);1749 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
17501750
1751 try self.genIntMulDivOpMir(switch (signedness) {1751 try self.genIntMulDivOpMir(switch (signedness) {
1752 .signed => .idiv,1752 .signed => .idiv,
...@@ -1791,67 +1791,72 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -1791,67 +1791,72 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
17911791
1792fn airDiv(self: *Self, inst: Air.Inst.Index) !void {1792fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1793 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1793 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1794 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1795 const tag = self.air.instructions.items(.tag)[inst];
1796 const ty = self.air.typeOfIndex(inst);
17971794
1798 if (ty.zigTypeTag() != .Int) {1795 if (self.liveness.isUnused(inst)) {
1799 return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });1796 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1800 }1797 }
18011798
1802 if (tag == .div_float) {1799 const tag = self.air.instructions.items(.tag)[inst];
1803 return self.fail("TODO implement {}", .{tag});1800 const ty = self.air.typeOfIndex(inst);
1801
1802 if (ty.zigTypeTag() != .Int) {
1803 return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });
1804 }
1805
1806 if (tag == .div_float) {
1807 return self.fail("TODO implement {}", .{tag});
1808 }
1809
1810 const signedness = ty.intInfo(self.target.*).signedness;
1811
1812 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1813 const track_rax: ?Air.Inst.Index = blk: {
1814 if (signedness == .unsigned) break :blk inst;
1815 switch (tag) {
1816 .div_exact, .div_trunc => break :blk inst,
1817 else => break :blk null,
1804 }1818 }
1819 };
1820 try self.register_manager.getReg(.rax, track_rax);
1821 try self.register_manager.getReg(.rdx, null);
1822 var reg_locks: [2]RegisterLock = undefined;
1823 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1824 defer for (reg_locks) |reg| {
1825 self.register_manager.unlockReg(reg);
1826 };
18051827
1806 const signedness = ty.intInfo(self.target.*).signedness;1828 const lhs = try self.resolveInst(bin_op.lhs);
1829 const lhs_lock: ?RegisterLock = switch (lhs) {
1830 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1831 else => null,
1832 };
1833 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
18071834
1808 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.1835 const rhs: MCValue = blk: {
1809 const track_rax: ?Air.Inst.Index = blk: {1836 const rhs = try self.resolveInst(bin_op.rhs);
1810 if (signedness == .unsigned) break :blk inst;1837 if (signedness == .signed) {
1811 switch (tag) {1838 switch (tag) {
1812 .div_exact, .div_trunc => break :blk inst,1839 .div_floor => {
1813 else => break :blk null,1840 const rhs_lock: ?RegisterLock = switch (rhs) {
1814 }1841 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1815 };1842 else => null,
1816 try self.register_manager.getReg(.rax, track_rax);1843 };
1817 try self.register_manager.getReg(.rdx, null);1844 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1818 var reg_locks: [2]RegisterLock = undefined;
1819 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1820 defer for (reg_locks) |reg| {
1821 self.register_manager.unlockReg(reg);
1822 };
1823
1824 const lhs = try self.resolveInst(bin_op.lhs);
1825 const lhs_lock: ?RegisterLock = switch (lhs) {
1826 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1827 else => null,
1828 };
1829 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
1830
1831 const rhs = blk: {
1832 const rhs = try self.resolveInst(bin_op.rhs);
1833 if (signedness == .signed) {
1834 switch (tag) {
1835 .div_floor => {
1836 const rhs_lock: ?RegisterLock = switch (rhs) {
1837 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1838 else => null,
1839 };
1840 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
18411845
1842 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);1846 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1843 },1847 },
1844 else => {},1848 else => {},
1845 }
1846 }1849 }
1847 break :blk rhs;1850 }
1848 };1851 break :blk rhs;
1849 const rhs_lock: ?RegisterLock = switch (rhs) {1852 };
1850 .register => |reg| self.register_manager.lockReg(reg),1853 const rhs_lock: ?RegisterLock = switch (rhs) {
1851 else => null,1854 .register => |reg| self.register_manager.lockReg(reg),
1852 };1855 else => null,
1853 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);1856 };
1857 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
18541858
1859 const result: MCValue = result: {
1855 if (signedness == .unsigned) {1860 if (signedness == .unsigned) {
1856 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);1861 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);
1857 break :result MCValue{ .register = .rax };1862 break :result MCValue{ .register = .rax };
...@@ -1871,59 +1876,69 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {...@@ -1871,59 +1876,69 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1871 else => unreachable,1876 else => unreachable,
1872 }1877 }
1873 };1878 };
1879
1874 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1880 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1875}1881}
18761882
1877fn airRem(self: *Self, inst: Air.Inst.Index) !void {1883fn airRem(self: *Self, inst: Air.Inst.Index) !void {
1878 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1884 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1879 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1880 const ty = self.air.typeOfIndex(inst);
1881 if (ty.zigTypeTag() != .Int) {
1882 return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
1883 }
1884 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1885 try self.register_manager.getReg(.rax, null);
1886 try self.register_manager.getReg(.rdx, inst);
1887 var reg_locks: [2]RegisterLock = undefined;
1888 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1889 defer for (reg_locks) |reg| {
1890 self.register_manager.unlockReg(reg);
1891 };
1892
1893 const lhs = try self.resolveInst(bin_op.lhs);
1894 const rhs = try self.resolveInst(bin_op.rhs);
18951885
1896 const signedness = ty.intInfo(self.target.*).signedness;1886 if (self.liveness.isUnused(inst)) {
1897 try self.genIntMulDivOpMir(switch (signedness) {1887 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1898 .signed => .idiv,1888 }
1899 .unsigned => .div,1889 const ty = self.air.typeOfIndex(inst);
1900 }, ty, signedness, lhs, rhs);1890 if (ty.zigTypeTag() != .Int) {
1901 break :result MCValue{ .register = .rdx };1891 return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
1892 }
1893 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1894 try self.register_manager.getReg(.rax, null);
1895 try self.register_manager.getReg(.rdx, inst);
1896 var reg_locks: [2]RegisterLock = undefined;
1897 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1898 defer for (reg_locks) |reg| {
1899 self.register_manager.unlockReg(reg);
1902 };1900 };
1901
1902 const lhs = try self.resolveInst(bin_op.lhs);
1903 const rhs = try self.resolveInst(bin_op.rhs);
1904
1905 const signedness = ty.intInfo(self.target.*).signedness;
1906 try self.genIntMulDivOpMir(switch (signedness) {
1907 .signed => .idiv,
1908 .unsigned => .div,
1909 }, ty, signedness, lhs, rhs);
1910
1911 const result: MCValue = .{ .register = .rdx };
1912
1903 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1913 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1904}1914}
19051915
1906fn airMod(self: *Self, inst: Air.Inst.Index) !void {1916fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1907 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1917 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1908 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1909 const ty = self.air.typeOfIndex(inst);
1910 if (ty.zigTypeTag() != .Int) {
1911 return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
1912 }
1913 const signedness = ty.intInfo(self.target.*).signedness;
19141918
1915 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.1919 if (self.liveness.isUnused(inst)) {
1916 try self.register_manager.getReg(.rax, null);1920 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1917 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);1921 }
1918 var reg_locks: [2]RegisterLock = undefined;
1919 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1920 defer for (reg_locks) |reg| {
1921 self.register_manager.unlockReg(reg);
1922 };
19231922
1924 const lhs = try self.resolveInst(bin_op.lhs);1923 const ty = self.air.typeOfIndex(inst);
1925 const rhs = try self.resolveInst(bin_op.rhs);1924 if (ty.zigTypeTag() != .Int) {
1925 return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
1926 }
1927 const signedness = ty.intInfo(self.target.*).signedness;
19261928
1929 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1930 try self.register_manager.getReg(.rax, null);
1931 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);
1932 var reg_locks: [2]RegisterLock = undefined;
1933 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1934 defer for (reg_locks) |reg| {
1935 self.register_manager.unlockReg(reg);
1936 };
1937
1938 const lhs = try self.resolveInst(bin_op.lhs);
1939 const rhs = try self.resolveInst(bin_op.rhs);
1940
1941 const result: MCValue = result: {
1927 switch (signedness) {1942 switch (signedness) {
1928 .unsigned => {1943 .unsigned => {
1929 try self.genIntMulDivOpMir(switch (signedness) {1944 try self.genIntMulDivOpMir(switch (signedness) {
...@@ -1943,6 +1958,7 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {...@@ -1943,6 +1958,7 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1943 },1958 },
1944 }1959 }
1945 };1960 };
1961
1946 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1962 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1947}1963}
19481964
...@@ -2017,7 +2033,7 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {...@@ -2017,7 +2033,7 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {
2017 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2033 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2018 else => null,2034 else => null,
2019 };2035 };
2020 defer if (value_lock) |reg| self.register_manager.unlockReg(reg);2036 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
20212037
2022 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value);2038 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value);
2023 _ = try self.addInst(.{2039 _ = try self.addInst(.{
...@@ -2117,7 +2133,7 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2117,7 +2133,7 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
2117 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2133 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2118 else => null,2134 else => null,
2119 };2135 };
2120 defer if (operand_lock) |reg| self.register_manager.unlockReg(reg);2136 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
21212137
2122 const result: MCValue = result: {2138 const result: MCValue = result: {
2123 if (!payload_ty.hasRuntimeBits()) break :result operand;2139 if (!payload_ty.hasRuntimeBits()) break :result operand;
...@@ -2150,7 +2166,7 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -2150,7 +2166,7 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
2150 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2166 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2151 else => null,2167 else => null,
2152 };2168 };
2153 defer if (operand_lock) |reg| self.register_manager.unlockReg(reg);2169 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
21542170
2155 const abi_align = err_union_ty.abiAlignment(self.target.*);2171 const abi_align = err_union_ty.abiAlignment(self.target.*);
2156 const err_ty = err_union_ty.errorUnionSet();2172 const err_ty = err_union_ty.errorUnionSet();
...@@ -2222,7 +2238,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2222,7 +2238,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2222 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2238 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2223 else => null,2239 else => null,
2224 };2240 };
2225 defer if (operand_lock) |reg| self.register_manager.unlockReg(reg);2241 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
22262242
2227 if (optional_ty.isPtrLikeOptional()) {2243 if (optional_ty.isPtrLikeOptional()) {
2228 // TODO should we check if we can reuse the operand?2244 // TODO should we check if we can reuse the operand?
...@@ -2359,7 +2375,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -2359,7 +2375,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2359 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2375 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2360 else => null,2376 else => null,
2361 };2377 };
2362 defer if (slice_mcv_lock) |reg| self.register_manager.unlockReg(reg);2378 defer if (slice_mcv_lock) |lock| self.register_manager.unlockReg(lock);
23632379
2364 const elem_ty = slice_ty.childType();2380 const elem_ty = slice_ty.childType();
2365 const elem_size = elem_ty.abiSize(self.target.*);2381 const elem_size = elem_ty.abiSize(self.target.*);
...@@ -2372,7 +2388,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -2372,7 +2388,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2372 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2388 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2373 else => null,2389 else => null,
2374 };2390 };
2375 defer if (index_mcv_lock) |reg| self.register_manager.unlockReg(reg);2391 defer if (index_mcv_lock) |lock| self.register_manager.unlockReg(lock);
23762392
2377 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);2393 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);
2378 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);2394 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
...@@ -2429,110 +2445,119 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2429,110 +2445,119 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
24292445
2430fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {2446fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2431 const bin_op = self.air.instructions.items(.data)[inst].bin_op;2447 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2432 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2433 const array_ty = self.air.typeOf(bin_op.lhs);
2434 const array = try self.resolveInst(bin_op.lhs);
2435 const array_lock: ?RegisterLock = switch (array) {
2436 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2437 else => null,
2438 };
2439 defer if (array_lock) |reg| self.register_manager.unlockReg(reg);
24402448
2441 const elem_ty = array_ty.childType();2449 if (self.liveness.isUnused(inst)) {
2442 const elem_abi_size = elem_ty.abiSize(self.target.*);2450 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2451 }
24432452
2444 const index_ty = self.air.typeOf(bin_op.rhs);2453 const array_ty = self.air.typeOf(bin_op.lhs);
2445 const index = try self.resolveInst(bin_op.rhs);2454 const array = try self.resolveInst(bin_op.lhs);
2446 const index_lock: ?RegisterLock = switch (index) {2455 const array_lock: ?RegisterLock = switch (array) {
2447 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2456 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2448 else => null,2457 else => null,
2449 };2458 };
2450 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);2459 defer if (array_lock) |lock| self.register_manager.unlockReg(lock);
24512460
2452 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);2461 const elem_ty = array_ty.childType();
2453 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);2462 const elem_abi_size = elem_ty.abiSize(self.target.*);
2454 defer self.register_manager.unlockReg(offset_reg_lock);
24552463
2456 const addr_reg = try self.register_manager.allocReg(null);2464 const index_ty = self.air.typeOf(bin_op.rhs);
2457 switch (array) {2465 const index = try self.resolveInst(bin_op.rhs);
2458 .register => {2466 const index_lock: ?RegisterLock = switch (index) {
2459 const off = @intCast(i32, try self.allocMem(2467 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2460 inst,2468 else => null,
2461 @intCast(u32, array_ty.abiSize(self.target.*)),
2462 array_ty.abiAlignment(self.target.*),
2463 ));
2464 try self.genSetStack(array_ty, off, array, .{});
2465 // lea reg, [rbp]
2466 _ = try self.addInst(.{
2467 .tag = .lea,
2468 .ops = (Mir.Ops{
2469 .reg1 = addr_reg.to64(),
2470 .reg2 = .rbp,
2471 }).encode(),
2472 .data = .{ .imm = @bitCast(u32, -off) },
2473 });
2474 },
2475 .stack_offset => |off| {
2476 // lea reg, [rbp]
2477 _ = try self.addInst(.{
2478 .tag = .lea,
2479 .ops = (Mir.Ops{
2480 .reg1 = addr_reg.to64(),
2481 .reg2 = .rbp,
2482 }).encode(),
2483 .data = .{ .imm = @bitCast(u32, -off) },
2484 });
2485 },
2486 .memory,
2487 .got_load,
2488 .direct_load,
2489 => {
2490 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
2491 },
2492 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2493 }
2494
2495 // TODO we could allocate register here, but need to expect addr register and potentially
2496 // offset register.
2497 const dst_mcv = try self.allocRegOrMem(inst, false);
2498 try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2499 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
2500 break :result dst_mcv;
2501 };2469 };
2502 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });2470 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
2471
2472 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2473 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2474 defer self.register_manager.unlockReg(offset_reg_lock);
2475
2476 const addr_reg = try self.register_manager.allocReg(null);
2477 switch (array) {
2478 .register => {
2479 const off = @intCast(i32, try self.allocMem(
2480 inst,
2481 @intCast(u32, array_ty.abiSize(self.target.*)),
2482 array_ty.abiAlignment(self.target.*),
2483 ));
2484 try self.genSetStack(array_ty, off, array, .{});
2485 // lea reg, [rbp]
2486 _ = try self.addInst(.{
2487 .tag = .lea,
2488 .ops = (Mir.Ops{
2489 .reg1 = addr_reg.to64(),
2490 .reg2 = .rbp,
2491 }).encode(),
2492 .data = .{ .imm = @bitCast(u32, -off) },
2493 });
2494 },
2495 .stack_offset => |off| {
2496 // lea reg, [rbp]
2497 _ = try self.addInst(.{
2498 .tag = .lea,
2499 .ops = (Mir.Ops{
2500 .reg1 = addr_reg.to64(),
2501 .reg2 = .rbp,
2502 }).encode(),
2503 .data = .{ .imm = @bitCast(u32, -off) },
2504 });
2505 },
2506 .memory,
2507 .got_load,
2508 .direct_load,
2509 => {
2510 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
2511 },
2512 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2513 }
2514
2515 // TODO we could allocate register here, but need to expect addr register and potentially
2516 // offset register.
2517 const dst_mcv = try self.allocRegOrMem(inst, false);
2518 try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2519 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
2520
2521 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2503}2522}
25042523
2505fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {2524fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2506 const is_volatile = false; // TODO2525 const is_volatile = false; // TODO
2507 const bin_op = self.air.instructions.items(.data)[inst].bin_op;2526 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2508 const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else result: {
2509 // this is identical to the `airPtrElemPtr` codegen expect here an
2510 // additional `mov` is needed at the end to get the actual value
25112527
2512 const ptr_ty = self.air.typeOf(bin_op.lhs);2528 if (!is_volatile and self.liveness.isUnused(inst)) {
2513 const ptr = try self.resolveInst(bin_op.lhs);2529 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2514 const ptr_lock: ?RegisterLock = switch (ptr) {2530 }
2515 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2516 else => null,
2517 };
2518 defer if (ptr_lock) |reg| self.register_manager.unlockReg(reg);
25192531
2520 const elem_ty = ptr_ty.elemType2();2532 // this is identical to the `airPtrElemPtr` codegen expect here an
2521 const elem_abi_size = elem_ty.abiSize(self.target.*);2533 // additional `mov` is needed at the end to get the actual value
2522 const index_ty = self.air.typeOf(bin_op.rhs);
2523 const index = try self.resolveInst(bin_op.rhs);
2524 const index_lock: ?RegisterLock = switch (index) {
2525 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2526 else => null,
2527 };
2528 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
25292534
2530 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);2535 const ptr_ty = self.air.typeOf(bin_op.lhs);
2531 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);2536 const ptr = try self.resolveInst(bin_op.lhs);
2532 defer self.register_manager.unlockReg(offset_reg_lock);2537 const ptr_lock: ?RegisterLock = switch (ptr) {
2538 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2539 else => null,
2540 };
2541 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
25332542
2534 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);2543 const elem_ty = ptr_ty.elemType2();
2535 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });2544 const elem_abi_size = elem_ty.abiSize(self.target.*);
2545 const index_ty = self.air.typeOf(bin_op.rhs);
2546 const index = try self.resolveInst(bin_op.rhs);
2547 const index_lock: ?RegisterLock = switch (index) {
2548 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2549 else => null,
2550 };
2551 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
2552
2553 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2554 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2555 defer self.register_manager.unlockReg(offset_reg_lock);
2556
2557 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2558 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2559
2560 const result: MCValue = result: {
2536 if (elem_abi_size > 8) {2561 if (elem_abi_size > 8) {
2537 return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});2562 return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});
2538 } else {2563 } else {
...@@ -2549,40 +2574,44 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2549,40 +2574,44 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2549 break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };2574 break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2550 }2575 }
2551 };2576 };
2577
2552 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });2578 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2553}2579}
25542580
2555fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {2581fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2556 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;2582 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2557 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;2583 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2558 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2559 const ptr_ty = self.air.typeOf(extra.lhs);
2560 const ptr = try self.resolveInst(extra.lhs);
2561 const ptr_lock: ?RegisterLock = switch (ptr) {
2562 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2563 else => null,
2564 };
2565 defer if (ptr_lock) |reg| self.register_manager.unlockReg(reg);
25662584
2567 const elem_ty = ptr_ty.elemType2();2585 if (self.liveness.isUnused(inst)) {
2568 const elem_abi_size = elem_ty.abiSize(self.target.*);2586 return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2569 const index_ty = self.air.typeOf(extra.rhs);2587 }
2570 const index = try self.resolveInst(extra.rhs);
2571 const index_lock: ?RegisterLock = switch (index) {
2572 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2573 else => null,
2574 };
2575 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
25762588
2577 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);2589 const ptr_ty = self.air.typeOf(extra.lhs);
2578 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);2590 const ptr = try self.resolveInst(extra.lhs);
2579 defer self.register_manager.unlockReg(offset_reg_lock);2591 const ptr_lock: ?RegisterLock = switch (ptr) {
2592 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2593 else => null,
2594 };
2595 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
25802596
2581 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);2597 const elem_ty = ptr_ty.elemType2();
2582 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });2598 const elem_abi_size = elem_ty.abiSize(self.target.*);
2583 break :result dst_mcv;2599 const index_ty = self.air.typeOf(extra.rhs);
2600 const index = try self.resolveInst(extra.rhs);
2601 const index_lock: ?RegisterLock = switch (index) {
2602 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2603 else => null,
2584 };2604 };
2585 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });2605 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
2606
2607 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2608 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2609 defer self.register_manager.unlockReg(offset_reg_lock);
2610
2611 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2612 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2613
2614 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
2586}2615}
25872616
2588fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {2617fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
...@@ -2601,14 +2630,14 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -2601,14 +2630,14 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2601 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2630 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2602 else => null,2631 else => null,
2603 };2632 };
2604 defer if (ptr_lock) |reg| self.register_manager.unlockReg(reg);2633 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
26052634
2606 const tag = try self.resolveInst(bin_op.rhs);2635 const tag = try self.resolveInst(bin_op.rhs);
2607 const tag_lock: ?RegisterLock = switch (tag) {2636 const tag_lock: ?RegisterLock = switch (tag) {
2608 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2637 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2609 else => null,2638 else => null,
2610 };2639 };
2611 defer if (tag_lock) |reg| self.register_manager.unlockReg(reg);2640 defer if (tag_lock) |lock| self.register_manager.unlockReg(lock);
26122641
2613 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {2642 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
2614 // TODO reusing the operand2643 // TODO reusing the operand
...@@ -2642,7 +2671,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -2642,7 +2671,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2642 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2671 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2643 else => null,2672 else => null,
2644 };2673 };
2645 defer if (operand_lock) |reg| self.register_manager.unlockReg(reg);2674 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
26462675
2647 const tag_abi_size = tag_ty.abiSize(self.target.*);2676 const tag_abi_size = tag_ty.abiSize(self.target.*);
2648 const dst_mcv: MCValue = blk: {2677 const dst_mcv: MCValue = blk: {
...@@ -2790,7 +2819,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -2790,7 +2819,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2790 },2819 },
2791 .register => |reg| {2820 .register => |reg| {
2792 const reg_lock = self.register_manager.lockReg(reg);2821 const reg_lock = self.register_manager.lockReg(reg);
2793 defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);2822 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
27942823
2795 switch (dst_mcv) {2824 switch (dst_mcv) {
2796 .dead => unreachable,2825 .dead => unreachable,
...@@ -2916,7 +2945,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2916,7 +2945,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2916 },2945 },
2917 .register => |reg| {2946 .register => |reg| {
2918 const reg_lock = self.register_manager.lockReg(reg);2947 const reg_lock = self.register_manager.lockReg(reg);
2919 defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);2948 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
29202949
2921 switch (value) {2950 switch (value) {
2922 .none => unreachable,2951 .none => unreachable,
...@@ -3010,7 +3039,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3010,7 +3039,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3010 .register => |reg| self.register_manager.lockReg(reg),3039 .register => |reg| self.register_manager.lockReg(reg),
3011 else => null,3040 else => null,
3012 };3041 };
3013 defer if (value_lock) |reg| self.register_manager.unlockReg(reg);3042 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
30143043
3015 const addr_reg = try self.register_manager.allocReg(null);3044 const addr_reg = try self.register_manager.allocReg(null);
3016 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);3045 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
...@@ -3198,7 +3227,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde...@@ -3198,7 +3227,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
3198 defer self.register_manager.unlockReg(offset_reg_lock);3227 defer self.register_manager.unlockReg(offset_reg_lock);
31993228
3200 const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);3229 const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
3201 const result_reg = blk: {3230 const result_reg: Register = blk: {
3202 if (can_reuse_operand) {3231 if (can_reuse_operand) {
3203 break :blk reg;3232 break :blk reg;
3204 } else {3233 } else {
...@@ -3208,7 +3237,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde...@@ -3208,7 +3237,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
3208 }3237 }
3209 };3238 };
3210 const result_reg_lock = self.register_manager.lockReg(result_reg);3239 const result_reg_lock = self.register_manager.lockReg(result_reg);
3211 defer if (result_reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);3240 defer if (result_reg_lock) |lock| self.register_manager.unlockReg(lock);
32123241
3213 try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });3242 try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });
3214 break :result MCValue{ .register = result_reg };3243 break :result MCValue{ .register = result_reg };
...@@ -3224,12 +3253,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3224,12 +3253,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3224 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;3253 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
3225 const operand = extra.struct_operand;3254 const operand = extra.struct_operand;
3226 const index = extra.field_index;3255 const index = extra.field_index;
3227 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3228 const mcv = try self.resolveInst(operand);
3229 const struct_ty = self.air.typeOf(operand);
3230 const struct_field_offset = struct_ty.structFieldOffset(index, self.target.*);
3231 const struct_field_ty = struct_ty.structFieldType(index);
32323256
3257 if (self.liveness.isUnused(inst)) {
3258 return self.finishAir(inst, .dead, .{ extra.struct_operand, .none, .none });
3259 }
3260
3261 const mcv = try self.resolveInst(operand);
3262 const struct_ty = self.air.typeOf(operand);
3263 const struct_field_offset = struct_ty.structFieldOffset(index, self.target.*);
3264 const struct_field_ty = struct_ty.structFieldType(index);
3265
3266 const result: MCValue = result: {
3233 switch (mcv) {3267 switch (mcv) {
3234 .stack_offset => |off| {3268 .stack_offset => |off| {
3235 const stack_offset = off - @intCast(i32, struct_field_offset);3269 const stack_offset = off - @intCast(i32, struct_field_offset);
...@@ -3239,7 +3273,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3239,7 +3273,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3239 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);3273 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3240 defer self.register_manager.unlockReg(reg_lock);3274 defer self.register_manager.unlockReg(reg_lock);
32413275
3242 const dst_mcv = blk: {3276 const dst_mcv: MCValue = blk: {
3243 if (self.reuseOperand(inst, operand, 0, mcv)) {3277 if (self.reuseOperand(inst, operand, 0, mcv)) {
3244 break :blk mcv;3278 break :blk mcv;
3245 } else {3279 } else {
...@@ -3250,10 +3284,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -3250,10 +3284,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3250 }3284 }
3251 };3285 };
3252 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {3286 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3253 .register => |reg| self.register_manager.lockReg(reg),3287 .register => |a_reg| self.register_manager.lockReg(a_reg),
3254 else => null,3288 else => null,
3255 };3289 };
3256 defer if (dst_mcv_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);3290 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
32573291
3258 // Shift by struct_field_offset.3292 // Shift by struct_field_offset.
3259 const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));3293 const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
...@@ -3342,17 +3376,17 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:...@@ -3342,17 +3376,17 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
3342 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),3376 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3343 else => null,3377 else => null,
3344 };3378 };
3345 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);3379 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
33463380
3347 const rhs = try self.resolveInst(op_rhs);3381 const rhs = try self.resolveInst(op_rhs);
3348 const rhs_lock: ?RegisterLock = switch (rhs) {3382 const rhs_lock: ?RegisterLock = switch (rhs) {
3349 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),3383 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3350 else => null,3384 else => null,
3351 };3385 };
3352 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);3386 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
33533387
3354 var flipped: bool = false;3388 var flipped: bool = false;
3355 const dst_mcv = blk: {3389 const dst_mcv: MCValue = blk: {
3356 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {3390 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
3357 break :blk lhs;3391 break :blk lhs;
3358 }3392 }
...@@ -3366,9 +3400,9 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:...@@ -3366,9 +3400,9 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
3366 .register => |reg| self.register_manager.lockReg(reg),3400 .register => |reg| self.register_manager.lockReg(reg),
3367 else => null,3401 else => null,
3368 };3402 };
3369 defer if (dst_mcv_lock) |reg| self.register_manager.unlockReg(reg);3403 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
33703404
3371 const src_mcv = blk: {3405 const src_mcv: MCValue = blk: {
3372 const mcv = if (flipped) lhs else rhs;3406 const mcv = if (flipped) lhs else rhs;
3373 if (mcv.isRegister() or mcv.isMemory()) break :blk mcv;3407 if (mcv.isRegister() or mcv.isMemory()) break :blk mcv;
3374 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) };3408 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) };
...@@ -3377,7 +3411,7 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:...@@ -3377,7 +3411,7 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
3377 .register => |reg| self.register_manager.lockReg(reg),3411 .register => |reg| self.register_manager.lockReg(reg),
3378 else => null,3412 else => null,
3379 };3413 };
3380 defer if (src_mcv_lock) |reg| self.register_manager.unlockReg(reg);3414 defer if (src_mcv_lock) |lock| self.register_manager.unlockReg(lock);
33813415
3382 const tag = self.air.instructions.items(.tag)[inst];3416 const tag = self.air.instructions.items(.tag)[inst];
3383 switch (tag) {3417 switch (tag) {
...@@ -3409,7 +3443,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC...@@ -3409,7 +3443,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
3409 .register_overflow_signed => unreachable,3443 .register_overflow_signed => unreachable,
3410 .ptr_stack_offset => {3444 .ptr_stack_offset => {
3411 const dst_reg_lock = self.register_manager.lockReg(dst_reg);3445 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3412 defer if (dst_reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);3446 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
34133447
3414 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);3448 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3415 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });3449 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
...@@ -3441,7 +3475,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC...@@ -3441,7 +3475,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
3441 => {3475 => {
3442 assert(abi_size <= 8);3476 assert(abi_size <= 8);
3443 const dst_reg_lock = self.register_manager.lockReg(dst_reg);3477 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3444 defer if (dst_reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);3478 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
34453479
3446 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);3480 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3447 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });3481 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
...@@ -3782,22 +3816,25 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions....@@ -3782,22 +3816,25 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3782 try self.register_manager.getReg(reg, null);3816 try self.register_manager.getReg(reg, null);
3783 }3817 }
37843818
3785 const rdi_lock: ?RegisterLock = if (info.return_value == .stack_offset) blk: {3819 const rdi_lock: ?RegisterLock = blk: {
3786 const ret_ty = fn_ty.fnReturnType();3820 if (info.return_value == .stack_offset) {
3787 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));3821 const ret_ty = fn_ty.fnReturnType();
3788 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));3822 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3789 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));3823 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
3790 log.debug("airCall: return value on stack at offset {}", .{stack_offset});3824 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
3825 log.debug("airCall: return value on stack at offset {}", .{stack_offset});
37913826
3792 try self.register_manager.getReg(.rdi, null);3827 try self.register_manager.getReg(.rdi, null);
3793 try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });3828 try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });
3794 const rdi_lock = self.register_manager.lockRegAssumeUnused(.rdi);3829 const rdi_lock = self.register_manager.lockRegAssumeUnused(.rdi);
37953830
3796 info.return_value.stack_offset = stack_offset;3831 info.return_value.stack_offset = stack_offset;
37973832
3798 break :blk rdi_lock;3833 break :blk rdi_lock;
3799 } else null;3834 }
3800 defer if (rdi_lock) |reg| self.register_manager.unlockReg(reg);3835 break :blk null;
3836 };
3837 defer if (rdi_lock) |lock| self.register_manager.unlockReg(lock);
38013838
3802 for (args) |arg, arg_i| {3839 for (args) |arg, arg_i| {
3803 const mc_arg = info.args[arg_i];3840 const mc_arg = info.args[arg_i];
...@@ -4107,7 +4144,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -4107,7 +4144,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
4107 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),4144 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4108 else => null,4145 else => null,
4109 };4146 };
4110 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);4147 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
41114148
4112 const dst_reg = try self.copyToTmpRegister(ty, lhs);4149 const dst_reg = try self.copyToTmpRegister(ty, lhs);
4113 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);4150 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
...@@ -4572,27 +4609,31 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {...@@ -4572,27 +4609,31 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
45724609
4573fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {4610fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4574 const un_op = self.air.instructions.items(.data)[inst].un_op;4611 const un_op = self.air.instructions.items(.data)[inst].un_op;
4575 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4576 const operand_ptr = try self.resolveInst(un_op);
45774612
4578 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {4613 if (self.liveness.isUnused(inst)) {
4579 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),4614 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4580 else => null,4615 }
4581 };
4582 defer if (operand_ptr_lock) |reg| self.register_manager.unlockReg(reg);
45834616
4584 const operand: MCValue = blk: {4617 const operand_ptr = try self.resolveInst(un_op);
4585 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {4618 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4586 // The MCValue that holds the pointer can be re-used as the value.4619 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4587 break :blk operand_ptr;4620 else => null,
4588 } else {4621 };
4589 break :blk try self.allocRegOrMem(inst, true);4622 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4590 }4623
4591 };4624 const operand: MCValue = blk: {
4592 const ptr_ty = self.air.typeOf(un_op);4625 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4593 try self.load(operand, operand_ptr, ptr_ty);4626 // The MCValue that holds the pointer can be re-used as the value.
4594 break :result try self.isNull(inst, ptr_ty.elemType(), operand);4627 break :blk operand_ptr;
4628 } else {
4629 break :blk try self.allocRegOrMem(inst, true);
4630 }
4595 };4631 };
4632 const ptr_ty = self.air.typeOf(un_op);
4633 try self.load(operand, operand_ptr, ptr_ty);
4634
4635 const result = try self.isNull(inst, ptr_ty.elemType(), operand);
4636
4596 return self.finishAir(inst, result, .{ un_op, .none, .none });4637 return self.finishAir(inst, result, .{ un_op, .none, .none });
4597}4638}
45984639
...@@ -4608,27 +4649,31 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {...@@ -4608,27 +4649,31 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
46084649
4609fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {4650fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4610 const un_op = self.air.instructions.items(.data)[inst].un_op;4651 const un_op = self.air.instructions.items(.data)[inst].un_op;
4611 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4612 const operand_ptr = try self.resolveInst(un_op);
46134652
4614 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {4653 if (self.liveness.isUnused(inst)) {
4615 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),4654 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4616 else => null,4655 }
4617 };
4618 defer if (operand_ptr_lock) |reg| self.register_manager.unlockReg(reg);
46194656
4620 const operand: MCValue = blk: {4657 const operand_ptr = try self.resolveInst(un_op);
4621 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {4658 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4622 // The MCValue that holds the pointer can be re-used as the value.4659 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4623 break :blk operand_ptr;4660 else => null,
4624 } else {
4625 break :blk try self.allocRegOrMem(inst, true);
4626 }
4627 };
4628 const ptr_ty = self.air.typeOf(un_op);
4629 try self.load(operand, operand_ptr, ptr_ty);
4630 break :result try self.isNonNull(inst, ptr_ty.elemType(), operand);
4631 };4661 };
4662 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4663
4664 const operand: MCValue = blk: {
4665 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4666 // The MCValue that holds the pointer can be re-used as the value.
4667 break :blk operand_ptr;
4668 } else {
4669 break :blk try self.allocRegOrMem(inst, true);
4670 }
4671 };
4672 const ptr_ty = self.air.typeOf(un_op);
4673 try self.load(operand, operand_ptr, ptr_ty);
4674
4675 const result = try self.isNonNull(inst, ptr_ty.elemType(), operand);
4676
4632 return self.finishAir(inst, result, .{ un_op, .none, .none });4677 return self.finishAir(inst, result, .{ un_op, .none, .none });
4633}4678}
46344679
...@@ -4644,27 +4689,31 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4644,27 +4689,31 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
46444689
4645fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {4690fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4646 const un_op = self.air.instructions.items(.data)[inst].un_op;4691 const un_op = self.air.instructions.items(.data)[inst].un_op;
4647 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4648 const operand_ptr = try self.resolveInst(un_op);
46494692
4650 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {4693 if (self.liveness.isUnused(inst)) {
4651 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),4694 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4652 else => null,4695 }
4653 };
4654 defer if (operand_ptr_lock) |reg| self.register_manager.unlockReg(reg);
46554696
4656 const operand: MCValue = blk: {4697 const operand_ptr = try self.resolveInst(un_op);
4657 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {4698 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4658 // The MCValue that holds the pointer can be re-used as the value.4699 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4659 break :blk operand_ptr;4700 else => null,
4660 } else {4701 };
4661 break :blk try self.allocRegOrMem(inst, true);4702 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4662 }4703
4663 };4704 const operand: MCValue = blk: {
4664 const ptr_ty = self.air.typeOf(un_op);4705 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4665 try self.load(operand, operand_ptr, ptr_ty);4706 // The MCValue that holds the pointer can be re-used as the value.
4666 break :result try self.isErr(inst, ptr_ty.elemType(), operand);4707 break :blk operand_ptr;
4708 } else {
4709 break :blk try self.allocRegOrMem(inst, true);
4710 }
4667 };4711 };
4712 const ptr_ty = self.air.typeOf(un_op);
4713 try self.load(operand, operand_ptr, ptr_ty);
4714
4715 const result = try self.isErr(inst, ptr_ty.elemType(), operand);
4716
4668 return self.finishAir(inst, result, .{ un_op, .none, .none });4717 return self.finishAir(inst, result, .{ un_op, .none, .none });
4669}4718}
46704719
...@@ -4680,27 +4729,31 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4680,27 +4729,31 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
46804729
4681fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {4730fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4682 const un_op = self.air.instructions.items(.data)[inst].un_op;4731 const un_op = self.air.instructions.items(.data)[inst].un_op;
4683 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4684 const operand_ptr = try self.resolveInst(un_op);
46854732
4686 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {4733 if (self.liveness.isUnused(inst)) {
4687 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),4734 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4688 else => null,4735 }
4689 };
4690 defer if (operand_ptr_lock) |reg| self.register_manager.unlockReg(reg);
46914736
4692 const operand: MCValue = blk: {4737 const operand_ptr = try self.resolveInst(un_op);
4693 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {4738 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4694 // The MCValue that holds the pointer can be re-used as the value.4739 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4695 break :blk operand_ptr;4740 else => null,
4696 } else {4741 };
4697 break :blk try self.allocRegOrMem(inst, true);4742 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4698 }4743
4699 };4744 const operand: MCValue = blk: {
4700 const ptr_ty = self.air.typeOf(un_op);4745 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4701 try self.load(operand, operand_ptr, ptr_ty);4746 // The MCValue that holds the pointer can be re-used as the value.
4702 break :result try self.isNonErr(inst, ptr_ty.elemType(), operand);4747 break :blk operand_ptr;
4748 } else {
4749 break :blk try self.allocRegOrMem(inst, true);
4750 }
4703 };4751 };
4752 const ptr_ty = self.air.typeOf(un_op);
4753 try self.load(operand, operand_ptr, ptr_ty);
4754
4755 const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);
4756
4704 return self.finishAir(inst, result, .{ un_op, .none, .none });4757 return self.finishAir(inst, result, .{ un_op, .none, .none });
4705}4758}
47064759
...@@ -4757,7 +4810,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u...@@ -4757,7 +4810,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4757 try self.spillCompareFlagsIfOccupied();4810 try self.spillCompareFlagsIfOccupied();
47584811
4759 const cond_reg_lock = self.register_manager.lockReg(cond_reg);4812 const cond_reg_lock = self.register_manager.lockReg(cond_reg);
4760 defer if (cond_reg_lock) |reg| self.register_manager.unlockReg(reg);4813 defer if (cond_reg_lock) |lock| self.register_manager.unlockReg(lock);
47614814
4762 switch (case) {4815 switch (case) {
4763 .none => unreachable,4816 .none => unreachable,
...@@ -5305,7 +5358,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -5305,7 +5358,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5305 .register_overflow_signed,5358 .register_overflow_signed,
5306 => |reg| {5359 => |reg| {
5307 const reg_lock = self.register_manager.lockReg(reg);5360 const reg_lock = self.register_manager.lockReg(reg);
5308 defer if (reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);5361 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
53095362
5310 const wrapped_ty = ty.structFieldType(0);5363 const wrapped_ty = ty.structFieldType(0);
5311 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg }, .{});5364 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg }, .{});
...@@ -5407,7 +5460,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -5407,7 +5460,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5407 const base_reg = opts.dest_stack_base orelse .rbp;5460 const base_reg = opts.dest_stack_base orelse .rbp;
5408 if (!math.isPowerOfTwo(abi_size)) {5461 if (!math.isPowerOfTwo(abi_size)) {
5409 const reg_lock = self.register_manager.lockReg(reg);5462 const reg_lock = self.register_manager.lockReg(reg);
5410 defer if (reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);5463 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
54115464
5412 const tmp_reg = try self.copyToTmpRegister(ty, mcv);5465 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
54135466
...@@ -5501,8 +5554,8 @@ fn genInlineMemcpy(...@@ -5501,8 +5554,8 @@ fn genInlineMemcpy(
55015554
5502 var reg_locks: [2]RegisterLock = undefined;5555 var reg_locks: [2]RegisterLock = undefined;
5503 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rcx }, &reg_locks);5556 self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rcx }, &reg_locks);
5504 defer for (reg_locks) |reg| {5557 defer for (reg_locks) |lock| {
5505 self.register_manager.unlockReg(reg);5558 self.register_manager.unlockReg(lock);
5506 };5559 };
55075560
5508 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|5561 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
...@@ -5515,7 +5568,7 @@ fn genInlineMemcpy(...@@ -5515,7 +5568,7 @@ fn genInlineMemcpy(
5515 self.register_manager.lockReg(reg)5568 self.register_manager.lockReg(reg)
5516 else5569 else
5517 null;5570 null;
5518 defer if (dsbase_lock) |reg| self.register_manager.unlockReg(reg);5571 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
55195572
5520 const dst_addr_reg = try self.register_manager.allocReg(null);5573 const dst_addr_reg = try self.register_manager.allocReg(null);
5521 switch (dst_ptr) {5574 switch (dst_ptr) {
...@@ -6174,21 +6227,21 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {...@@ -6174,21 +6227,21 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
6174 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),6227 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6175 else => null,6228 else => null,
6176 };6229 };
6177 defer if (dst_ptr_lock) |reg| self.register_manager.unlockReg(reg);6230 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
61786231
6179 const src_val = try self.resolveInst(extra.lhs);6232 const src_val = try self.resolveInst(extra.lhs);
6180 const src_val_lock: ?RegisterLock = switch (src_val) {6233 const src_val_lock: ?RegisterLock = switch (src_val) {
6181 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),6234 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6182 else => null,6235 else => null,
6183 };6236 };
6184 defer if (src_val_lock) |reg| self.register_manager.unlockReg(reg);6237 defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock);
61856238
6186 const len = try self.resolveInst(extra.rhs);6239 const len = try self.resolveInst(extra.rhs);
6187 const len_lock: ?RegisterLock = switch (len) {6240 const len_lock: ?RegisterLock = switch (len) {
6188 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),6241 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6189 else => null,6242 else => null,
6190 };6243 };
6191 defer if (len_lock) |reg| self.register_manager.unlockReg(reg);6244 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
61926245
6193 try self.genInlineMemset(dst_ptr, src_val, len, .{});6246 try self.genInlineMemset(dst_ptr, src_val, len, .{});
61946247
...@@ -6204,7 +6257,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -6204,7 +6257,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6204 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),6257 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6205 else => null,6258 else => null,
6206 };6259 };
6207 defer if (dst_ptr_lock) |reg| self.register_manager.unlockReg(reg);6260 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
62086261
6209 const src_ty = self.air.typeOf(extra.lhs);6262 const src_ty = self.air.typeOf(extra.lhs);
6210 const src_ptr = try self.resolveInst(extra.lhs);6263 const src_ptr = try self.resolveInst(extra.lhs);
...@@ -6212,14 +6265,14 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -6212,14 +6265,14 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6212 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),6265 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6213 else => null,6266 else => null,
6214 };6267 };
6215 defer if (src_ptr_lock) |reg| self.register_manager.unlockReg(reg);6268 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);
62166269
6217 const len = try self.resolveInst(extra.rhs);6270 const len = try self.resolveInst(extra.rhs);
6218 const len_lock: ?RegisterLock = switch (len) {6271 const len_lock: ?RegisterLock = switch (len) {
6219 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),6272 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6220 else => null,6273 else => null,
6221 };6274 };
6222 defer if (len_lock) |reg| self.register_manager.unlockReg(reg);6275 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
62236276
6224 // TODO Is this the only condition for pointer dereference for memcpy?6277 // TODO Is this the only condition for pointer dereference for memcpy?
6225 const src: MCValue = blk: {6278 const src: MCValue = blk: {
...@@ -6245,7 +6298,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -6245,7 +6298,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6245 .register => |reg| self.register_manager.lockReg(reg),6298 .register => |reg| self.register_manager.lockReg(reg),
6246 else => null,6299 else => null,
6247 };6300 };
6248 defer if (src_lock) |reg| self.register_manager.unlockReg(reg);6301 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
62496302
6250 try self.genInlineMemcpy(dst_ptr, src, len, .{});6303 try self.genInlineMemcpy(dst_ptr, src, len, .{});
62516304
src/register_manager.zig+1-1
...@@ -153,7 +153,7 @@ pub fn RegisterManager(...@@ -153,7 +153,7 @@ pub fn RegisterManager(
153 regs: [count]Register,153 regs: [count]Register,
154 buf: *[count]RegisterLock,154 buf: *[count]RegisterLock,
155 ) void {155 ) void {
156 for (&regs) |reg, i| {156 for (regs) |reg, i| {
157 buf[i] = self.lockRegAssumeUnused(reg);157 buf[i] = self.lockRegAssumeUnused(reg);
158 }158 }
159 }159 }