authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-08 11:20:14+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-05-08 11:20:14+02:00
log9416b4d993e5e386c9208e9a52e4e881eee8cf72
tree118a802c038879f57888c8d3643c233f031c449c
parentaa05cd48097b42d0c52ff5e0f3dd1ece68d1cdcd
parent6bf67eada47b7c6f4819a759268503658c97e9ec
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11608 from ziglang/stage2-regalloc

7 files changed, 1092 insertions(+), 686 deletions(-)

src/arch/aarch64/CodeGen.zig+151-68
......@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
2525const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2627
2728const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
2829const FnResult = @import("../../codegen.zig").FnResult;
......@@ -727,7 +728,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
727728 // zig fmt: on
728729 }
729730
730 assert(!self.register_manager.frozenRegsExist());
731 assert(!self.register_manager.lockedRegsExist());
731732
732733 if (std.debug.runtime_safety) {
733734 if (self.air_bookkeeping < old_air_bookkeeping + 1) {
......@@ -910,16 +911,16 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
910911fn spillCompareFlagsIfOccupied(self: *Self) !void {
911912 if (self.compare_flags_inst) |inst_to_save| {
912913 const mcv = self.getResolvedInstValue(inst_to_save);
913 switch (mcv) {
914 const new_mcv = switch (mcv) {
914915 .compare_flags_signed,
915916 .compare_flags_unsigned,
917 => try self.allocRegOrMem(inst_to_save, true),
916918 .register_c_flag,
917919 .register_v_flag,
918 => {},
920 => try self.allocRegOrMem(inst_to_save, false),
919921 else => unreachable, // mcv doesn't occupy the compare flags
920 }
922 };
921923
922 const new_mcv = try self.allocRegOrMem(inst_to_save, true);
923924 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
924925 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
925926
......@@ -927,6 +928,15 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
927928 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
928929
929930 self.compare_flags_inst = null;
931
932 // TODO consolidate with register manager and spillInstruction
933 // this call should really belong in the register manager!
934 switch (mcv) {
935 .register_c_flag,
936 .register_v_flag,
937 => |reg| self.register_manager.freeReg(reg),
938 else => {},
939 }
930940 }
931941}
932942
......@@ -1048,8 +1058,8 @@ fn trunc(
10481058 }
10491059 },
10501060 };
1051 self.register_manager.freezeRegs(&.{operand_reg});
1052 defer self.register_manager.unfreezeRegs(&.{operand_reg});
1061 const lock = self.register_manager.lockReg(operand_reg);
1062 defer if (lock) |reg| self.register_manager.unlockReg(reg);
10531063
10541064 const dest_reg = if (maybe_inst) |inst| blk: {
10551065 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
......@@ -1135,8 +1145,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11351145 .register => |r| r,
11361146 else => try self.copyToTmpRegister(operand_ty, operand),
11371147 };
1138 self.register_manager.freezeRegs(&.{op_reg});
1139 defer self.register_manager.unfreezeRegs(&.{op_reg});
1148 const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1149 defer self.register_manager.unlockReg(reg_lock);
11401150
11411151 const dest_reg = blk: {
11421152 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1168,8 +1178,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11681178 .register => |r| r,
11691179 else => try self.copyToTmpRegister(operand_ty, operand),
11701180 };
1171 self.register_manager.freezeRegs(&.{op_reg});
1172 defer self.register_manager.unfreezeRegs(&.{op_reg});
1181 const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1182 defer self.register_manager.unlockReg(reg_lock);
11731183
11741184 const dest_reg = blk: {
11751185 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1257,8 +1267,17 @@ fn binOpRegister(
12571267 const lhs_is_register = lhs == .register;
12581268 const rhs_is_register = rhs == .register;
12591269
1260 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1261 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
1270 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1271 self.register_manager.lockReg(lhs.register)
1272 else
1273 null;
1274 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
1275
1276 const rhs_lock: ?RegisterLock = if (rhs_is_register)
1277 self.register_manager.lockReg(rhs.register)
1278 else
1279 null;
1280 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
12621281
12631282 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
12641283
......@@ -1270,13 +1289,13 @@ fn binOpRegister(
12701289
12711290 const raw_reg = try self.register_manager.allocReg(track_inst);
12721291 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1273 self.register_manager.freezeRegs(&.{reg});
12741292
12751293 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
12761294
12771295 break :blk reg;
12781296 };
1279 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1297 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
1298 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
12801299
12811300 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
12821301 const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
......@@ -1286,13 +1305,13 @@ fn binOpRegister(
12861305
12871306 const raw_reg = try self.register_manager.allocReg(track_inst);
12881307 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1289 self.register_manager.freezeRegs(&.{reg});
12901308
12911309 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
12921310
12931311 break :blk reg;
12941312 };
1295 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1313 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
1314 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
12961315
12971316 const dest_reg = switch (mir_tag) {
12981317 .cmp_shifted_register => undefined, // cmp has no destination register
......@@ -1394,7 +1413,11 @@ fn binOpImmediate(
13941413) !MCValue {
13951414 const lhs_is_register = lhs == .register;
13961415
1397 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1416 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1417 self.register_manager.lockReg(lhs.register)
1418 else
1419 null;
1420 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
13981421
13991422 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
14001423
......@@ -1408,13 +1431,13 @@ fn binOpImmediate(
14081431
14091432 const raw_reg = try self.register_manager.allocReg(track_inst);
14101433 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1411 self.register_manager.freezeRegs(&.{reg});
14121434
14131435 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
14141436
14151437 break :blk reg;
14161438 };
1417 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1439 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
1440 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
14181441
14191442 const dest_reg = switch (mir_tag) {
14201443 .cmp_immediate => undefined, // cmp has no destination register
......@@ -1758,7 +1781,10 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void {
17581781 const lhs_ty = self.air.typeOf(bin_op.lhs);
17591782 const rhs_ty = self.air.typeOf(bin_op.rhs);
17601783
1761 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
1784 const result: MCValue = if (self.liveness.isUnused(inst))
1785 .dead
1786 else
1787 try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
17621788 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
17631789}
17641790
......@@ -1815,13 +1841,13 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
18151841 };
18161842 const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
18171843 const dest_reg = dest.register;
1818 self.register_manager.freezeRegs(&.{dest_reg});
1819 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1844 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1845 defer self.register_manager.unlockReg(dest_reg_lock);
18201846
18211847 const raw_truncated_reg = try self.register_manager.allocReg(null);
18221848 const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*));
1823 self.register_manager.freezeRegs(&.{truncated_reg});
1824 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1849 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1850 defer self.register_manager.unlockReg(truncated_reg_lock);
18251851
18261852 // sbfx/ubfx truncated, dest, #0, #bits
18271853 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
......@@ -1922,12 +1948,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
19221948
19231949 const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
19241950 const dest_reg = dest.register;
1925 self.register_manager.freezeRegs(&.{dest_reg});
1926 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1951 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1952 defer self.register_manager.unlockReg(dest_reg_lock);
19271953
19281954 const truncated_reg = try self.register_manager.allocReg(null);
1929 self.register_manager.freezeRegs(&.{truncated_reg});
1930 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1955 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1956 defer self.register_manager.unlockReg(truncated_reg_lock);
19311957
19321958 try self.truncRegister(
19331959 dest_reg.to32(),
......@@ -1977,36 +2003,44 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
19772003 const lhs_is_register = lhs == .register;
19782004 const rhs_is_register = rhs == .register;
19792005
1980 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1981 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
2006 const lhs_lock: ?RegisterLock = if (lhs_is_register)
2007 self.register_manager.lockRegAssumeUnused(lhs.register)
2008 else
2009 null;
2010 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
2011
2012 const rhs_lock: ?RegisterLock = if (rhs_is_register)
2013 self.register_manager.lockRegAssumeUnused(rhs.register)
2014 else
2015 null;
2016 defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
19822017
19832018 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
19842019 const raw_reg = try self.register_manager.allocReg(null);
19852020 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1986 self.register_manager.freezeRegs(&.{reg});
19872021 break :blk reg;
19882022 };
1989 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
2023 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
2024 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
19902025
19912026 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
19922027 const raw_reg = try self.register_manager.allocReg(null);
19932028 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1994 self.register_manager.freezeRegs(&.{reg});
19952029 break :blk reg;
19962030 };
1997 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
2031 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
2032 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
19982033
19992034 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
20002035 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
20012036
2002 // TODO reuse operands
20032037 const dest_reg = blk: {
20042038 const raw_reg = try self.register_manager.allocReg(null);
20052039 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
2006 self.register_manager.freezeRegs(&.{reg});
20072040 break :blk reg;
20082041 };
2009 defer self.register_manager.unfreezeRegs(&.{dest_reg});
2042 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
2043 defer self.register_manager.unlockReg(dest_reg_lock);
20102044
20112045 switch (int_info.signedness) {
20122046 .signed => {
......@@ -2021,8 +2055,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20212055 });
20222056
20232057 const dest_high_reg = try self.register_manager.allocReg(null);
2024 self.register_manager.freezeRegs(&.{dest_high_reg});
2025 defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
2058 const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg);
2059 defer self.register_manager.unlockReg(dest_high_reg_lock);
20262060
20272061 // smulh dest_high, lhs, rhs
20282062 _ = try self.addInst(.{
......@@ -2071,8 +2105,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20712105 },
20722106 .unsigned => {
20732107 const dest_high_reg = try self.register_manager.allocReg(null);
2074 self.register_manager.freezeRegs(&.{dest_high_reg});
2075 defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
2108 const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg);
2109 defer self.register_manager.unlockReg(dest_high_reg_lock);
20762110
20772111 // umulh dest_high, lhs, rhs
20782112 _ = try self.addInst(.{
......@@ -2127,8 +2161,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21272161 }
21282162
21292163 const truncated_reg = try self.register_manager.allocReg(null);
2130 self.register_manager.freezeRegs(&.{truncated_reg});
2131 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
2164 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
2165 defer self.register_manager.unlockReg(truncated_reg_lock);
21322166
21332167 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
21342168
......@@ -2168,14 +2202,20 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21682202 if (int_info.bits <= 64) {
21692203 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
21702204
2171 if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});
2172 defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});
2205 const lhs_lock: ?RegisterLock = if (lhs == .register)
2206 self.register_manager.lockRegAssumeUnused(lhs.register)
2207 else
2208 null;
2209 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
21732210
21742211 try self.spillCompareFlagsIfOccupied();
21752212 self.compare_flags_inst = null;
21762213
21772214 // lsl dest, lhs, rhs
21782215 const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
2216 const dest_reg = dest.register;
2217 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
2218 defer self.register_manager.unlockReg(dest_reg_lock);
21792219
21802220 // asr/lsr reconstructed, dest, rhs
21812221 const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);
......@@ -2184,7 +2224,9 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21842224 _ = try self.binOp(.cmp_eq, null, lhs, reconstructed, lhs_ty, lhs_ty);
21852225
21862226 try self.genSetStack(lhs_ty, stack_offset, dest);
2187 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags_unsigned = .neq });
2227 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
2228 .compare_flags_unsigned = .neq,
2229 });
21882230
21892231 break :result MCValue{ .stack_offset = stack_offset };
21902232 } else {
......@@ -2411,14 +2453,18 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
24112453 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
24122454 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
24132455
2414 if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register});
2415 defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register});
2456 const index_lock: ?RegisterLock = if (index_is_register)
2457 self.register_manager.lockRegAssumeUnused(index_mcv.register)
2458 else
2459 null;
2460 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
24162461
24172462 const base_mcv: MCValue = switch (slice_mcv) {
24182463 .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
24192464 else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
24202465 };
2421 self.register_manager.freezeRegs(&.{base_mcv.register});
2466 const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register);
2467 defer self.register_manager.unlockReg(base_lock);
24222468
24232469 switch (elem_size) {
24242470 else => {
......@@ -2559,8 +2605,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
25592605 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
25602606 .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
25612607 .register => |addr_reg| {
2562 self.register_manager.freezeRegs(&.{addr_reg});
2563 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2608 const addr_reg_lock = self.register_manager.lockReg(addr_reg);
2609 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
25642610
25652611 switch (dst_mcv) {
25662612 .dead => unreachable,
......@@ -2573,16 +2619,18 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
25732619 if (elem_size <= 8) {
25742620 const raw_tmp_reg = try self.register_manager.allocReg(null);
25752621 const tmp_reg = registerAlias(raw_tmp_reg, elem_size);
2576 self.register_manager.freezeRegs(&.{tmp_reg});
2577 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2622 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2623 defer self.register_manager.unlockReg(tmp_reg_lock);
25782624
25792625 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
25802626 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
25812627 } else {
25822628 // TODO optimize the register allocation
25832629 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2584 self.register_manager.freezeRegs(&regs);
2585 defer self.register_manager.unfreezeRegs(&regs);
2630 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2631 defer for (regs_locks) |reg| {
2632 self.register_manager.unlockReg(reg);
2633 };
25862634
25872635 const src_reg = addr_reg;
25882636 const dst_reg = regs[0];
......@@ -2784,8 +2832,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27842832 try self.genSetStack(value_ty, off, value);
27852833 },
27862834 .register => |addr_reg| {
2787 self.register_manager.freezeRegs(&.{addr_reg});
2788 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2835 const addr_reg_lock = self.register_manager.lockReg(addr_reg);
2836 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
27892837
27902838 switch (value) {
27912839 .register => |value_reg| {
......@@ -2795,8 +2843,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27952843 if (abi_size <= 8) {
27962844 const raw_tmp_reg = try self.register_manager.allocReg(null);
27972845 const tmp_reg = registerAlias(raw_tmp_reg, abi_size);
2798 self.register_manager.freezeRegs(&.{tmp_reg});
2799 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2846 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2847 defer self.register_manager.unlockReg(tmp_reg_lock);
28002848
28012849 try self.genSetReg(value_ty, tmp_reg, value);
28022850 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
......@@ -2856,12 +2904,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
28562904 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
28572905 .immediate = struct_field_offset,
28582906 });
2859 self.register_manager.freezeRegs(&.{offset_reg});
2860 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2907 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2908 defer self.register_manager.unlockReg(offset_reg_lock);
28612909
28622910 const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
2863 self.register_manager.freezeRegs(&.{addr_reg});
2864 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2911 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
2912 defer self.register_manager.unlockReg(addr_reg_lock);
28652913
28662914 const dest = try self.binOp(
28672915 .add,
......@@ -3369,6 +3417,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
33693417 const parent_compare_flags_inst = self.compare_flags_inst;
33703418
33713419 try self.branch_stack.append(.{});
3420 errdefer {
3421 _ = self.branch_stack.pop();
3422 }
33723423
33733424 try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
33743425 for (liveness_condbr.then_deaths) |operand| {
......@@ -3955,8 +4006,38 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
39554006 },
39564007 .register_c_flag,
39574008 .register_v_flag,
3958 => {
3959 return self.fail("TODO implement genSetStack {}", .{mcv});
4009 => |reg| {
4010 const reg_lock = self.register_manager.lockReg(reg);
4011 defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
4012
4013 const wrapped_ty = ty.structFieldType(0);
4014 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg });
4015
4016 const overflow_bit_ty = ty.structFieldType(1);
4017 const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
4018 const raw_cond_reg = try self.register_manager.allocReg(null);
4019 const cond_reg = registerAlias(
4020 raw_cond_reg,
4021 @intCast(u32, overflow_bit_ty.abiSize(self.target.*)),
4022 );
4023
4024 // C flag: cset reg, cs
4025 // V flag: cset reg, vs
4026 _ = try self.addInst(.{
4027 .tag = .cset,
4028 .data = .{ .r_cond = .{
4029 .rd = cond_reg,
4030 .cond = switch (mcv) {
4031 .register_c_flag => .cs,
4032 .register_v_flag => .vs,
4033 else => unreachable,
4034 },
4035 } },
4036 });
4037
4038 try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
4039 .register = cond_reg,
4040 });
39604041 },
39614042 .got_load,
39624043 .direct_load,
......@@ -3983,8 +4064,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
39834064
39844065 // TODO call extern memcpy
39854066 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null });
3986 self.register_manager.freezeRegs(&regs);
3987 defer self.register_manager.unfreezeRegs(&regs);
4067 const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
4068 defer for (regs_locks) |reg| {
4069 self.register_manager.unlockReg(reg);
4070 };
39884071
39894072 const src_reg = regs[0];
39904073 const dst_reg = regs[1];
src/arch/arm/CodeGen.zig+146-82
......@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
2525const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26const RegisterLock = RegisterManager.RegisterLock;
2627
2728const FnResult = @import("../../codegen.zig").FnResult;
2829const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
......@@ -734,7 +735,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
734735 // zig fmt: on
735736 }
736737
737 assert(!self.register_manager.frozenRegsExist());
738 assert(!self.register_manager.lockedRegsExist());
738739
739740 if (std.debug.runtime_safety) {
740741 if (self.air_bookkeeping < old_air_bookkeeping + 1) {
......@@ -897,16 +898,16 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
897898fn spillCompareFlagsIfOccupied(self: *Self) !void {
898899 if (self.compare_flags_inst) |inst_to_save| {
899900 const mcv = self.getResolvedInstValue(inst_to_save);
900 switch (mcv) {
901 const new_mcv = switch (mcv) {
901902 .compare_flags_signed,
902903 .compare_flags_unsigned,
904 => try self.allocRegOrMem(inst_to_save, true),
903905 .register_c_flag,
904906 .register_v_flag,
905 => {},
907 => try self.allocRegOrMem(inst_to_save, false),
906908 else => unreachable, // mcv doesn't occupy the compare flags
907 }
909 };
908910
909 const new_mcv = try self.allocRegOrMem(inst_to_save, true);
910911 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
911912 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
912913
......@@ -914,6 +915,15 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
914915 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
915916
916917 self.compare_flags_inst = null;
918
919 // TODO consolidate with register manager and spillInstruction
920 // this call should really belong in the register manager!
921 switch (mcv) {
922 .register_c_flag,
923 .register_v_flag,
924 => |reg| self.register_manager.freeReg(reg),
925 else => {},
926 }
917927 }
918928}
919929
......@@ -1038,8 +1048,8 @@ fn trunc(
10381048 }
10391049 },
10401050 };
1041 self.register_manager.freezeRegs(&.{operand_reg});
1042 defer self.register_manager.unfreezeRegs(&.{operand_reg});
1051 const operand_reg_lock = self.register_manager.lockReg(operand_reg);
1052 defer if (operand_reg_lock) |reg| self.register_manager.unlockReg(reg);
10431053
10441054 const dest_reg = if (maybe_inst) |inst| blk: {
10451055 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
......@@ -1127,8 +1137,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11271137 .register => |r| r,
11281138 else => try self.copyToTmpRegister(operand_ty, operand),
11291139 };
1130 self.register_manager.freezeRegs(&.{op_reg});
1131 defer self.register_manager.unfreezeRegs(&.{op_reg});
1140 const op_reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1141 defer self.register_manager.unlockReg(op_reg_lock);
11321142
11331143 const dest_reg = blk: {
11341144 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1157,8 +1167,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11571167 .register => |r| r,
11581168 else => try self.copyToTmpRegister(operand_ty, operand),
11591169 };
1160 self.register_manager.freezeRegs(&.{op_reg});
1161 defer self.register_manager.unfreezeRegs(&.{op_reg});
1170 const op_reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1171 defer self.register_manager.unlockReg(op_reg_lock);
11621172
11631173 const dest_reg = blk: {
11641174 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1218,15 +1228,15 @@ fn minMax(
12181228 .register => |r| r,
12191229 else => try self.copyToTmpRegister(lhs_ty, lhs),
12201230 };
1221 self.register_manager.freezeRegs(&.{lhs_reg});
1222 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1231 const lhs_reg_lock = self.register_manager.lockReg(lhs_reg);
1232 defer if (lhs_reg_lock) |reg| self.register_manager.unlockReg(reg);
12231233
12241234 const rhs_reg = switch (rhs) {
12251235 .register => |r| r,
12261236 else => try self.copyToTmpRegister(rhs_ty, rhs),
12271237 };
1228 self.register_manager.freezeRegs(&.{rhs_reg});
1229 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1238 const rhs_reg_lock = self.register_manager.lockReg(rhs_reg);
1239 defer if (rhs_reg_lock) |reg| self.register_manager.unlockReg(reg);
12301240
12311241 const dest_reg = if (maybe_inst) |inst| blk: {
12321242 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
......@@ -1392,12 +1402,12 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
13921402 };
13931403 const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
13941404 const dest_reg = dest.register;
1395 self.register_manager.freezeRegs(&.{dest_reg});
1396 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1405 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1406 defer self.register_manager.unlockReg(dest_reg_lock);
13971407
13981408 const truncated_reg = try self.register_manager.allocReg(null);
1399 self.register_manager.freezeRegs(&.{truncated_reg});
1400 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1409 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1410 defer self.register_manager.unlockReg(truncated_reg_lock);
14011411
14021412 // sbfx/ubfx truncated, dest, #0, #bits
14031413 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
......@@ -1493,12 +1503,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14931503
14941504 const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
14951505 const dest_reg = dest.register;
1496 self.register_manager.freezeRegs(&.{dest_reg});
1497 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1506 const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1507 defer self.register_manager.unlockReg(dest_reg_lock);
14981508
14991509 const truncated_reg = try self.register_manager.allocReg(null);
1500 self.register_manager.freezeRegs(&.{truncated_reg});
1501 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1510 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1511 defer self.register_manager.unlockReg(truncated_reg_lock);
15021512
15031513 // sbfx/ubfx truncated, dest, #0, #bits
15041514 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
......@@ -1526,28 +1536,31 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15261536 const lhs_is_register = lhs == .register;
15271537 const rhs_is_register = rhs == .register;
15281538
1529 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1530 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
1531
1532 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
1533 const reg = try self.register_manager.allocReg(null);
1534 self.register_manager.freezeRegs(&.{reg});
1535
1536 break :blk reg;
1537 };
1538 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1539 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1540 self.register_manager.lockReg(lhs.register)
1541 else
1542 null;
1543 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
15391544
1540 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
1541 const reg = try self.register_manager.allocReg(null);
1542 self.register_manager.freezeRegs(&.{reg});
1545 const lhs_reg = if (lhs_is_register)
1546 lhs.register
1547 else
1548 try self.register_manager.allocReg(null);
1549 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
1550 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
15431551
1544 break :blk reg;
1545 };
1546 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1552 const rhs_reg = if (rhs_is_register)
1553 rhs.register
1554 else
1555 try self.register_manager.allocReg(null);
1556 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
1557 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
15471558
15481559 const dest_regs = try self.register_manager.allocRegs(2, .{ null, null });
1549 self.register_manager.freezeRegs(&dest_regs);
1550 defer self.register_manager.unfreezeRegs(&dest_regs);
1560 const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs);
1561 defer for (dest_regs_locks) |reg| {
1562 self.register_manager.unlockReg(reg);
1563 };
15511564 const rdlo = dest_regs[0];
15521565 const rdhi = dest_regs[1];
15531566
......@@ -1555,8 +1568,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15551568 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
15561569
15571570 const truncated_reg = try self.register_manager.allocReg(null);
1558 self.register_manager.freezeRegs(&.{truncated_reg});
1559 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1571 const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1572 defer self.register_manager.unlockReg(truncated_reg_lock);
15601573
15611574 _ = try self.addInst(.{
15621575 .tag = base_tag,
......@@ -1648,14 +1661,20 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
16481661 if (int_info.bits <= 32) {
16491662 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
16501663
1651 if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});
1652 defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});
1664 const lhs_lock: ?RegisterLock = if (lhs == .register)
1665 self.register_manager.lockRegAssumeUnused(lhs.register)
1666 else
1667 null;
1668 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
16531669
16541670 try self.spillCompareFlagsIfOccupied();
16551671 self.compare_flags_inst = null;
16561672
16571673 // lsl dest, lhs, rhs
16581674 const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
1675 const dest_reg = dest.register;
1676 const dest_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1677 defer self.register_manager.unlockReg(dest_lock);
16591678
16601679 // asr/lsr reconstructed, dest, rhs
16611680 const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);
......@@ -1939,8 +1958,11 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
19391958 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
19401959 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
19411960
1942 if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register});
1943 defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register});
1961 const index_lock: ?RegisterLock = if (index_is_register)
1962 self.register_manager.lockRegAssumeUnused(index_mcv.register)
1963 else
1964 null;
1965 defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
19441966
19451967 const base_mcv = slicePtr(slice_mcv);
19461968
......@@ -1950,20 +1972,20 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
19501972 .register => |r| r,
19511973 else => try self.copyToTmpRegister(slice_ptr_field_type, base_mcv),
19521974 };
1953 self.register_manager.freezeRegs(&.{base_reg});
1954 defer self.register_manager.unfreezeRegs(&.{base_reg});
1975 const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg);
1976 defer self.register_manager.unlockReg(base_reg_lock);
19551977
19561978 const dst_reg = try self.register_manager.allocReg(inst);
19571979 const dst_mcv = MCValue{ .register = dst_reg };
1958 self.register_manager.freezeRegs(&.{dst_reg});
1959 defer self.register_manager.unfreezeRegs(&.{dst_reg});
1980 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1981 defer self.register_manager.unlockReg(dst_reg_lock);
19601982
19611983 const index_reg: Register = switch (index_mcv) {
19621984 .register => |reg| reg,
19631985 else => try self.copyToTmpRegister(Type.usize, index_mcv),
19641986 };
1965 self.register_manager.freezeRegs(&.{index_reg});
1966 defer self.register_manager.unfreezeRegs(&.{index_reg});
1987 const index_reg_lock = self.register_manager.lockReg(index_reg);
1988 defer if (index_reg_lock) |lock| self.register_manager.unlockReg(lock);
19671989
19681990 const tag: Mir.Inst.Tag = switch (elem_size) {
19691991 1 => .ldrb,
......@@ -2149,8 +2171,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
21492171 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
21502172 .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
21512173 .register => |reg| {
2152 self.register_manager.freezeRegs(&.{reg});
2153 defer self.register_manager.unfreezeRegs(&.{reg});
2174 const reg_lock = self.register_manager.lockReg(reg);
2175 defer if (reg_lock) |reg_locked| self.register_manager.unlockReg(reg_locked);
21542176
21552177 switch (dst_mcv) {
21562178 .dead => unreachable,
......@@ -2162,16 +2184,18 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
21622184 .stack_offset => |off| {
21632185 if (elem_size <= 4) {
21642186 const tmp_reg = try self.register_manager.allocReg(null);
2165 self.register_manager.freezeRegs(&.{tmp_reg});
2166 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2187 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2188 defer self.register_manager.unlockReg(tmp_reg_lock);
21672189
21682190 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
21692191 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
21702192 } else {
21712193 // TODO optimize the register allocation
21722194 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2173 self.register_manager.freezeRegs(&regs);
2174 defer self.register_manager.unfreezeRegs(&regs);
2195 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2196 defer for (regs_locks) |reg_locked| {
2197 self.register_manager.unlockReg(reg_locked);
2198 };
21752199
21762200 const src_reg = reg;
21772201 const dst_reg = regs[0];
......@@ -2197,8 +2221,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
21972221 .stack_argument_offset,
21982222 => {
21992223 const reg = try self.register_manager.allocReg(null);
2200 self.register_manager.freezeRegs(&.{reg});
2201 defer self.register_manager.unfreezeRegs(&.{reg});
2224 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
2225 defer self.register_manager.unlockReg(reg_lock);
22022226
22032227 try self.genSetReg(ptr_ty, reg, ptr);
22042228 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
......@@ -2252,8 +2276,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
22522276 try self.genSetStack(value_ty, off, value);
22532277 },
22542278 .register => |addr_reg| {
2255 self.register_manager.freezeRegs(&.{addr_reg});
2256 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2279 const addr_reg_lock = self.register_manager.lockReg(addr_reg);
2280 defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
22572281
22582282 switch (value) {
22592283 .dead => unreachable,
......@@ -2264,15 +2288,17 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
22642288 else => {
22652289 if (elem_size <= 4) {
22662290 const tmp_reg = try self.register_manager.allocReg(null);
2267 self.register_manager.freezeRegs(&.{tmp_reg});
2268 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2291 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2292 defer self.register_manager.unlockReg(tmp_reg_lock);
22692293
22702294 try self.genSetReg(value_ty, tmp_reg, value);
22712295 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
22722296 } else {
22732297 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2274 self.register_manager.freezeRegs(&regs);
2275 defer self.register_manager.unfreezeRegs(&regs);
2298 const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2299 defer for (regs_locks) |reg| {
2300 self.register_manager.unlockReg(reg);
2301 };
22762302
22772303 const src_reg = regs[0];
22782304 const dst_reg = addr_reg;
......@@ -2356,12 +2382,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
23562382 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
23572383 .immediate = struct_field_offset,
23582384 });
2359 self.register_manager.freezeRegs(&.{offset_reg});
2360 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2385 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2386 defer self.register_manager.unlockReg(offset_reg_lock);
23612387
23622388 const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
2363 self.register_manager.freezeRegs(&.{addr_reg});
2364 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2389 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
2390 defer self.register_manager.unlockReg(addr_reg_lock);
23652391
23662392 const dest = try self.binOp(
23672393 .add,
......@@ -2477,8 +2503,11 @@ fn binOpRegister(
24772503 const lhs_is_register = lhs == .register;
24782504 const rhs_is_register = rhs == .register;
24792505
2480 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
2481 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
2506 const lhs_lock: ?RegisterLock = if (lhs_is_register)
2507 self.register_manager.lockReg(lhs.register)
2508 else
2509 null;
2510 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
24822511
24832512 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
24842513
......@@ -2489,13 +2518,13 @@ fn binOpRegister(
24892518 } else null;
24902519
24912520 const reg = try self.register_manager.allocReg(track_inst);
2492 self.register_manager.freezeRegs(&.{reg});
24932521
24942522 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
24952523
24962524 break :blk reg;
24972525 };
2498 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
2526 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
2527 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
24992528
25002529 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
25012530 const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
......@@ -2504,13 +2533,13 @@ fn binOpRegister(
25042533 } else null;
25052534
25062535 const reg = try self.register_manager.allocReg(track_inst);
2507 self.register_manager.freezeRegs(&.{reg});
25082536
25092537 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
25102538
25112539 break :blk reg;
25122540 };
2513 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
2541 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
2542 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
25142543
25152544 const dest_reg = switch (mir_tag) {
25162545 .cmp => .r0, // cmp has no destination regardless
......@@ -2593,7 +2622,11 @@ fn binOpImmediate(
25932622) !MCValue {
25942623 const lhs_is_register = lhs == .register;
25952624
2596 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
2625 const lhs_lock: ?RegisterLock = if (lhs_is_register)
2626 self.register_manager.lockReg(lhs.register)
2627 else
2628 null;
2629 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
25972630
25982631 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
25992632
......@@ -2606,13 +2639,13 @@ fn binOpImmediate(
26062639 } else null;
26072640
26082641 const reg = try self.register_manager.allocReg(track_inst);
2609 self.register_manager.freezeRegs(&.{reg});
26102642
26112643 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
26122644
26132645 break :blk reg;
26142646 };
2615 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
2647 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
2648 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
26162649
26172650 const dest_reg = switch (mir_tag) {
26182651 .cmp => .r0, // cmp has no destination reg
......@@ -3656,6 +3689,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
36563689 const parent_compare_flags_inst = self.compare_flags_inst;
36573690
36583691 try self.branch_stack.append(.{});
3692 errdefer {
3693 _ = self.branch_stack.pop();
3694 }
36593695
36603696 try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
36613697 for (liveness_condbr.then_deaths) |operand| {
......@@ -4264,8 +4300,36 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
42644300 },
42654301 .register_c_flag,
42664302 .register_v_flag,
4267 => {
4268 return self.fail("TODO implement genSetStack {}", .{mcv});
4303 => |reg| {
4304 const reg_lock = self.register_manager.lockReg(reg);
4305 defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
4306
4307 const wrapped_ty = ty.structFieldType(0);
4308 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg });
4309
4310 const overflow_bit_ty = ty.structFieldType(1);
4311 const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
4312 const cond_reg = try self.register_manager.allocReg(null);
4313
4314 // C flag: movcs reg, #1
4315 // V flag: movvs reg, #1
4316 _ = try self.addInst(.{
4317 .tag = .mov,
4318 .cond = switch (mcv) {
4319 .register_c_flag => .cs,
4320 .register_v_flag => .vs,
4321 else => unreachable,
4322 },
4323 .data = .{ .rr_op = .{
4324 .rd = cond_reg,
4325 .rn = .r0,
4326 .op = Instruction.Operand.fromU32(1).?,
4327 } },
4328 });
4329
4330 try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
4331 .register = cond_reg,
4332 });
42694333 },
42704334 .memory,
42714335 .stack_argument_offset,
src/arch/riscv64/CodeGen.zig+12-8
......@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
2525const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2627
2728const FnResult = @import("../../codegen.zig").FnResult;
2829const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
......@@ -937,8 +938,11 @@ fn binOpRegister(
937938 const lhs_is_register = lhs == .register;
938939 const rhs_is_register = rhs == .register;
939940
940 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
941 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
941 const lhs_lock: ?RegisterLock = if (lhs_is_register)
942 self.register_manager.lockReg(lhs.register)
943 else
944 null;
945 defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
942946
943947 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
944948
......@@ -949,13 +953,13 @@ fn binOpRegister(
949953 } else null;
950954
951955 const reg = try self.register_manager.allocReg(track_inst);
952 self.register_manager.freezeRegs(&.{reg});
953956
954957 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
955958
956959 break :blk reg;
957960 };
958 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
961 const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
962 defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
959963
960964 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
961965 const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
......@@ -964,13 +968,13 @@ fn binOpRegister(
964968 } else null;
965969
966970 const reg = try self.register_manager.allocReg(track_inst);
967 self.register_manager.freezeRegs(&.{reg});
968971
969972 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
970973
971974 break :blk reg;
972975 };
973 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
976 const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
977 defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
974978
975979 const dest_reg = if (maybe_inst) |inst| blk: {
976980 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
......@@ -1448,8 +1452,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
14481452 .stack_offset,
14491453 => {
14501454 const reg = try self.register_manager.allocReg(null);
1451 self.register_manager.freezeRegs(&.{reg});
1452 defer self.register_manager.unfreezeRegs(&.{reg});
1455 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
1456 defer self.register_manager.unlockReg(reg_lock);
14531457
14541458 try self.genSetReg(ptr_ty, reg, ptr);
14551459 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
src/arch/x86_64/CodeGen.zig+701-482
......@@ -22,6 +22,8 @@ const Liveness = @import("../../Liveness.zig");
2222const Mir = @import("Mir.zig");
2323const Module = @import("../../Module.zig");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26const RegisterLock = RegisterManager.RegisterLock;
2527const Target = std.Target;
2628const Type = @import("../../type.zig").Type;
2729const TypedValue = @import("../../TypedValue.zig");
......@@ -42,8 +44,6 @@ const InnerError = error{
4244 OutOfRegisters,
4345};
4446
45const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
46
4747gpa: Allocator,
4848air: Air,
4949liveness: Liveness,
......@@ -191,60 +191,12 @@ pub const MCValue = union(enum) {
191191 };
192192 }
193193
194 fn usesCompareFlags(mcv: MCValue) bool {
195 return switch (mcv) {
196 .compare_flags_unsigned,
197 .compare_flags_signed,
198 .register_overflow_unsigned,
199 .register_overflow_signed,
200 => true,
201 else => false,
202 };
203 }
204
205194 fn isRegister(mcv: MCValue) bool {
206195 return switch (mcv) {
207 .register,
208 .register_overflow_unsigned,
209 .register_overflow_signed,
210 => true,
196 .register => true,
211197 else => false,
212198 };
213199 }
214
215 fn freezeIfRegister(mcv: MCValue, mgr: *RegisterManager) void {
216 switch (mcv) {
217 .register,
218 .register_overflow_signed,
219 .register_overflow_unsigned,
220 => |reg| {
221 mgr.freezeRegs(&.{reg});
222 },
223 else => {},
224 }
225 }
226
227 fn unfreezeIfRegister(mcv: MCValue, mgr: *RegisterManager) void {
228 switch (mcv) {
229 .register,
230 .register_overflow_signed,
231 .register_overflow_unsigned,
232 => |reg| {
233 mgr.unfreezeRegs(&.{reg});
234 },
235 else => {},
236 }
237 }
238
239 fn asRegister(mcv: MCValue) ?Register {
240 return switch (mcv) {
241 .register,
242 .register_overflow_signed,
243 .register_overflow_unsigned,
244 => |reg| reg,
245 else => null,
246 };
247 }
248200};
249201
250202const Branch = struct {
......@@ -819,7 +771,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
819771 // zig fmt: on
820772 }
821773
822 assert(!self.register_manager.frozenRegsExist());
774 assert(!self.register_manager.lockedRegsExist());
823775
824776 if (std.debug.runtime_safety) {
825777 if (self.air_bookkeeping < old_air_bookkeeping + 1) {
......@@ -876,15 +828,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
876828 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
877829 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
878830
879 if (result.asRegister()) |reg| {
880 // In some cases (such as bitcast), an operand
881 // may be the same MCValue as the result. If
882 // that operand died and was a register, it
883 // was freed by processDeath. We have to
884 // "re-allocate" the register.
885 if (self.register_manager.isRegFree(reg)) {
886 self.register_manager.getRegAssumeFree(reg, inst);
887 }
831 switch (result) {
832 .register,
833 .register_overflow_signed,
834 .register_overflow_unsigned,
835 => |reg| {
836 // In some cases (such as bitcast), an operand
837 // may be the same MCValue as the result. If
838 // that operand died and was a register, it
839 // was freed by processDeath. We have to
840 // "re-allocate" the register.
841 if (self.register_manager.isRegFree(reg)) {
842 self.register_manager.getRegAssumeFree(reg, inst);
843 }
844 },
845 else => {},
888846 }
889847 }
890848 self.finishAirBookkeeping();
......@@ -955,7 +913,15 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
955913 const stack_mcv = try self.allocRegOrMem(inst, false);
956914 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
957915 const reg_mcv = self.getResolvedInstValue(inst);
958 assert(reg.to64() == reg_mcv.asRegister().?.to64());
916 switch (reg_mcv) {
917 .register,
918 .register_overflow_unsigned,
919 .register_overflow_signed,
920 => |other| {
921 assert(reg.to64() == other.to64());
922 },
923 else => {},
924 }
959925 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
960926 try branch.inst_table.put(self.gpa, inst, stack_mcv);
961927 try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
......@@ -964,18 +930,32 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
964930pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
965931 if (self.compare_flags_inst) |inst_to_save| {
966932 const mcv = self.getResolvedInstValue(inst_to_save);
967 assert(mcv.usesCompareFlags());
933 const new_mcv = switch (mcv) {
934 .register_overflow_signed,
935 .register_overflow_unsigned,
936 => try self.allocRegOrMem(inst_to_save, false),
937 .compare_flags_signed,
938 .compare_flags_unsigned,
939 => try self.allocRegOrMem(inst_to_save, true),
940 else => unreachable,
941 };
968942
969 const new_mcv = try self.allocRegOrMem(inst_to_save, !mcv.isRegister());
970943 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
971944 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
945
972946 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
973947 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
974948
975949 self.compare_flags_inst = null;
950
976951 // TODO consolidate with register manager and spillInstruction
977952 // this call should really belong in the register manager!
978 if (mcv.isRegister()) self.register_manager.freeReg(mcv.asRegister().?);
953 switch (mcv) {
954 .register_overflow_signed,
955 .register_overflow_unsigned,
956 => |reg| self.register_manager.freeReg(reg),
957 else => {},
958 }
979959 }
980960}
981961
......@@ -1043,8 +1023,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
10431023 return self.fail("TODO implement intCast for abi sizes larger than 8", .{});
10441024 }
10451025
1046 operand.freezeIfRegister(&self.register_manager);
1047 defer operand.unfreezeIfRegister(&self.register_manager);
1026 const operand_lock: ?RegisterLock = switch (operand) {
1027 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1028 else => null,
1029 };
1030 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
10481031
10491032 const reg = try self.register_manager.allocReg(inst);
10501033 try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
......@@ -1071,8 +1054,11 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
10711054 return self.fail("TODO implement trunc for abi sizes larger than 8", .{});
10721055 }
10731056
1074 operand.freezeIfRegister(&self.register_manager);
1075 defer operand.unfreezeIfRegister(&self.register_manager);
1057 const operand_lock: ?RegisterLock = switch (operand) {
1058 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1059 else => null,
1060 };
1061 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
10761062
10771063 const reg: Register = blk: {
10781064 if (operand.isRegister()) {
......@@ -1156,16 +1142,22 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
11561142 // TODO improve by checking if any operand can be reused.
11571143 // TODO audit register allocation
11581144 const lhs = try self.resolveInst(bin_op.lhs);
1159 lhs.freezeIfRegister(&self.register_manager);
1160 defer lhs.unfreezeIfRegister(&self.register_manager);
1145 const lhs_lock: ?RegisterLock = switch (lhs) {
1146 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1147 else => null,
1148 };
1149 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
11611150
11621151 const lhs_reg = try self.copyToTmpRegister(ty, lhs);
1163 self.register_manager.freezeRegs(&.{lhs_reg});
1164 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1152 const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);
1153 defer self.register_manager.unlockReg(lhs_reg_lock);
11651154
11661155 const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32);
1167 rhs_mcv.freezeIfRegister(&self.register_manager);
1168 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1156 const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
1157 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1158 else => null,
1159 };
1160 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
11691161
11701162 try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);
11711163
......@@ -1200,28 +1192,37 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
12001192 const offset = try self.resolveInst(op_rhs);
12011193 const offset_ty = self.air.typeOf(op_rhs);
12021194
1203 offset.freezeIfRegister(&self.register_manager);
1204 defer offset.unfreezeIfRegister(&self.register_manager);
1195 const offset_lock: ?RegisterLock = switch (offset) {
1196 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1197 else => null,
1198 };
1199 defer if (offset_lock) |lock| self.register_manager.unlockReg(lock);
12051200
1206 const dst_mcv = blk: {
1201 const dst_mcv: MCValue = blk: {
12071202 if (self.reuseOperand(inst, op_lhs, 0, ptr)) {
12081203 if (ptr.isMemory() or ptr.isRegister()) break :blk ptr;
12091204 }
12101205 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, ptr) };
12111206 };
12121207
1213 dst_mcv.freezeIfRegister(&self.register_manager);
1214 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
1208 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
1209 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1210 else => null,
1211 };
1212 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
12151213
1216 const offset_mcv = blk: {
1214 const offset_mcv: MCValue = blk: {
12171215 if (self.reuseOperand(inst, op_rhs, 1, offset)) {
12181216 if (offset.isRegister()) break :blk offset;
12191217 }
12201218 break :blk MCValue{ .register = try self.copyToTmpRegister(offset_ty, offset) };
12211219 };
12221220
1223 offset_mcv.freezeIfRegister(&self.register_manager);
1224 defer offset_mcv.unfreezeIfRegister(&self.register_manager);
1221 const offset_mcv_lock: ?RegisterLock = switch (offset_mcv) {
1222 .register => |reg| self.register_manager.lockReg(reg),
1223 else => null,
1224 };
1225 defer if (offset_mcv_lock) |lock| self.register_manager.unlockReg(lock);
12251226
12261227 try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });
12271228
......@@ -1306,30 +1307,40 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
13061307 const dst_ty = self.air.typeOf(op_lhs);
13071308
13081309 const lhs = try self.resolveInst(op_lhs);
1309 lhs.freezeIfRegister(&self.register_manager);
1310 defer lhs.unfreezeIfRegister(&self.register_manager);
1310 const lhs_lock: ?RegisterLock = switch (lhs) {
1311 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1312 else => null,
1313 };
1314 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
13111315
13121316 const rhs = try self.resolveInst(op_rhs);
1313 rhs.freezeIfRegister(&self.register_manager);
1314 defer rhs.unfreezeIfRegister(&self.register_manager);
1317 const rhs_lock: ?RegisterLock = switch (rhs) {
1318 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1319 else => null,
1320 };
1321 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
13151322
1316 const dst_mcv = blk: {
1323 const dst_mcv: MCValue = blk: {
13171324 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
13181325 break :blk lhs;
13191326 }
13201327 break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);
13211328 };
1329 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
1330 .register => |reg| self.register_manager.lockReg(reg),
1331 else => null,
1332 };
1333 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
13221334
1323 dst_mcv.freezeIfRegister(&self.register_manager);
1324 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
1325
1326 const rhs_mcv = blk: {
1335 const rhs_mcv: MCValue = blk: {
13271336 if (rhs.isMemory() or rhs.isRegister()) break :blk rhs;
13281337 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
13291338 };
1330
1331 rhs_mcv.freezeIfRegister(&self.register_manager);
1332 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1339 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
1340 .register => |reg| self.register_manager.lockReg(reg),
1341 else => null,
1342 };
1343 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
13331344
13341345 try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv);
13351346
......@@ -1366,8 +1377,10 @@ fn airMul(self: *Self, inst: Air.Inst.Index) !void {
13661377 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
13671378 try self.register_manager.getReg(.rax, inst);
13681379 try self.register_manager.getReg(.rdx, null);
1369 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1370 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1380 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1381 defer for (reg_locks) |reg| {
1382 self.register_manager.unlockReg(reg);
1383 };
13711384
13721385 const lhs = try self.resolveInst(bin_op.lhs);
13731386 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1458,9 +1471,13 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14581471fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14591472 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
14601473 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1461 const result = if (self.liveness.isUnused(inst)) .dead else result: {
1462 const ty = self.air.typeOf(bin_op.lhs);
14631474
1475 if (self.liveness.isUnused(inst)) {
1476 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1477 }
1478
1479 const ty = self.air.typeOf(bin_op.lhs);
1480 const result: MCValue = result: {
14641481 switch (ty.zigTypeTag()) {
14651482 .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
14661483 .Int => {
......@@ -1477,8 +1494,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14771494 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
14781495 try self.register_manager.getReg(.rax, inst);
14791496 try self.register_manager.getReg(.rdx, null);
1480 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1481 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1497 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1498 defer for (reg_locks) |reg| {
1499 self.register_manager.unlockReg(reg);
1500 };
14821501
14831502 const lhs = try self.resolveInst(bin_op.lhs);
14841503 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1504,21 +1523,28 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15041523 const lhs = try self.resolveInst(bin_op.lhs);
15051524 const rhs = try self.resolveInst(bin_op.rhs);
15061525
1507 rhs.freezeIfRegister(&self.register_manager);
1508 defer rhs.unfreezeIfRegister(&self.register_manager);
1526 const rhs_lock: ?RegisterLock = switch (rhs) {
1527 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1528 else => null,
1529 };
1530 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
15091531
15101532 const dst_reg: Register = blk: {
15111533 if (lhs.isRegister()) break :blk lhs.register;
15121534 break :blk try self.copyToTmpRegister(ty, lhs);
15131535 };
1514 self.register_manager.freezeRegs(&.{dst_reg});
1536 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1537 defer self.register_manager.unlockReg(dst_reg_lock);
15151538
1516 const rhs_mcv = blk: {
1539 const rhs_mcv: MCValue = blk: {
15171540 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
15181541 break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };
15191542 };
1520 rhs_mcv.freezeIfRegister(&self.register_manager);
1521 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1543 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
1544 .register => |reg| self.register_manager.lockReg(reg),
1545 else => null,
1546 };
1547 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
15221548
15231549 try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
15241550
......@@ -1528,8 +1554,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15281554 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
15291555 try self.register_manager.getReg(.rax, null);
15301556 try self.register_manager.getReg(.rdx, null);
1531 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1532 defer self.register_manager.unfreezeRegs(&.{.rdx});
1557 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1558 defer for (reg_locks) |reg| {
1559 self.register_manager.unlockReg(reg);
1560 };
15331561
15341562 const lhs = try self.resolveInst(bin_op.lhs);
15351563 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1540,7 +1568,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15401568 },
15411569 }
15421570 };
1543 defer self.register_manager.unfreezeRegs(&.{dst_reg});
1571 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1572 defer self.register_manager.unlockReg(dst_reg_lock);
15441573
15451574 const tuple_ty = self.air.typeOfIndex(inst);
15461575 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
......@@ -1554,8 +1583,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15541583 };
15551584
15561585 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1557 self.register_manager.freezeRegs(&temp_regs);
1558 defer self.register_manager.unfreezeRegs(&temp_regs);
1586 const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
1587 defer for (temp_regs_locks) |reg| {
1588 self.register_manager.unlockReg(reg);
1589 };
15591590
15601591 const overflow_reg = temp_regs[0];
15611592 const flags: u2 = switch (int_info.signedness) {
......@@ -1699,18 +1730,19 @@ fn genIntMulDivOpMir(
16991730/// Clobbers .rax and .rdx registers.
17001731fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
17011732 const signedness = ty.intInfo(self.target.*).signedness;
1702 const dividend = switch (lhs) {
1733 const dividend: Register = switch (lhs) {
17031734 .register => |reg| reg,
17041735 else => try self.copyToTmpRegister(ty, lhs),
17051736 };
1706 self.register_manager.freezeRegs(&.{dividend});
1737 const dividend_lock = self.register_manager.lockReg(dividend);
1738 defer if (dividend_lock) |lock| self.register_manager.unlockReg(lock);
17071739
1708 const divisor = switch (rhs) {
1740 const divisor: Register = switch (rhs) {
17091741 .register => |reg| reg,
17101742 else => try self.copyToTmpRegister(ty, rhs),
17111743 };
1712 self.register_manager.freezeRegs(&.{divisor});
1713 defer self.register_manager.unfreezeRegs(&.{ dividend, divisor });
1744 const divisor_lock = self.register_manager.lockReg(divisor);
1745 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
17141746
17151747 try self.genIntMulDivOpMir(switch (signedness) {
17161748 .signed => .idiv,
......@@ -1755,54 +1787,71 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
17551787
17561788fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
17571789 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1758 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1759 const tag = self.air.instructions.items(.tag)[inst];
1760 const ty = self.air.typeOfIndex(inst);
17611790
1762 if (ty.zigTypeTag() != .Int) {
1763 return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });
1764 }
1791 if (self.liveness.isUnused(inst)) {
1792 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1793 }
1794
1795 const tag = self.air.instructions.items(.tag)[inst];
1796 const ty = self.air.typeOfIndex(inst);
17651797
1766 if (tag == .div_float) {
1767 return self.fail("TODO implement {}", .{tag});
1798 if (ty.zigTypeTag() != .Int) {
1799 return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });
1800 }
1801
1802 if (tag == .div_float) {
1803 return self.fail("TODO implement {}", .{tag});
1804 }
1805
1806 const signedness = ty.intInfo(self.target.*).signedness;
1807
1808 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1809 const track_rax: ?Air.Inst.Index = blk: {
1810 if (signedness == .unsigned) break :blk inst;
1811 switch (tag) {
1812 .div_exact, .div_trunc => break :blk inst,
1813 else => break :blk null,
17681814 }
1815 };
1816 try self.register_manager.getReg(.rax, track_rax);
1817 try self.register_manager.getReg(.rdx, null);
1818 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1819 defer for (reg_locks) |reg| {
1820 self.register_manager.unlockReg(reg);
1821 };
17691822
1770 const signedness = ty.intInfo(self.target.*).signedness;
1823 const lhs = try self.resolveInst(bin_op.lhs);
1824 const lhs_lock: ?RegisterLock = switch (lhs) {
1825 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1826 else => null,
1827 };
1828 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
17711829
1772 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1773 const track_rax: ?Air.Inst.Index = blk: {
1774 if (signedness == .unsigned) break :blk inst;
1830 const rhs: MCValue = blk: {
1831 const rhs = try self.resolveInst(bin_op.rhs);
1832 if (signedness == .signed) {
17751833 switch (tag) {
1776 .div_exact, .div_trunc => break :blk inst,
1777 else => break :blk null,
1778 }
1779 };
1780 try self.register_manager.getReg(.rax, track_rax);
1781 try self.register_manager.getReg(.rdx, null);
1782 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1783 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1834 .div_floor => {
1835 const rhs_lock: ?RegisterLock = switch (rhs) {
1836 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1837 else => null,
1838 };
1839 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
17841840
1785 const lhs = try self.resolveInst(bin_op.lhs);
1786 lhs.freezeIfRegister(&self.register_manager);
1787 defer lhs.unfreezeIfRegister(&self.register_manager);
1788
1789 const rhs = blk: {
1790 const rhs = try self.resolveInst(bin_op.rhs);
1791 if (signedness == .signed) {
1792 switch (tag) {
1793 .div_floor => {
1794 rhs.freezeIfRegister(&self.register_manager);
1795 defer rhs.unfreezeIfRegister(&self.register_manager);
1796 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1797 },
1798 else => {},
1799 }
1841 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1842 },
1843 else => {},
18001844 }
1801 break :blk rhs;
1802 };
1803 rhs.freezeIfRegister(&self.register_manager);
1804 defer rhs.unfreezeIfRegister(&self.register_manager);
1845 }
1846 break :blk rhs;
1847 };
1848 const rhs_lock: ?RegisterLock = switch (rhs) {
1849 .register => |reg| self.register_manager.lockReg(reg),
1850 else => null,
1851 };
1852 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
18051853
1854 const result: MCValue = result: {
18061855 if (signedness == .unsigned) {
18071856 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);
18081857 break :result MCValue{ .register = .rax };
......@@ -1822,53 +1871,67 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
18221871 else => unreachable,
18231872 }
18241873 };
1874
18251875 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
18261876}
18271877
18281878fn airRem(self: *Self, inst: Air.Inst.Index) !void {
18291879 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1830 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1831 const ty = self.air.typeOfIndex(inst);
1832 if (ty.zigTypeTag() != .Int) {
1833 return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
1834 }
1835 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1836 try self.register_manager.getReg(.rax, null);
1837 try self.register_manager.getReg(.rdx, inst);
1838 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1839 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1840
1841 const lhs = try self.resolveInst(bin_op.lhs);
1842 const rhs = try self.resolveInst(bin_op.rhs);
18431880
1844 const signedness = ty.intInfo(self.target.*).signedness;
1845 try self.genIntMulDivOpMir(switch (signedness) {
1846 .signed => .idiv,
1847 .unsigned => .div,
1848 }, ty, signedness, lhs, rhs);
1849 break :result MCValue{ .register = .rdx };
1881 if (self.liveness.isUnused(inst)) {
1882 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1883 }
1884 const ty = self.air.typeOfIndex(inst);
1885 if (ty.zigTypeTag() != .Int) {
1886 return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
1887 }
1888 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1889 try self.register_manager.getReg(.rax, null);
1890 try self.register_manager.getReg(.rdx, inst);
1891 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1892 defer for (reg_locks) |reg| {
1893 self.register_manager.unlockReg(reg);
18501894 };
1895
1896 const lhs = try self.resolveInst(bin_op.lhs);
1897 const rhs = try self.resolveInst(bin_op.rhs);
1898
1899 const signedness = ty.intInfo(self.target.*).signedness;
1900 try self.genIntMulDivOpMir(switch (signedness) {
1901 .signed => .idiv,
1902 .unsigned => .div,
1903 }, ty, signedness, lhs, rhs);
1904
1905 const result: MCValue = .{ .register = .rdx };
1906
18511907 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
18521908}
18531909
18541910fn airMod(self: *Self, inst: Air.Inst.Index) !void {
18551911 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1856 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1857 const ty = self.air.typeOfIndex(inst);
1858 if (ty.zigTypeTag() != .Int) {
1859 return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
1860 }
1861 const signedness = ty.intInfo(self.target.*).signedness;
18621912
1863 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1864 try self.register_manager.getReg(.rax, null);
1865 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);
1866 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1867 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1913 if (self.liveness.isUnused(inst)) {
1914 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1915 }
18681916
1869 const lhs = try self.resolveInst(bin_op.lhs);
1870 const rhs = try self.resolveInst(bin_op.rhs);
1917 const ty = self.air.typeOfIndex(inst);
1918 if (ty.zigTypeTag() != .Int) {
1919 return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
1920 }
1921 const signedness = ty.intInfo(self.target.*).signedness;
1922
1923 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1924 try self.register_manager.getReg(.rax, null);
1925 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);
1926 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1927 defer for (reg_locks) |reg| {
1928 self.register_manager.unlockReg(reg);
1929 };
1930
1931 const lhs = try self.resolveInst(bin_op.lhs);
1932 const rhs = try self.resolveInst(bin_op.rhs);
18711933
1934 const result: MCValue = result: {
18721935 switch (signedness) {
18731936 .unsigned => {
18741937 try self.genIntMulDivOpMir(switch (signedness) {
......@@ -1888,6 +1951,7 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {
18881951 },
18891952 }
18901953 };
1954
18911955 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
18921956}
18931957
......@@ -1954,12 +2018,15 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {
19542018 try self.register_manager.getReg(.rcx, null);
19552019 try self.genSetReg(shift_ty, .rcx, shift);
19562020 }
1957 self.register_manager.freezeRegs(&.{.rcx});
1958 defer self.register_manager.unfreezeRegs(&.{.rcx});
2021 const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
2022 defer self.register_manager.unlockReg(rcx_lock);
19592023
19602024 const value = try self.resolveInst(bin_op.lhs);
1961 value.freezeIfRegister(&self.register_manager);
1962 defer value.unfreezeIfRegister(&self.register_manager);
2025 const value_lock: ?RegisterLock = switch (value) {
2026 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2027 else => null,
2028 };
2029 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
19632030
19642031 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value);
19652032 _ = try self.addInst(.{
......@@ -2055,8 +2122,11 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
20552122 const err_ty = err_union_ty.errorUnionSet();
20562123 const payload_ty = err_union_ty.errorUnionPayload();
20572124 const operand = try self.resolveInst(ty_op.operand);
2058 operand.freezeIfRegister(&self.register_manager);
2059 defer operand.unfreezeIfRegister(&self.register_manager);
2125 const operand_lock: ?RegisterLock = switch (operand) {
2126 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2127 else => null,
2128 };
2129 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
20602130
20612131 const result: MCValue = result: {
20622132 if (!payload_ty.hasRuntimeBits()) break :result operand;
......@@ -2085,8 +2155,11 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
20852155 if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
20862156
20872157 const operand = try self.resolveInst(ty_op.operand);
2088 operand.freezeIfRegister(&self.register_manager);
2089 defer operand.unfreezeIfRegister(&self.register_manager);
2158 const operand_lock: ?RegisterLock = switch (operand) {
2159 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2160 else => null,
2161 };
2162 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
20902163
20912164 const abi_align = err_union_ty.abiAlignment(self.target.*);
20922165 const err_ty = err_union_ty.errorUnionSet();
......@@ -2154,8 +2227,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
21542227
21552228 const optional_ty = self.air.typeOfIndex(inst);
21562229 const operand = try self.resolveInst(ty_op.operand);
2157 operand.freezeIfRegister(&self.register_manager);
2158 defer operand.unfreezeIfRegister(&self.register_manager);
2230 const operand_lock: ?RegisterLock = switch (operand) {
2231 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2232 else => null,
2233 };
2234 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
21592235
21602236 if (optional_ty.isPtrLikeOptional()) {
21612237 // TODO should we check if we can reuse the operand?
......@@ -2288,8 +2364,11 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
22882364fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
22892365 const slice_ty = self.air.typeOf(lhs);
22902366 const slice_mcv = try self.resolveInst(lhs);
2291 slice_mcv.freezeIfRegister(&self.register_manager);
2292 defer slice_mcv.unfreezeIfRegister(&self.register_manager);
2367 const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) {
2368 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2369 else => null,
2370 };
2371 defer if (slice_mcv_lock) |lock| self.register_manager.unlockReg(lock);
22932372
22942373 const elem_ty = slice_ty.childType();
22952374 const elem_size = elem_ty.abiSize(self.target.*);
......@@ -2298,12 +2377,15 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
22982377
22992378 const index_ty = self.air.typeOf(rhs);
23002379 const index_mcv = try self.resolveInst(rhs);
2301 index_mcv.freezeIfRegister(&self.register_manager);
2302 defer index_mcv.unfreezeIfRegister(&self.register_manager);
2380 const index_mcv_lock: ?RegisterLock = switch (index_mcv) {
2381 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2382 else => null,
2383 };
2384 defer if (index_mcv_lock) |lock| self.register_manager.unlockReg(lock);
23032385
23042386 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);
2305 self.register_manager.freezeRegs(&.{offset_reg});
2306 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2387 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2388 defer self.register_manager.unlockReg(offset_reg_lock);
23072389
23082390 const addr_reg = try self.register_manager.allocReg(null);
23092391 switch (slice_mcv) {
......@@ -2356,98 +2438,119 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
23562438
23572439fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
23582440 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2359 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2360 const array_ty = self.air.typeOf(bin_op.lhs);
2361 const array = try self.resolveInst(bin_op.lhs);
2362 array.freezeIfRegister(&self.register_manager);
2363 defer array.unfreezeIfRegister(&self.register_manager);
2364
2365 const elem_ty = array_ty.childType();
2366 const elem_abi_size = elem_ty.abiSize(self.target.*);
23672441
2368 const index_ty = self.air.typeOf(bin_op.rhs);
2369 const index = try self.resolveInst(bin_op.rhs);
2370 index.freezeIfRegister(&self.register_manager);
2371 defer index.unfreezeIfRegister(&self.register_manager);
2442 if (self.liveness.isUnused(inst)) {
2443 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2444 }
23722445
2373 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2374 self.register_manager.freezeRegs(&.{offset_reg});
2375 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2446 const array_ty = self.air.typeOf(bin_op.lhs);
2447 const array = try self.resolveInst(bin_op.lhs);
2448 const array_lock: ?RegisterLock = switch (array) {
2449 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2450 else => null,
2451 };
2452 defer if (array_lock) |lock| self.register_manager.unlockReg(lock);
23762453
2377 const addr_reg = try self.register_manager.allocReg(null);
2378 switch (array) {
2379 .register => {
2380 const off = @intCast(i32, try self.allocMem(
2381 inst,
2382 @intCast(u32, array_ty.abiSize(self.target.*)),
2383 array_ty.abiAlignment(self.target.*),
2384 ));
2385 try self.genSetStack(array_ty, off, array, .{});
2386 // lea reg, [rbp]
2387 _ = try self.addInst(.{
2388 .tag = .lea,
2389 .ops = (Mir.Ops{
2390 .reg1 = addr_reg.to64(),
2391 .reg2 = .rbp,
2392 }).encode(),
2393 .data = .{ .imm = @bitCast(u32, -off) },
2394 });
2395 },
2396 .stack_offset => |off| {
2397 // lea reg, [rbp]
2398 _ = try self.addInst(.{
2399 .tag = .lea,
2400 .ops = (Mir.Ops{
2401 .reg1 = addr_reg.to64(),
2402 .reg2 = .rbp,
2403 }).encode(),
2404 .data = .{ .imm = @bitCast(u32, -off) },
2405 });
2406 },
2407 .memory,
2408 .got_load,
2409 .direct_load,
2410 => {
2411 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
2412 },
2413 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2414 }
2454 const elem_ty = array_ty.childType();
2455 const elem_abi_size = elem_ty.abiSize(self.target.*);
24152456
2416 // TODO we could allocate register here, but need to expect addr register and potentially
2417 // offset register.
2418 const dst_mcv = try self.allocRegOrMem(inst, false);
2419 try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2420 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
2421 break :result dst_mcv;
2457 const index_ty = self.air.typeOf(bin_op.rhs);
2458 const index = try self.resolveInst(bin_op.rhs);
2459 const index_lock: ?RegisterLock = switch (index) {
2460 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2461 else => null,
24222462 };
2423 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2463 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
2464
2465 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2466 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2467 defer self.register_manager.unlockReg(offset_reg_lock);
2468
2469 const addr_reg = try self.register_manager.allocReg(null);
2470 switch (array) {
2471 .register => {
2472 const off = @intCast(i32, try self.allocMem(
2473 inst,
2474 @intCast(u32, array_ty.abiSize(self.target.*)),
2475 array_ty.abiAlignment(self.target.*),
2476 ));
2477 try self.genSetStack(array_ty, off, array, .{});
2478 // lea reg, [rbp]
2479 _ = try self.addInst(.{
2480 .tag = .lea,
2481 .ops = (Mir.Ops{
2482 .reg1 = addr_reg.to64(),
2483 .reg2 = .rbp,
2484 }).encode(),
2485 .data = .{ .imm = @bitCast(u32, -off) },
2486 });
2487 },
2488 .stack_offset => |off| {
2489 // lea reg, [rbp]
2490 _ = try self.addInst(.{
2491 .tag = .lea,
2492 .ops = (Mir.Ops{
2493 .reg1 = addr_reg.to64(),
2494 .reg2 = .rbp,
2495 }).encode(),
2496 .data = .{ .imm = @bitCast(u32, -off) },
2497 });
2498 },
2499 .memory,
2500 .got_load,
2501 .direct_load,
2502 => {
2503 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
2504 },
2505 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2506 }
2507
2508 // TODO we could allocate register here, but need to expect addr register and potentially
2509 // offset register.
2510 const dst_mcv = try self.allocRegOrMem(inst, false);
2511 try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2512 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
2513
2514 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
24242515}
24252516
24262517fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
24272518 const is_volatile = false; // TODO
24282519 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2429 const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else result: {
2430 // this is identical to the `airPtrElemPtr` codegen expect here an
2431 // additional `mov` is needed at the end to get the actual value
2432
2433 const ptr_ty = self.air.typeOf(bin_op.lhs);
2434 const ptr = try self.resolveInst(bin_op.lhs);
2435 ptr.freezeIfRegister(&self.register_manager);
2436 defer ptr.unfreezeIfRegister(&self.register_manager);
2437
2438 const elem_ty = ptr_ty.elemType2();
2439 const elem_abi_size = elem_ty.abiSize(self.target.*);
2440 const index_ty = self.air.typeOf(bin_op.rhs);
2441 const index = try self.resolveInst(bin_op.rhs);
2442 index.freezeIfRegister(&self.register_manager);
2443 defer index.unfreezeIfRegister(&self.register_manager);
2444
2445 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2446 self.register_manager.freezeRegs(&.{offset_reg});
2447 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2448
2449 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2450 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2520
2521 if (!is_volatile and self.liveness.isUnused(inst)) {
2522 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2523 }
2524
2525 // this is identical to the `airPtrElemPtr` codegen expect here an
2526 // additional `mov` is needed at the end to get the actual value
2527
2528 const ptr_ty = self.air.typeOf(bin_op.lhs);
2529 const ptr = try self.resolveInst(bin_op.lhs);
2530 const ptr_lock: ?RegisterLock = switch (ptr) {
2531 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2532 else => null,
2533 };
2534 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
2535
2536 const elem_ty = ptr_ty.elemType2();
2537 const elem_abi_size = elem_ty.abiSize(self.target.*);
2538 const index_ty = self.air.typeOf(bin_op.rhs);
2539 const index = try self.resolveInst(bin_op.rhs);
2540 const index_lock: ?RegisterLock = switch (index) {
2541 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2542 else => null,
2543 };
2544 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
2545
2546 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2547 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2548 defer self.register_manager.unlockReg(offset_reg_lock);
2549
2550 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2551 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2552
2553 const result: MCValue = result: {
24512554 if (elem_abi_size > 8) {
24522555 return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});
24532556 } else {
......@@ -2464,34 +2567,44 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
24642567 break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
24652568 }
24662569 };
2570
24672571 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
24682572}
24692573
24702574fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
24712575 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
24722576 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2473 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2474 const ptr_ty = self.air.typeOf(extra.lhs);
2475 const ptr = try self.resolveInst(extra.lhs);
2476 ptr.freezeIfRegister(&self.register_manager);
2477 defer ptr.unfreezeIfRegister(&self.register_manager);
2478
2479 const elem_ty = ptr_ty.elemType2();
2480 const elem_abi_size = elem_ty.abiSize(self.target.*);
2481 const index_ty = self.air.typeOf(extra.rhs);
2482 const index = try self.resolveInst(extra.rhs);
2483 index.freezeIfRegister(&self.register_manager);
2484 defer index.unfreezeIfRegister(&self.register_manager);
2485
2486 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2487 self.register_manager.freezeRegs(&.{offset_reg});
2488 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2489
2490 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2491 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2492 break :result dst_mcv;
2577
2578 if (self.liveness.isUnused(inst)) {
2579 return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2580 }
2581
2582 const ptr_ty = self.air.typeOf(extra.lhs);
2583 const ptr = try self.resolveInst(extra.lhs);
2584 const ptr_lock: ?RegisterLock = switch (ptr) {
2585 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2586 else => null,
24932587 };
2494 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2588 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
2589
2590 const elem_ty = ptr_ty.elemType2();
2591 const elem_abi_size = elem_ty.abiSize(self.target.*);
2592 const index_ty = self.air.typeOf(extra.rhs);
2593 const index = try self.resolveInst(extra.rhs);
2594 const index_lock: ?RegisterLock = switch (index) {
2595 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2596 else => null,
2597 };
2598 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
2599
2600 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2601 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2602 defer self.register_manager.unlockReg(offset_reg_lock);
2603
2604 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2605 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2606
2607 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
24952608}
24962609
24972610fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
......@@ -2506,12 +2619,18 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
25062619 }
25072620
25082621 const ptr = try self.resolveInst(bin_op.lhs);
2509 ptr.freezeIfRegister(&self.register_manager);
2510 defer ptr.unfreezeIfRegister(&self.register_manager);
2622 const ptr_lock: ?RegisterLock = switch (ptr) {
2623 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2624 else => null,
2625 };
2626 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
25112627
25122628 const tag = try self.resolveInst(bin_op.rhs);
2513 tag.freezeIfRegister(&self.register_manager);
2514 defer tag.unfreezeIfRegister(&self.register_manager);
2629 const tag_lock: ?RegisterLock = switch (tag) {
2630 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2631 else => null,
2632 };
2633 defer if (tag_lock) |lock| self.register_manager.unlockReg(lock);
25152634
25162635 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
25172636 // TODO reusing the operand
......@@ -2541,8 +2660,11 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
25412660
25422661 // TODO reusing the operand
25432662 const operand = try self.resolveInst(ty_op.operand);
2544 operand.freezeIfRegister(&self.register_manager);
2545 defer operand.unfreezeIfRegister(&self.register_manager);
2663 const operand_lock: ?RegisterLock = switch (operand) {
2664 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2665 else => null,
2666 };
2667 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
25462668
25472669 const tag_abi_size = tag_ty.abiSize(self.target.*);
25482670 const dst_mcv: MCValue = blk: {
......@@ -2689,8 +2811,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
26892811 try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
26902812 },
26912813 .register => |reg| {
2692 self.register_manager.freezeRegs(&.{reg});
2693 defer self.register_manager.unfreezeRegs(&.{reg});
2814 const reg_lock = self.register_manager.lockReg(reg);
2815 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
26942816
26952817 switch (dst_mcv) {
26962818 .dead => unreachable,
......@@ -2815,8 +2937,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
28152937 try self.genSetStack(value_ty, off, value, .{});
28162938 },
28172939 .register => |reg| {
2818 self.register_manager.freezeRegs(&.{reg});
2819 defer self.register_manager.unfreezeRegs(&.{reg});
2940 const reg_lock = self.register_manager.lockReg(reg);
2941 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
28202942
28212943 switch (value) {
28222944 .none => unreachable,
......@@ -2906,12 +3028,15 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
29063028 .direct_load,
29073029 .memory,
29083030 => {
2909 value.freezeIfRegister(&self.register_manager);
2910 defer value.unfreezeIfRegister(&self.register_manager);
3031 const value_lock: ?RegisterLock = switch (value) {
3032 .register => |reg| self.register_manager.lockReg(reg),
3033 else => null,
3034 };
3035 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
29113036
29123037 const addr_reg = try self.register_manager.allocReg(null);
2913 self.register_manager.freezeRegs(&.{addr_reg});
2914 defer self.register_manager.unfreezeRegs(&.{addr_reg});
3038 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3039 defer self.register_manager.unlockReg(addr_reg_lock);
29153040
29163041 try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
29173042
......@@ -2982,8 +3107,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
29823107 => {
29833108 if (abi_size <= 8) {
29843109 const tmp_reg = try self.register_manager.allocReg(null);
2985 self.register_manager.freezeRegs(&.{tmp_reg});
2986 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
3110 const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3111 defer self.register_manager.unlockReg(tmp_reg_lock);
29873112
29883113 try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
29893114
......@@ -3073,8 +3198,8 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
30733198 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
30743199 .immediate = struct_field_offset,
30753200 });
3076 self.register_manager.freezeRegs(&.{offset_reg});
3077 defer self.register_manager.unfreezeRegs(&.{offset_reg});
3201 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3202 defer self.register_manager.unlockReg(offset_reg_lock);
30783203
30793204 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, mcv);
30803205 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
......@@ -3085,24 +3210,27 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
30853210 break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };
30863211 },
30873212 .register => |reg| {
3213 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3214 defer self.register_manager.unlockReg(reg_lock);
3215
30883216 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
30893217 .immediate = struct_field_offset,
30903218 });
3091 self.register_manager.freezeRegs(&.{offset_reg});
3092 defer self.register_manager.unfreezeRegs(&.{offset_reg});
3219 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3220 defer self.register_manager.unlockReg(offset_reg_lock);
30933221
30943222 const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
3095 const result_reg = blk: {
3223 const result_reg: Register = blk: {
30963224 if (can_reuse_operand) {
30973225 break :blk reg;
30983226 } else {
3099 self.register_manager.freezeRegs(&.{reg});
31003227 const result_reg = try self.register_manager.allocReg(inst);
31013228 try self.genSetReg(ptr_ty, result_reg, mcv);
31023229 break :blk result_reg;
31033230 }
31043231 };
3105 defer if (!can_reuse_operand) self.register_manager.unfreezeRegs(&.{reg});
3232 const result_reg_lock = self.register_manager.lockReg(result_reg);
3233 defer if (result_reg_lock) |lock| self.register_manager.unlockReg(lock);
31063234
31073235 try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });
31083236 break :result MCValue{ .register = result_reg };
......@@ -3118,22 +3246,27 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
31183246 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
31193247 const operand = extra.struct_operand;
31203248 const index = extra.field_index;
3121 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3122 const mcv = try self.resolveInst(operand);
3123 const struct_ty = self.air.typeOf(operand);
3124 const struct_field_offset = struct_ty.structFieldOffset(index, self.target.*);
3125 const struct_field_ty = struct_ty.structFieldType(index);
31263249
3250 if (self.liveness.isUnused(inst)) {
3251 return self.finishAir(inst, .dead, .{ extra.struct_operand, .none, .none });
3252 }
3253
3254 const mcv = try self.resolveInst(operand);
3255 const struct_ty = self.air.typeOf(operand);
3256 const struct_field_offset = struct_ty.structFieldOffset(index, self.target.*);
3257 const struct_field_ty = struct_ty.structFieldType(index);
3258
3259 const result: MCValue = result: {
31273260 switch (mcv) {
31283261 .stack_offset => |off| {
31293262 const stack_offset = off - @intCast(i32, struct_field_offset);
31303263 break :result MCValue{ .stack_offset = stack_offset };
31313264 },
31323265 .register => |reg| {
3133 self.register_manager.freezeRegs(&.{reg});
3134 defer self.register_manager.unfreezeRegs(&.{reg});
3266 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3267 defer self.register_manager.unlockReg(reg_lock);
31353268
3136 const dst_mcv = blk: {
3269 const dst_mcv: MCValue = blk: {
31373270 if (self.reuseOperand(inst, operand, 0, mcv)) {
31383271 break :blk mcv;
31393272 } else {
......@@ -3143,8 +3276,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
31433276 break :blk dst_mcv;
31443277 }
31453278 };
3146 dst_mcv.freezeIfRegister(&self.register_manager);
3147 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
3279 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3280 .register => |a_reg| self.register_manager.lockReg(a_reg),
3281 else => null,
3282 };
3283 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
31483284
31493285 // Shift by struct_field_offset.
31503286 const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
......@@ -3186,8 +3322,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
31863322 },
31873323 1 => {
31883324 // Get overflow bit.
3189 mcv.freezeIfRegister(&self.register_manager);
3190 defer mcv.unfreezeIfRegister(&self.register_manager);
3325 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3326 defer self.register_manager.unlockReg(reg_lock);
31913327
31923328 const dst_reg = try self.register_manager.allocReg(inst);
31933329 const flags: u2 = switch (mcv) {
......@@ -3229,15 +3365,21 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
32293365 const dst_ty = self.air.typeOf(op_lhs);
32303366
32313367 const lhs = try self.resolveInst(op_lhs);
3232 lhs.freezeIfRegister(&self.register_manager);
3233 defer lhs.unfreezeIfRegister(&self.register_manager);
3368 const lhs_lock: ?RegisterLock = switch (lhs) {
3369 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3370 else => null,
3371 };
3372 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
32343373
32353374 const rhs = try self.resolveInst(op_rhs);
3236 rhs.freezeIfRegister(&self.register_manager);
3237 defer rhs.unfreezeIfRegister(&self.register_manager);
3375 const rhs_lock: ?RegisterLock = switch (rhs) {
3376 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3377 else => null,
3378 };
3379 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
32383380
32393381 var flipped: bool = false;
3240 const dst_mcv = blk: {
3382 const dst_mcv: MCValue = blk: {
32413383 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
32423384 break :blk lhs;
32433385 }
......@@ -3247,16 +3389,22 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
32473389 }
32483390 break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);
32493391 };
3250 dst_mcv.freezeIfRegister(&self.register_manager);
3251 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
3392 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3393 .register => |reg| self.register_manager.lockReg(reg),
3394 else => null,
3395 };
3396 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
32523397
3253 const src_mcv = blk: {
3398 const src_mcv: MCValue = blk: {
32543399 const mcv = if (flipped) lhs else rhs;
32553400 if (mcv.isRegister() or mcv.isMemory()) break :blk mcv;
32563401 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) };
32573402 };
3258 src_mcv.freezeIfRegister(&self.register_manager);
3259 defer src_mcv.unfreezeIfRegister(&self.register_manager);
3403 const src_mcv_lock: ?RegisterLock = switch (src_mcv) {
3404 .register => |reg| self.register_manager.lockReg(reg),
3405 else => null,
3406 };
3407 defer if (src_mcv_lock) |lock| self.register_manager.unlockReg(lock);
32603408
32613409 const tag = self.air.instructions.items(.tag)[inst];
32623410 switch (tag) {
......@@ -3287,8 +3435,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
32873435 .register_overflow_unsigned => unreachable,
32883436 .register_overflow_signed => unreachable,
32893437 .ptr_stack_offset => {
3290 self.register_manager.freezeRegs(&.{dst_reg});
3291 defer self.register_manager.unfreezeRegs(&.{dst_reg});
3438 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3439 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
3440
32923441 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
32933442 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
32943443 },
......@@ -3318,8 +3467,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
33183467 .compare_flags_unsigned,
33193468 => {
33203469 assert(abi_size <= 8);
3321 self.register_manager.freezeRegs(&.{dst_reg});
3322 defer self.register_manager.unfreezeRegs(&.{dst_reg});
3470 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3471 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
3472
33233473 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
33243474 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
33253475 },
......@@ -3659,20 +3809,25 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
36593809 try self.register_manager.getReg(reg, null);
36603810 }
36613811
3662 if (info.return_value == .stack_offset) {
3663 const ret_ty = fn_ty.fnReturnType();
3664 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3665 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
3666 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
3667 log.debug("airCall: return value on stack at offset {}", .{stack_offset});
3812 const rdi_lock: ?RegisterLock = blk: {
3813 if (info.return_value == .stack_offset) {
3814 const ret_ty = fn_ty.fnReturnType();
3815 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3816 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
3817 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
3818 log.debug("airCall: return value on stack at offset {}", .{stack_offset});
36683819
3669 try self.register_manager.getReg(.rdi, null);
3670 try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });
3671 self.register_manager.freezeRegs(&.{.rdi});
3820 try self.register_manager.getReg(.rdi, null);
3821 try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });
3822 const rdi_lock = self.register_manager.lockRegAssumeUnused(.rdi);
36723823
3673 info.return_value.stack_offset = stack_offset;
3674 }
3675 defer if (info.return_value == .stack_offset) self.register_manager.unfreezeRegs(&.{.rdi});
3824 info.return_value.stack_offset = stack_offset;
3825
3826 break :blk rdi_lock;
3827 }
3828 break :blk null;
3829 };
3830 defer if (rdi_lock) |lock| self.register_manager.unlockReg(lock);
36763831
36773832 for (args) |arg, arg_i| {
36783833 const mc_arg = info.args[arg_i];
......@@ -3891,11 +4046,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
38914046 const ret_ty = self.fn_type.fnReturnType();
38924047 switch (self.ret_mcv) {
38934048 .stack_offset => {
3894 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3895 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
38964049 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3897 self.register_manager.freezeRegs(&.{reg});
3898 defer self.register_manager.unfreezeRegs(&.{reg});
4050 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4051 defer self.register_manager.unlockReg(reg_lock);
4052
38994053 try self.genSetStack(ret_ty, 0, operand, .{
39004054 .source_stack_base = .rbp,
39014055 .dest_stack_base = reg,
......@@ -3926,11 +4080,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
39264080 const elem_ty = ptr_ty.elemType();
39274081 switch (self.ret_mcv) {
39284082 .stack_offset => {
3929 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3930 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
39314083 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3932 self.register_manager.freezeRegs(&.{reg});
3933 defer self.register_manager.unfreezeRegs(&.{reg});
4084 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4085 defer self.register_manager.unlockReg(reg_lock);
4086
39344087 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
39354088 .source_stack_base = .rbp,
39364089 .dest_stack_base = reg,
......@@ -3980,12 +4133,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
39804133 // Source operand can be an immediate, 8 bits or 32 bits.
39814134 // TODO look into reusing the operand
39824135 const lhs = try self.resolveInst(bin_op.lhs);
3983 lhs.freezeIfRegister(&self.register_manager);
3984 defer lhs.unfreezeIfRegister(&self.register_manager);
4136 const lhs_lock: ?RegisterLock = switch (lhs) {
4137 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4138 else => null,
4139 };
4140 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
39854141
39864142 const dst_reg = try self.copyToTmpRegister(ty, lhs);
3987 self.register_manager.freezeRegs(&.{dst_reg});
3988 defer self.register_manager.unfreezeRegs(&.{dst_reg});
4143 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4144 defer self.register_manager.unlockReg(dst_reg_lock);
39894145
39904146 const dst_mcv = MCValue{ .register = dst_reg };
39914147
......@@ -4446,22 +4602,31 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
44464602
44474603fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
44484604 const un_op = self.air.instructions.items(.data)[inst].un_op;
4449 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4450 const operand_ptr = try self.resolveInst(un_op);
4451 operand_ptr.freezeIfRegister(&self.register_manager);
4452 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4453 const operand: MCValue = blk: {
4454 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4455 // The MCValue that holds the pointer can be re-used as the value.
4456 break :blk operand_ptr;
4457 } else {
4458 break :blk try self.allocRegOrMem(inst, true);
4459 }
4460 };
4461 const ptr_ty = self.air.typeOf(un_op);
4462 try self.load(operand, operand_ptr, ptr_ty);
4463 break :result try self.isNull(inst, ptr_ty.elemType(), operand);
4605
4606 if (self.liveness.isUnused(inst)) {
4607 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4608 }
4609
4610 const operand_ptr = try self.resolveInst(un_op);
4611 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4612 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4613 else => null,
4614 };
4615 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4616
4617 const operand: MCValue = blk: {
4618 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4619 // The MCValue that holds the pointer can be re-used as the value.
4620 break :blk operand_ptr;
4621 } else {
4622 break :blk try self.allocRegOrMem(inst, true);
4623 }
44644624 };
4625 const ptr_ty = self.air.typeOf(un_op);
4626 try self.load(operand, operand_ptr, ptr_ty);
4627
4628 const result = try self.isNull(inst, ptr_ty.elemType(), operand);
4629
44654630 return self.finishAir(inst, result, .{ un_op, .none, .none });
44664631}
44674632
......@@ -4477,22 +4642,31 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
44774642
44784643fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
44794644 const un_op = self.air.instructions.items(.data)[inst].un_op;
4480 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4481 const operand_ptr = try self.resolveInst(un_op);
4482 operand_ptr.freezeIfRegister(&self.register_manager);
4483 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4484 const operand: MCValue = blk: {
4485 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4486 // The MCValue that holds the pointer can be re-used as the value.
4487 break :blk operand_ptr;
4488 } else {
4489 break :blk try self.allocRegOrMem(inst, true);
4490 }
4491 };
4492 const ptr_ty = self.air.typeOf(un_op);
4493 try self.load(operand, operand_ptr, ptr_ty);
4494 break :result try self.isNonNull(inst, ptr_ty.elemType(), operand);
4645
4646 if (self.liveness.isUnused(inst)) {
4647 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4648 }
4649
4650 const operand_ptr = try self.resolveInst(un_op);
4651 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4652 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4653 else => null,
4654 };
4655 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4656
4657 const operand: MCValue = blk: {
4658 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4659 // The MCValue that holds the pointer can be re-used as the value.
4660 break :blk operand_ptr;
4661 } else {
4662 break :blk try self.allocRegOrMem(inst, true);
4663 }
44954664 };
4665 const ptr_ty = self.air.typeOf(un_op);
4666 try self.load(operand, operand_ptr, ptr_ty);
4667
4668 const result = try self.isNonNull(inst, ptr_ty.elemType(), operand);
4669
44964670 return self.finishAir(inst, result, .{ un_op, .none, .none });
44974671}
44984672
......@@ -4508,22 +4682,31 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
45084682
45094683fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
45104684 const un_op = self.air.instructions.items(.data)[inst].un_op;
4511 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4512 const operand_ptr = try self.resolveInst(un_op);
4513 operand_ptr.freezeIfRegister(&self.register_manager);
4514 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4515 const operand: MCValue = blk: {
4516 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4517 // The MCValue that holds the pointer can be re-used as the value.
4518 break :blk operand_ptr;
4519 } else {
4520 break :blk try self.allocRegOrMem(inst, true);
4521 }
4522 };
4523 const ptr_ty = self.air.typeOf(un_op);
4524 try self.load(operand, operand_ptr, ptr_ty);
4525 break :result try self.isErr(inst, ptr_ty.elemType(), operand);
4685
4686 if (self.liveness.isUnused(inst)) {
4687 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4688 }
4689
4690 const operand_ptr = try self.resolveInst(un_op);
4691 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4692 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4693 else => null,
45264694 };
4695 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4696
4697 const operand: MCValue = blk: {
4698 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4699 // The MCValue that holds the pointer can be re-used as the value.
4700 break :blk operand_ptr;
4701 } else {
4702 break :blk try self.allocRegOrMem(inst, true);
4703 }
4704 };
4705 const ptr_ty = self.air.typeOf(un_op);
4706 try self.load(operand, operand_ptr, ptr_ty);
4707
4708 const result = try self.isErr(inst, ptr_ty.elemType(), operand);
4709
45274710 return self.finishAir(inst, result, .{ un_op, .none, .none });
45284711}
45294712
......@@ -4539,22 +4722,31 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
45394722
45404723fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
45414724 const un_op = self.air.instructions.items(.data)[inst].un_op;
4542 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4543 const operand_ptr = try self.resolveInst(un_op);
4544 operand_ptr.freezeIfRegister(&self.register_manager);
4545 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4546 const operand: MCValue = blk: {
4547 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4548 // The MCValue that holds the pointer can be re-used as the value.
4549 break :blk operand_ptr;
4550 } else {
4551 break :blk try self.allocRegOrMem(inst, true);
4552 }
4553 };
4554 const ptr_ty = self.air.typeOf(un_op);
4555 try self.load(operand, operand_ptr, ptr_ty);
4556 break :result try self.isNonErr(inst, ptr_ty.elemType(), operand);
4725
4726 if (self.liveness.isUnused(inst)) {
4727 return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4728 }
4729
4730 const operand_ptr = try self.resolveInst(un_op);
4731 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4732 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
4733 else => null,
4734 };
4735 defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
4736
4737 const operand: MCValue = blk: {
4738 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4739 // The MCValue that holds the pointer can be re-used as the value.
4740 break :blk operand_ptr;
4741 } else {
4742 break :blk try self.allocRegOrMem(inst, true);
4743 }
45574744 };
4745 const ptr_ty = self.air.typeOf(un_op);
4746 try self.load(operand, operand_ptr, ptr_ty);
4747
4748 const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);
4749
45584750 return self.finishAir(inst, result, .{ un_op, .none, .none });
45594751}
45604752
......@@ -4610,8 +4802,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
46104802 .register => |cond_reg| {
46114803 try self.spillCompareFlagsIfOccupied();
46124804
4613 self.register_manager.freezeRegs(&.{cond_reg});
4614 defer self.register_manager.unfreezeRegs(&.{cond_reg});
4805 const cond_reg_lock = self.register_manager.lockReg(cond_reg);
4806 defer if (cond_reg_lock) |lock| self.register_manager.unlockReg(lock);
46154807
46164808 switch (case) {
46174809 .none => unreachable,
......@@ -4670,8 +4862,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
46704862
46714863 if (abi_size <= 8) {
46724864 const reg = try self.copyToTmpRegister(ty, condition);
4673 self.register_manager.freezeRegs(&.{reg});
4674 defer self.register_manager.unfreezeRegs(&.{reg});
4865 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4866 defer self.register_manager.unlockReg(reg_lock);
46754867 return self.genCondSwitchMir(ty, .{ .register = reg }, case);
46764868 }
46774869
......@@ -5158,8 +5350,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
51585350 .register_overflow_unsigned,
51595351 .register_overflow_signed,
51605352 => |reg| {
5161 self.register_manager.freezeRegs(&.{reg});
5162 defer self.register_manager.unfreezeRegs(&.{reg});
5353 const reg_lock = self.register_manager.lockReg(reg);
5354 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
51635355
51645356 const wrapped_ty = ty.structFieldType(0);
51655357 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg }, .{});
......@@ -5260,8 +5452,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
52605452
52615453 const base_reg = opts.dest_stack_base orelse .rbp;
52625454 if (!math.isPowerOfTwo(abi_size)) {
5263 self.register_manager.freezeRegs(&.{reg});
5264 defer self.register_manager.unfreezeRegs(&.{reg});
5455 const reg_lock = self.register_manager.lockReg(reg);
5456 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
52655457
52665458 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
52675459
......@@ -5350,13 +5542,25 @@ fn genInlineMemcpy(
53505542 len: MCValue,
53515543 opts: InlineMemcpyOpts,
53525544) InnerError!void {
5353 self.register_manager.freezeRegs(&.{ .rax, .rcx });
5545 try self.register_manager.getReg(.rax, null);
5546 try self.register_manager.getReg(.rcx, null);
53545547
5355 if (opts.source_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
5356 defer if (opts.source_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
5548 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rcx });
5549 defer for (reg_locks) |lock| {
5550 self.register_manager.unlockReg(lock);
5551 };
5552
5553 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
5554 self.register_manager.lockReg(reg)
5555 else
5556 null;
5557 defer if (ssbase_lock) |reg| self.register_manager.unlockReg(reg);
53575558
5358 if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
5359 defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
5559 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
5560 self.register_manager.lockReg(reg)
5561 else
5562 null;
5563 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
53605564
53615565 const dst_addr_reg = try self.register_manager.allocReg(null);
53625566 switch (dst_ptr) {
......@@ -5390,8 +5594,8 @@ fn genInlineMemcpy(
53905594 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
53915595 },
53925596 }
5393 self.register_manager.freezeRegs(&.{dst_addr_reg});
5394 defer self.register_manager.unfreezeRegs(&.{dst_addr_reg});
5597 const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5598 defer self.register_manager.unlockReg(dst_addr_reg_lock);
53955599
53965600 const src_addr_reg = try self.register_manager.allocReg(null);
53975601 switch (src_ptr) {
......@@ -5425,18 +5629,13 @@ fn genInlineMemcpy(
54255629 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
54265630 },
54275631 }
5428 self.register_manager.freezeRegs(&.{src_addr_reg});
5429 defer self.register_manager.unfreezeRegs(&.{src_addr_reg});
5632 const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5633 defer self.register_manager.unlockReg(src_addr_reg_lock);
54305634
54315635 const regs = try self.register_manager.allocRegs(2, .{ null, null });
54325636 const count_reg = regs[0].to64();
54335637 const tmp_reg = regs[1].to8();
54345638
5435 self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
5436
5437 try self.register_manager.getReg(.rax, null);
5438 try self.register_manager.getReg(.rcx, null);
5439
54405639 try self.genSetReg(Type.usize, count_reg, len);
54415640
54425641 // mov rcx, 0
......@@ -5540,7 +5739,9 @@ fn genInlineMemset(
55405739 len: MCValue,
55415740 opts: InlineMemcpyOpts,
55425741) InnerError!void {
5543 self.register_manager.freezeRegs(&.{.rax});
5742 try self.register_manager.getReg(.rax, null);
5743 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
5744 defer self.register_manager.unlockReg(rax_lock);
55445745
55455746 const addr_reg = try self.register_manager.allocReg(null);
55465747 switch (dst_ptr) {
......@@ -5574,11 +5775,8 @@ fn genInlineMemset(
55745775 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
55755776 },
55765777 }
5577 self.register_manager.freezeRegs(&.{addr_reg});
5578 defer self.register_manager.unfreezeRegs(&.{addr_reg});
5579
5580 self.register_manager.unfreezeRegs(&.{.rax});
5581 try self.register_manager.getReg(.rax, null);
5778 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
5779 defer self.register_manager.unlockReg(addr_reg_lock);
55825780
55835781 try self.genSetReg(Type.usize, .rax, len);
55845782 try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });
......@@ -6017,16 +6215,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
60176215 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
60186216
60196217 const dst_ptr = try self.resolveInst(pl_op.operand);
6020 dst_ptr.freezeIfRegister(&self.register_manager);
6021 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
6218 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
6219 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6220 else => null,
6221 };
6222 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
60226223
60236224 const src_val = try self.resolveInst(extra.lhs);
6024 src_val.freezeIfRegister(&self.register_manager);
6025 defer src_val.unfreezeIfRegister(&self.register_manager);
6225 const src_val_lock: ?RegisterLock = switch (src_val) {
6226 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6227 else => null,
6228 };
6229 defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock);
60266230
60276231 const len = try self.resolveInst(extra.rhs);
6028 len.freezeIfRegister(&self.register_manager);
6029 defer len.unfreezeIfRegister(&self.register_manager);
6232 const len_lock: ?RegisterLock = switch (len) {
6233 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6234 else => null,
6235 };
6236 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
60306237
60316238 try self.genInlineMemset(dst_ptr, src_val, len, .{});
60326239
......@@ -6038,17 +6245,26 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
60386245 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
60396246
60406247 const dst_ptr = try self.resolveInst(pl_op.operand);
6041 dst_ptr.freezeIfRegister(&self.register_manager);
6042 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
6248 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
6249 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6250 else => null,
6251 };
6252 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
60436253
60446254 const src_ty = self.air.typeOf(extra.lhs);
60456255 const src_ptr = try self.resolveInst(extra.lhs);
6046 src_ptr.freezeIfRegister(&self.register_manager);
6047 defer src_ptr.unfreezeIfRegister(&self.register_manager);
6256 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
6257 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6258 else => null,
6259 };
6260 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);
60486261
60496262 const len = try self.resolveInst(extra.rhs);
6050 len.freezeIfRegister(&self.register_manager);
6051 defer len.unfreezeIfRegister(&self.register_manager);
6263 const len_lock: ?RegisterLock = switch (len) {
6264 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
6265 else => null,
6266 };
6267 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
60526268
60536269 // TODO Is this the only condition for pointer dereference for memcpy?
60546270 const src: MCValue = blk: {
......@@ -6070,8 +6286,11 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
60706286 else => break :blk src_ptr,
60716287 }
60726288 };
6073 src.freezeIfRegister(&self.register_manager);
6074 defer src.unfreezeIfRegister(&self.register_manager);
6289 const src_lock: ?RegisterLock = switch (src) {
6290 .register => |reg| self.register_manager.lockReg(reg),
6291 else => null,
6292 };
6293 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
60756294
60766295 try self.genInlineMemcpy(dst_ptr, src, len, .{});
60776296
src/register_manager.zig+79-46
......@@ -45,9 +45,8 @@ pub fn RegisterManager(
4545 /// Tracks all registers allocated in the course of this
4646 /// function
4747 allocated_registers: FreeRegInt = 0,
48 /// Tracks registers which are temporarily blocked from being
49 /// allocated
50 frozen_registers: FreeRegInt = 0,
48 /// Tracks registers which are locked from being allocated
49 locked_registers: FreeRegInt = 0,
5150
5251 const Self = @This();
5352
......@@ -108,34 +107,70 @@ pub fn RegisterManager(
108107 return self.allocated_registers & mask != 0;
109108 }
110109
111 /// Returns whether this register is frozen
110 /// Returns whether this register is locked
112111 ///
113112 /// Returns false when this register is not tracked
114 pub fn isRegFrozen(self: Self, reg: Register) bool {
113 pub fn isRegLocked(self: Self, reg: Register) bool {
115114 const mask = getRegisterMask(reg) orelse return false;
116 return self.frozen_registers & mask != 0;
115 return self.locked_registers & mask != 0;
117116 }
118117
119 /// Prevents the registers from being allocated until they are
120 /// unfrozen again
121 pub fn freezeRegs(self: *Self, regs: []const Register) void {
122 for (regs) |reg| {
123 const mask = getRegisterMask(reg) orelse continue;
124 self.frozen_registers |= mask;
118 pub const RegisterLock = struct {
119 register: Register,
120 };
121
122 /// Prevents the register from being allocated until they are
123 /// unlocked again.
124 /// Returns `RegisterLock` if the register was not already
125 /// locked, or `null` otherwise.
126 /// Only the owner of the `RegisterLock` can unlock the
127 /// register later.
128 pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
129 log.debug("locking {}", .{reg});
130 if (self.isRegLocked(reg)) {
131 log.debug(" register already locked", .{});
132 return null;
125133 }
134 const mask = getRegisterMask(reg) orelse return null;
135 self.locked_registers |= mask;
136 return RegisterLock{ .register = reg };
126137 }
127138
128 /// Enables the allocation of the registers
129 pub fn unfreezeRegs(self: *Self, regs: []const Register) void {
130 for (regs) |reg| {
131 const mask = getRegisterMask(reg) orelse continue;
132 self.frozen_registers &= ~mask;
139 /// Like `lockReg` but asserts the register was unused always
140 /// returning a valid lock.
141 pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
142 log.debug("locking asserting free {}", .{reg});
143 assert(!self.isRegLocked(reg));
144 const mask = getRegisterMask(reg) orelse unreachable;
145 self.locked_registers |= mask;
146 return RegisterLock{ .register = reg };
147 }
148
149 /// Like `lockRegAssumeUnused` but locks multiple registers.
150 pub fn lockRegsAssumeUnused(
151 self: *Self,
152 comptime count: comptime_int,
153 regs: [count]Register,
154 ) [count]RegisterLock {
155 var buf: [count]RegisterLock = undefined;
156 for (regs) |reg, i| {
157 buf[i] = self.lockRegAssumeUnused(reg);
133158 }
159 return buf;
160 }
161
162 /// Unlocks the register allowing its re-allocation and re-use.
163 /// Requires `RegisterLock` to unlock a register.
164 /// Call `lockReg` to obtain the lock first.
165 pub fn unlockReg(self: *Self, lock: RegisterLock) void {
166 log.debug("unlocking {}", .{lock.register});
167 const mask = getRegisterMask(lock.register) orelse return;
168 self.locked_registers &= ~mask;
134169 }
135170
136 /// Returns true when at least one register is frozen
137 pub fn frozenRegsExist(self: Self) bool {
138 return self.frozen_registers != 0;
171 /// Returns true when at least one register is locked
172 pub fn lockedRegsExist(self: Self) bool {
173 return self.locked_registers != 0;
139174 }
140175
141176 /// Allocates a specified number of registers, optionally
......@@ -148,15 +183,15 @@ pub fn RegisterManager(
148183 ) ?[count]Register {
149184 comptime assert(count > 0 and count <= tracked_registers.len);
150185
151 const free_and_not_frozen_registers = self.free_registers & ~self.frozen_registers;
152 const free_and_not_frozen_registers_count = @popCount(FreeRegInt, free_and_not_frozen_registers);
153 if (free_and_not_frozen_registers_count < count) return null;
186 const free_and_not_locked_registers = self.free_registers & ~self.locked_registers;
187 const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers);
188 if (free_and_not_locked_registers_count < count) return null;
154189
155190 var regs: [count]Register = undefined;
156191 var i: usize = 0;
157192 for (tracked_registers) |reg| {
158193 if (i >= count) break;
159 if (self.isRegFrozen(reg)) continue;
194 if (self.isRegLocked(reg)) continue;
160195 if (!self.isRegFree(reg)) continue;
161196
162197 regs[i] = reg;
......@@ -194,8 +229,8 @@ pub fn RegisterManager(
194229 insts: [count]?Air.Inst.Index,
195230 ) AllocateRegistersError![count]Register {
196231 comptime assert(count > 0 and count <= tracked_registers.len);
197 const frozen_registers_count = @popCount(FreeRegInt, self.frozen_registers);
198 if (count > tracked_registers.len - frozen_registers_count) return error.OutOfRegisters;
232 const locked_registers_count = @popCount(FreeRegInt, self.locked_registers);
233 if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters;
199234
200235 const result = self.tryAllocRegs(count, insts) orelse blk: {
201236 // We'll take over the first count registers. Spill
......@@ -205,7 +240,7 @@ pub fn RegisterManager(
205240 var i: usize = 0;
206241 for (tracked_registers) |reg| {
207242 if (i >= count) break;
208 if (self.isRegFrozen(reg)) continue;
243 if (self.isRegLocked(reg)) continue;
209244
210245 regs[i] = reg;
211246 self.markRegAllocated(reg);
......@@ -416,15 +451,15 @@ test "allocReg: spilling" {
416451 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
417452 try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
418453
419 // Frozen registers
454 // Locked registers
420455 function.register_manager.freeReg(.r3);
421456 {
422 function.register_manager.freezeRegs(&.{.r2});
423 defer function.register_manager.unfreezeRegs(&.{.r2});
457 const lock = function.register_manager.lockReg(.r2);
458 defer if (lock) |reg| function.register_manager.unlockReg(reg);
424459
425460 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
426461 }
427 try expect(!function.register_manager.frozenRegsExist());
462 try expect(!function.register_manager.lockedRegsExist());
428463}
429464
430465test "tryAllocRegs" {
......@@ -442,17 +477,17 @@ test "tryAllocRegs" {
442477 try expect(function.register_manager.isRegAllocated(.r2));
443478 try expect(!function.register_manager.isRegAllocated(.r3));
444479
445 // Frozen registers
480 // Locked registers
446481 function.register_manager.freeReg(.r0);
447482 function.register_manager.freeReg(.r2);
448483 function.register_manager.freeReg(.r3);
449484 {
450 function.register_manager.freezeRegs(&.{.r1});
451 defer function.register_manager.unfreezeRegs(&.{.r1});
485 const lock = function.register_manager.lockReg(.r1);
486 defer if (lock) |reg| function.register_manager.unlockReg(reg);
452487
453488 try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
454489 }
455 try expect(!function.register_manager.frozenRegsExist());
490 try expect(!function.register_manager.lockedRegsExist());
456491
457492 try expect(function.register_manager.isRegAllocated(.r0));
458493 try expect(function.register_manager.isRegAllocated(.r1));
......@@ -475,19 +510,19 @@ test "allocRegs: normal usage" {
475510
476511 // The result register is known and fixed at this point, we
477512 // don't want to accidentally allocate lhs or rhs to the
478 // result register, this is why we freeze it.
513 // result register, this is why we lock it.
479514 //
480 // Using defer unfreeze right after freeze is a good idea in
481 // most cases as you probably are using the frozen registers
515 // Using defer unlock right after lock is a good idea in
516 // most cases as you probably are using the locked registers
482517 // in the remainder of this scope and don't need to use it
483518 // after the end of this scope. However, in some situations,
484 // it may make sense to manually unfreeze registers before the
519 // it may make sense to manually unlock registers before the
485520 // end of the scope when you are certain that they don't
486521 // contain any valuable data anymore and can be reused. For an
487522 // example of that, see `selectively reducing register
488523 // pressure`.
489 function.register_manager.freezeRegs(&.{result_reg});
490 defer function.register_manager.unfreezeRegs(&.{result_reg});
524 const lock = function.register_manager.lockReg(result_reg);
525 defer if (lock) |reg| function.register_manager.unlockReg(reg);
491526
492527 const regs = try function.register_manager.allocRegs(2, .{ null, null });
493528 try function.genAdd(result_reg, regs[0], regs[1]);
......@@ -507,16 +542,14 @@ test "allocRegs: selectively reducing register pressure" {
507542 {
508543 const result_reg: MockRegister2 = .r1;
509544
510 function.register_manager.freezeRegs(&.{result_reg});
511 defer function.register_manager.unfreezeRegs(&.{result_reg});
545 const lock = function.register_manager.lockReg(result_reg);
512546
513 // Here, we don't defer unfreeze because we manually unfreeze
547 // Here, we don't defer unlock because we manually unlock
514548 // after genAdd
515549 const regs = try function.register_manager.allocRegs(2, .{ null, null });
516 function.register_manager.freezeRegs(&.{result_reg});
517550
518551 try function.genAdd(result_reg, regs[0], regs[1]);
519 function.register_manager.unfreezeRegs(&regs);
552 function.register_manager.unlockReg(lock.?);
520553
521554 const extra_summand_reg = try function.register_manager.allocReg(null);
522555 try function.genAdd(result_reg, result_reg, extra_summand_reg);
test/behavior/align.zig+2
......@@ -7,6 +7,8 @@ var foo: u8 align(4) = 100;
77
88test "global variable alignment" {
99 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
11 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1012
1113 comptime try expect(@typeInfo(@TypeOf(&foo)).Pointer.alignment == 4);
1214 comptime try expect(@TypeOf(&foo) == *align(4) u8);
test/behavior/byval_arg_var.zig+1
......@@ -6,6 +6,7 @@ var result: []const u8 = "wrong";
66test "pass string literal byvalue to a generic var param" {
77 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
88 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
910
1011 start();
1112 blowUpStack(10);