authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-07 00:57:55+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-07 00:57:55+02:00
logac954eb539ec242af568fe3565094cea12c4beea
tree388b3c11137f23393ff877699f86df4b43666501
parentac1aaec9c38eb44b93099ff18579a9401f107100

regalloc: ensure we only freeze/unfreeze at the outermost scope

This prevents a nasty type of bugs where we accidentally unfreeze a register that was frozen purposely in the outer scope, risking accidental realloc of a taken register. Fix CF flags spilling on aarch64 backend.

6 files changed, 735 insertions(+), 413 deletions(-)

src/arch/aarch64/CodeGen.zig+152-67
......@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
2525const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2627
2728const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
2829const FnResult = @import("../../codegen.zig").FnResult;
......@@ -910,16 +911,16 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
910911fn spillCompareFlagsIfOccupied(self: *Self) !void {
911912 if (self.compare_flags_inst) |inst_to_save| {
912913 const mcv = self.getResolvedInstValue(inst_to_save);
913 switch (mcv) {
914 const new_mcv = switch (mcv) {
914915 .compare_flags_signed,
915916 .compare_flags_unsigned,
917 => try self.allocRegOrMem(inst_to_save, true),
916918 .register_c_flag,
917919 .register_v_flag,
918 => {},
920 => try self.allocRegOrMem(inst_to_save, false),
919921 else => unreachable, // mcv doesn't occupy the compare flags
920 }
922 };
921923
922 const new_mcv = try self.allocRegOrMem(inst_to_save, true);
923924 try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
924925 log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
925926
......@@ -927,6 +928,15 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
927928 try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
928929
929930 self.compare_flags_inst = null;
931
932 // TODO consolidate with register manager and spillInstruction
933 // this call should really belong in the register manager!
934 switch (mcv) {
935 .register_c_flag,
936 .register_v_flag,
937 => |reg| self.register_manager.freeReg(reg),
938 else => {},
939 }
930940 }
931941}
932942
......@@ -1048,8 +1058,8 @@ fn trunc(
10481058 }
10491059 },
10501060 };
1051 self.register_manager.freezeRegs(&.{operand_reg});
1052 defer self.register_manager.unfreezeRegs(&.{operand_reg});
1061 const lock = self.register_manager.freezeReg(operand_reg);
1062 defer if (lock) |reg| self.register_manager.unfreezeReg(reg);
10531063
10541064 const dest_reg = if (maybe_inst) |inst| blk: {
10551065 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
......@@ -1135,8 +1145,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11351145 .register => |r| r,
11361146 else => try self.copyToTmpRegister(operand_ty, operand),
11371147 };
1138 self.register_manager.freezeRegs(&.{op_reg});
1139 defer self.register_manager.unfreezeRegs(&.{op_reg});
1148 const reg_lock = self.register_manager.freezeRegAssumeUnused(op_reg);
1149 defer self.register_manager.unfreezeReg(reg_lock);
11401150
11411151 const dest_reg = blk: {
11421152 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1168,8 +1178,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11681178 .register => |r| r,
11691179 else => try self.copyToTmpRegister(operand_ty, operand),
11701180 };
1171 self.register_manager.freezeRegs(&.{op_reg});
1172 defer self.register_manager.unfreezeRegs(&.{op_reg});
1181 const reg_lock = self.register_manager.freezeRegAssumeUnused(op_reg);
1182 defer self.register_manager.unfreezeReg(reg_lock);
11731183
11741184 const dest_reg = blk: {
11751185 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1257,8 +1267,17 @@ fn binOpRegister(
12571267 const lhs_is_register = lhs == .register;
12581268 const rhs_is_register = rhs == .register;
12591269
1260 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1261 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
1270 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1271 self.register_manager.freezeReg(lhs.register)
1272 else
1273 null;
1274 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
1275
1276 const rhs_lock: ?RegisterLock = if (rhs_is_register)
1277 self.register_manager.freezeReg(rhs.register)
1278 else
1279 null;
1280 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
12621281
12631282 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
12641283
......@@ -1270,13 +1289,13 @@ fn binOpRegister(
12701289
12711290 const raw_reg = try self.register_manager.allocReg(track_inst);
12721291 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1273 self.register_manager.freezeRegs(&.{reg});
12741292
12751293 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
12761294
12771295 break :blk reg;
12781296 };
1279 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1297 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
1298 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
12801299
12811300 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
12821301 const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
......@@ -1286,13 +1305,13 @@ fn binOpRegister(
12861305
12871306 const raw_reg = try self.register_manager.allocReg(track_inst);
12881307 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1289 self.register_manager.freezeRegs(&.{reg});
12901308
12911309 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
12921310
12931311 break :blk reg;
12941312 };
1295 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1313 const new_rhs_lock = self.register_manager.freezeReg(rhs_reg);
1314 defer if (new_rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
12961315
12971316 const dest_reg = switch (mir_tag) {
12981317 .cmp_shifted_register => undefined, // cmp has no destination register
......@@ -1394,7 +1413,11 @@ fn binOpImmediate(
13941413) !MCValue {
13951414 const lhs_is_register = lhs == .register;
13961415
1397 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1416 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1417 self.register_manager.freezeReg(lhs.register)
1418 else
1419 null;
1420 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
13981421
13991422 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
14001423
......@@ -1408,13 +1431,13 @@ fn binOpImmediate(
14081431
14091432 const raw_reg = try self.register_manager.allocReg(track_inst);
14101433 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1411 self.register_manager.freezeRegs(&.{reg});
14121434
14131435 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
14141436
14151437 break :blk reg;
14161438 };
1417 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1439 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
1440 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
14181441
14191442 const dest_reg = switch (mir_tag) {
14201443 .cmp_immediate => undefined, // cmp has no destination register
......@@ -1758,7 +1781,10 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void {
17581781 const lhs_ty = self.air.typeOf(bin_op.lhs);
17591782 const rhs_ty = self.air.typeOf(bin_op.rhs);
17601783
1761 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
1784 const result: MCValue = if (self.liveness.isUnused(inst))
1785 .dead
1786 else
1787 try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
17621788 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
17631789}
17641790
......@@ -1815,13 +1841,13 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
18151841 };
18161842 const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
18171843 const dest_reg = dest.register;
1818 self.register_manager.freezeRegs(&.{dest_reg});
1819 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1844 const dest_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_reg);
1845 defer self.register_manager.unfreezeReg(dest_reg_lock);
18201846
18211847 const raw_truncated_reg = try self.register_manager.allocReg(null);
18221848 const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*));
1823 self.register_manager.freezeRegs(&.{truncated_reg});
1824 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1849 const truncated_reg_lock = self.register_manager.freezeRegAssumeUnused(truncated_reg);
1850 defer self.register_manager.unfreezeReg(truncated_reg_lock);
18251851
18261852 // sbfx/ubfx truncated, dest, #0, #bits
18271853 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
......@@ -1922,12 +1948,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
19221948
19231949 const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
19241950 const dest_reg = dest.register;
1925 self.register_manager.freezeRegs(&.{dest_reg});
1926 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1951 const dest_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_reg);
1952 defer self.register_manager.unfreezeReg(dest_reg_lock);
19271953
19281954 const truncated_reg = try self.register_manager.allocReg(null);
1929 self.register_manager.freezeRegs(&.{truncated_reg});
1930 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1955 const truncated_reg_lock = self.register_manager.freezeRegAssumeUnused(truncated_reg);
1956 defer self.register_manager.unfreezeReg(truncated_reg_lock);
19311957
19321958 try self.truncRegister(
19331959 dest_reg.to32(),
......@@ -1977,36 +2003,44 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
19772003 const lhs_is_register = lhs == .register;
19782004 const rhs_is_register = rhs == .register;
19792005
1980 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1981 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
2006 const lhs_lock: ?RegisterLock = if (lhs_is_register)
2007 self.register_manager.freezeRegAssumeUnused(lhs.register)
2008 else
2009 null;
2010 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
2011
2012 const rhs_lock: ?RegisterLock = if (rhs_is_register)
2013 self.register_manager.freezeRegAssumeUnused(rhs.register)
2014 else
2015 null;
2016 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
19822017
19832018 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
19842019 const raw_reg = try self.register_manager.allocReg(null);
19852020 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1986 self.register_manager.freezeRegs(&.{reg});
19872021 break :blk reg;
19882022 };
1989 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
2023 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
2024 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
19902025
19912026 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
19922027 const raw_reg = try self.register_manager.allocReg(null);
19932028 const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
1994 self.register_manager.freezeRegs(&.{reg});
19952029 break :blk reg;
19962030 };
1997 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
2031 const new_rhs_lock = self.register_manager.freezeReg(rhs_reg);
2032 defer if (new_rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
19982033
19992034 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
20002035 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
20012036
2002 // TODO reuse operands
20032037 const dest_reg = blk: {
20042038 const raw_reg = try self.register_manager.allocReg(null);
20052039 const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
2006 self.register_manager.freezeRegs(&.{reg});
20072040 break :blk reg;
20082041 };
2009 defer self.register_manager.unfreezeRegs(&.{dest_reg});
2042 const dest_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_reg);
2043 defer self.register_manager.unfreezeReg(dest_reg_lock);
20102044
20112045 switch (int_info.signedness) {
20122046 .signed => {
......@@ -2021,8 +2055,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20212055 });
20222056
20232057 const dest_high_reg = try self.register_manager.allocReg(null);
2024 self.register_manager.freezeRegs(&.{dest_high_reg});
2025 defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
2058 const dest_high_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_high_reg);
2059 defer self.register_manager.unfreezeReg(dest_high_reg_lock);
20262060
20272061 // smulh dest_high, lhs, rhs
20282062 _ = try self.addInst(.{
......@@ -2071,8 +2105,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20712105 },
20722106 .unsigned => {
20732107 const dest_high_reg = try self.register_manager.allocReg(null);
2074 self.register_manager.freezeRegs(&.{dest_high_reg});
2075 defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
2108 const dest_high_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_high_reg);
2109 defer self.register_manager.unfreezeReg(dest_high_reg_lock);
20762110
20772111 // umulh dest_high, lhs, rhs
20782112 _ = try self.addInst(.{
......@@ -2127,8 +2161,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21272161 }
21282162
21292163 const truncated_reg = try self.register_manager.allocReg(null);
2130 self.register_manager.freezeRegs(&.{truncated_reg});
2131 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
2164 const truncated_reg_lock = self.register_manager.freezeRegAssumeUnused(truncated_reg);
2165 defer self.register_manager.unfreezeReg(truncated_reg_lock);
21322166
21332167 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
21342168
......@@ -2168,14 +2202,20 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21682202 if (int_info.bits <= 64) {
21692203 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
21702204
2171 if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});
2172 defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});
2205 const lhs_lock: ?RegisterLock = if (lhs == .register)
2206 self.register_manager.freezeRegAssumeUnused(lhs.register)
2207 else
2208 null;
2209 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
21732210
21742211 try self.spillCompareFlagsIfOccupied();
21752212 self.compare_flags_inst = null;
21762213
21772214 // lsl dest, lhs, rhs
21782215 const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
2216 const dest_reg = dest.register;
2217 const dest_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_reg);
2218 defer self.register_manager.unfreezeReg(dest_reg_lock);
21792219
21802220 // asr/lsr reconstructed, dest, rhs
21812221 const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);
......@@ -2184,7 +2224,9 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21842224 _ = try self.binOp(.cmp_eq, null, lhs, reconstructed, lhs_ty, lhs_ty);
21852225
21862226 try self.genSetStack(lhs_ty, stack_offset, dest);
2187 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags_unsigned = .neq });
2227 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
2228 .compare_flags_unsigned = .neq,
2229 });
21882230
21892231 break :result MCValue{ .stack_offset = stack_offset };
21902232 } else {
......@@ -2411,14 +2453,18 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
24112453 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
24122454 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
24132455
2414 if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register});
2415 defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register});
2456 const index_lock: ?RegisterLock = if (index_is_register)
2457 self.register_manager.freezeRegAssumeUnused(index_mcv.register)
2458 else
2459 null;
2460 defer if (index_lock) |reg| self.register_manager.unfreezeReg(reg);
24162461
24172462 const base_mcv: MCValue = switch (slice_mcv) {
24182463 .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
24192464 else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
24202465 };
2421 self.register_manager.freezeRegs(&.{base_mcv.register});
2466 const base_lock = self.register_manager.freezeRegAssumeUnused(base_mcv.register);
2467 defer self.register_manager.unfreezeReg(base_lock);
24222468
24232469 switch (elem_size) {
24242470 else => {
......@@ -2559,8 +2605,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
25592605 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
25602606 .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
25612607 .register => |addr_reg| {
2562 self.register_manager.freezeRegs(&.{addr_reg});
2563 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2608 const addr_reg_lock = self.register_manager.freezeReg(addr_reg);
2609 defer if (addr_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
25642610
25652611 switch (dst_mcv) {
25662612 .dead => unreachable,
......@@ -2573,16 +2619,19 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
25732619 if (elem_size <= 8) {
25742620 const raw_tmp_reg = try self.register_manager.allocReg(null);
25752621 const tmp_reg = registerAlias(raw_tmp_reg, elem_size);
2576 self.register_manager.freezeRegs(&.{tmp_reg});
2577 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2622 const tmp_reg_lock = self.register_manager.freezeRegAssumeUnused(tmp_reg);
2623 defer self.register_manager.unfreezeReg(tmp_reg_lock);
25782624
25792625 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
25802626 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
25812627 } else {
25822628 // TODO optimize the register allocation
25832629 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2584 self.register_manager.freezeRegs(&regs);
2585 defer self.register_manager.unfreezeRegs(&regs);
2630 var regs_locks: [4]RegisterLock = undefined;
2631 self.register_manager.freezeRegsAssumeUnused(4, regs, &regs_locks);
2632 defer for (regs_locks) |reg| {
2633 self.register_manager.unfreezeReg(reg);
2634 };
25862635
25872636 const src_reg = addr_reg;
25882637 const dst_reg = regs[0];
......@@ -2784,8 +2833,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27842833 try self.genSetStack(value_ty, off, value);
27852834 },
27862835 .register => |addr_reg| {
2787 self.register_manager.freezeRegs(&.{addr_reg});
2788 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2836 const addr_reg_lock = self.register_manager.freezeReg(addr_reg);
2837 defer if (addr_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
27892838
27902839 switch (value) {
27912840 .register => |value_reg| {
......@@ -2795,8 +2844,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
27952844 if (abi_size <= 8) {
27962845 const raw_tmp_reg = try self.register_manager.allocReg(null);
27972846 const tmp_reg = registerAlias(raw_tmp_reg, abi_size);
2798 self.register_manager.freezeRegs(&.{tmp_reg});
2799 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2847 const tmp_reg_lock = self.register_manager.freezeRegAssumeUnused(tmp_reg);
2848 defer self.register_manager.unfreezeReg(tmp_reg_lock);
28002849
28012850 try self.genSetReg(value_ty, tmp_reg, value);
28022851 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
......@@ -2856,12 +2905,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
28562905 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
28572906 .immediate = struct_field_offset,
28582907 });
2859 self.register_manager.freezeRegs(&.{offset_reg});
2860 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2908 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
2909 defer self.register_manager.unfreezeReg(offset_reg_lock);
28612910
28622911 const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
2863 self.register_manager.freezeRegs(&.{addr_reg});
2864 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2912 const addr_reg_lock = self.register_manager.freezeRegAssumeUnused(addr_reg);
2913 defer self.register_manager.unfreezeReg(addr_reg_lock);
28652914
28662915 const dest = try self.binOp(
28672916 .add,
......@@ -3369,6 +3418,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
33693418 const parent_compare_flags_inst = self.compare_flags_inst;
33703419
33713420 try self.branch_stack.append(.{});
3421 errdefer {
3422 _ = self.branch_stack.pop();
3423 }
33723424
33733425 try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
33743426 for (liveness_condbr.then_deaths) |operand| {
......@@ -3955,8 +4007,38 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
39554007 },
39564008 .register_c_flag,
39574009 .register_v_flag,
3958 => {
3959 return self.fail("TODO implement genSetStack {}", .{mcv});
4010 => |reg| {
4011 const reg_lock = self.register_manager.freezeReg(reg);
4012 defer if (reg_lock) |locked_reg| self.register_manager.unfreezeReg(locked_reg);
4013
4014 const wrapped_ty = ty.structFieldType(0);
4015 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg });
4016
4017 const overflow_bit_ty = ty.structFieldType(1);
4018 const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
4019 const raw_cond_reg = try self.register_manager.allocReg(null);
4020 const cond_reg = registerAlias(
4021 raw_cond_reg,
4022 @intCast(u32, overflow_bit_ty.abiSize(self.target.*)),
4023 );
4024
4025 // C flag: cset reg, cs
4026 // V flag: cset reg, vs
4027 _ = try self.addInst(.{
4028 .tag = .cset,
4029 .data = .{ .r_cond = .{
4030 .rd = cond_reg,
4031 .cond = switch (mcv) {
4032 .register_c_flag => .cs,
4033 .register_v_flag => .vs,
4034 else => unreachable,
4035 },
4036 } },
4037 });
4038
4039 try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
4040 .register = cond_reg,
4041 });
39604042 },
39614043 .got_load,
39624044 .direct_load,
......@@ -3983,8 +4065,11 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
39834065
39844066 // TODO call extern memcpy
39854067 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null });
3986 self.register_manager.freezeRegs(&regs);
3987 defer self.register_manager.unfreezeRegs(&regs);
4068 var regs_locks: [5]RegisterLock = undefined;
4069 self.register_manager.freezeRegsAssumeUnused(5, regs, &regs_locks);
4070 defer for (regs_locks) |reg| {
4071 self.register_manager.unfreezeReg(reg);
4072 };
39884073
39894074 const src_reg = regs[0];
39904075 const dst_reg = regs[1];
src/arch/arm/CodeGen.zig+99-75
......@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
2525const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26const RegisterLock = RegisterManager.RegisterLock;
2627
2728const FnResult = @import("../../codegen.zig").FnResult;
2829const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
......@@ -1038,8 +1039,8 @@ fn trunc(
10381039 }
10391040 },
10401041 };
1041 self.register_manager.freezeRegs(&.{operand_reg});
1042 defer self.register_manager.unfreezeRegs(&.{operand_reg});
1042 const operand_reg_lock = self.register_manager.freezeReg(operand_reg);
1043 defer if (operand_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
10431044
10441045 const dest_reg = if (maybe_inst) |inst| blk: {
10451046 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
......@@ -1127,8 +1128,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11271128 .register => |r| r,
11281129 else => try self.copyToTmpRegister(operand_ty, operand),
11291130 };
1130 self.register_manager.freezeRegs(&.{op_reg});
1131 defer self.register_manager.unfreezeRegs(&.{op_reg});
1131 const op_reg_lock = self.register_manager.freezeRegAssumeUnused(op_reg);
1132 defer self.register_manager.unfreezeReg(op_reg_lock);
11321133
11331134 const dest_reg = blk: {
11341135 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1157,8 +1158,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
11571158 .register => |r| r,
11581159 else => try self.copyToTmpRegister(operand_ty, operand),
11591160 };
1160 self.register_manager.freezeRegs(&.{op_reg});
1161 defer self.register_manager.unfreezeRegs(&.{op_reg});
1161 const op_reg_lock = self.register_manager.freezeRegAssumeUnused(op_reg);
1162 defer self.register_manager.unfreezeReg(op_reg_lock);
11621163
11631164 const dest_reg = blk: {
11641165 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
......@@ -1218,15 +1219,15 @@ fn minMax(
12181219 .register => |r| r,
12191220 else => try self.copyToTmpRegister(lhs_ty, lhs),
12201221 };
1221 self.register_manager.freezeRegs(&.{lhs_reg});
1222 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1222 const lhs_reg_lock = self.register_manager.freezeReg(lhs_reg);
1223 defer if (lhs_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
12231224
12241225 const rhs_reg = switch (rhs) {
12251226 .register => |r| r,
12261227 else => try self.copyToTmpRegister(rhs_ty, rhs),
12271228 };
1228 self.register_manager.freezeRegs(&.{rhs_reg});
1229 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1229 const rhs_reg_lock = self.register_manager.freezeReg(rhs_reg);
1230 defer if (rhs_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
12301231
12311232 const dest_reg = if (maybe_inst) |inst| blk: {
12321233 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
......@@ -1392,12 +1393,12 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
13921393 };
13931394 const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
13941395 const dest_reg = dest.register;
1395 self.register_manager.freezeRegs(&.{dest_reg});
1396 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1396 const dest_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_reg);
1397 defer self.register_manager.unfreezeReg(dest_reg_lock);
13971398
13981399 const truncated_reg = try self.register_manager.allocReg(null);
1399 self.register_manager.freezeRegs(&.{truncated_reg});
1400 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1400 const truncated_reg_lock = self.register_manager.freezeRegAssumeUnused(truncated_reg);
1401 defer self.register_manager.unfreezeReg(truncated_reg_lock);
14011402
14021403 // sbfx/ubfx truncated, dest, #0, #bits
14031404 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
......@@ -1493,12 +1494,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14931494
14941495 const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
14951496 const dest_reg = dest.register;
1496 self.register_manager.freezeRegs(&.{dest_reg});
1497 defer self.register_manager.unfreezeRegs(&.{dest_reg});
1497 const dest_reg_lock = self.register_manager.freezeRegAssumeUnused(dest_reg);
1498 defer self.register_manager.unfreezeReg(dest_reg_lock);
14981499
14991500 const truncated_reg = try self.register_manager.allocReg(null);
1500 self.register_manager.freezeRegs(&.{truncated_reg});
1501 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1501 const truncated_reg_lock = self.register_manager.freezeRegAssumeUnused(truncated_reg);
1502 defer self.register_manager.unfreezeReg(truncated_reg_lock);
15021503
15031504 // sbfx/ubfx truncated, dest, #0, #bits
15041505 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
......@@ -1526,28 +1527,32 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15261527 const lhs_is_register = lhs == .register;
15271528 const rhs_is_register = rhs == .register;
15281529
1529 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
1530 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
1531
1532 const lhs_reg = if (lhs_is_register) lhs.register else blk: {
1533 const reg = try self.register_manager.allocReg(null);
1534 self.register_manager.freezeRegs(&.{reg});
1535
1536 break :blk reg;
1537 };
1538 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1530 const lhs_lock: ?RegisterLock = if (lhs_is_register)
1531 self.register_manager.freezeReg(lhs.register)
1532 else
1533 null;
1534 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
15391535
1540 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
1541 const reg = try self.register_manager.allocReg(null);
1542 self.register_manager.freezeRegs(&.{reg});
1536 const lhs_reg = if (lhs_is_register)
1537 lhs.register
1538 else
1539 try self.register_manager.allocReg(null);
1540 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
1541 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
15431542
1544 break :blk reg;
1545 };
1546 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
1543 const rhs_reg = if (rhs_is_register)
1544 rhs.register
1545 else
1546 try self.register_manager.allocReg(null);
1547 const new_rhs_lock = self.register_manager.freezeReg(rhs_reg);
1548 defer if (new_rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
15471549
15481550 const dest_regs = try self.register_manager.allocRegs(2, .{ null, null });
1549 self.register_manager.freezeRegs(&dest_regs);
1550 defer self.register_manager.unfreezeRegs(&dest_regs);
1551 var dest_regs_locks: [2]RegisterLock = undefined;
1552 self.register_manager.freezeRegsAssumeUnused(2, dest_regs, &dest_regs_locks);
1553 defer for (dest_regs_locks) |reg| {
1554 self.register_manager.unfreezeReg(reg);
1555 };
15511556 const rdlo = dest_regs[0];
15521557 const rdhi = dest_regs[1];
15531558
......@@ -1555,8 +1560,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15551560 if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
15561561
15571562 const truncated_reg = try self.register_manager.allocReg(null);
1558 self.register_manager.freezeRegs(&.{truncated_reg});
1559 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
1563 const truncated_reg_lock = self.register_manager.freezeRegAssumeUnused(truncated_reg);
1564 defer self.register_manager.unfreezeReg(truncated_reg_lock);
15601565
15611566 _ = try self.addInst(.{
15621567 .tag = base_tag,
......@@ -1648,8 +1653,11 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
16481653 if (int_info.bits <= 32) {
16491654 const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
16501655
1651 if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});
1652 defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});
1656 const lhs_lock: ?RegisterLock = if (lhs == .register)
1657 self.register_manager.freezeRegAssumeUnused(lhs.register)
1658 else
1659 null;
1660 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
16531661
16541662 try self.spillCompareFlagsIfOccupied();
16551663 self.compare_flags_inst = null;
......@@ -1939,8 +1947,11 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
19391947 var buf: Type.SlicePtrFieldTypeBuffer = undefined;
19401948 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
19411949
1942 if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register});
1943 defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register});
1950 const index_lock: ?RegisterLock = if (index_is_register)
1951 self.register_manager.freezeRegAssumeUnused(index_mcv.register)
1952 else
1953 null;
1954 defer if (index_lock) |reg| self.register_manager.unfreezeReg(reg);
19441955
19451956 const base_mcv = slicePtr(slice_mcv);
19461957
......@@ -1950,20 +1961,20 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
19501961 .register => |r| r,
19511962 else => try self.copyToTmpRegister(slice_ptr_field_type, base_mcv),
19521963 };
1953 self.register_manager.freezeRegs(&.{base_reg});
1954 defer self.register_manager.unfreezeRegs(&.{base_reg});
1964 const base_reg_lock = self.register_manager.freezeRegAssumeUnused(base_reg);
1965 defer self.register_manager.unfreezeReg(base_reg_lock);
19551966
19561967 const dst_reg = try self.register_manager.allocReg(inst);
19571968 const dst_mcv = MCValue{ .register = dst_reg };
1958 self.register_manager.freezeRegs(&.{dst_reg});
1959 defer self.register_manager.unfreezeRegs(&.{dst_reg});
1969 const dst_reg_lock = self.register_manager.freezeRegAssumeUnused(dst_reg);
1970 defer self.register_manager.unfreezeReg(dst_reg_lock);
19601971
19611972 const index_reg: Register = switch (index_mcv) {
19621973 .register => |reg| reg,
19631974 else => try self.copyToTmpRegister(Type.usize, index_mcv),
19641975 };
1965 self.register_manager.freezeRegs(&.{index_reg});
1966 defer self.register_manager.unfreezeRegs(&.{index_reg});
1976 const index_reg_lock = self.register_manager.freezeRegAssumeUnused(index_reg);
1977 defer self.register_manager.unfreezeReg(index_reg_lock);
19671978
19681979 const tag: Mir.Inst.Tag = switch (elem_size) {
19691980 1 => .ldrb,
......@@ -2149,8 +2160,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
21492160 .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
21502161 .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
21512162 .register => |reg| {
2152 self.register_manager.freezeRegs(&.{reg});
2153 defer self.register_manager.unfreezeRegs(&.{reg});
2163 const reg_lock = self.register_manager.freezeReg(reg);
2164 defer if (reg_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
21542165
21552166 switch (dst_mcv) {
21562167 .dead => unreachable,
......@@ -2162,16 +2173,19 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
21622173 .stack_offset => |off| {
21632174 if (elem_size <= 4) {
21642175 const tmp_reg = try self.register_manager.allocReg(null);
2165 self.register_manager.freezeRegs(&.{tmp_reg});
2166 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2176 const tmp_reg_lock = self.register_manager.freezeRegAssumeUnused(tmp_reg);
2177 defer self.register_manager.unfreezeReg(tmp_reg_lock);
21672178
21682179 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
21692180 try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
21702181 } else {
21712182 // TODO optimize the register allocation
21722183 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2173 self.register_manager.freezeRegs(&regs);
2174 defer self.register_manager.unfreezeRegs(&regs);
2184 var regs_locks: [4]RegisterLock = undefined;
2185 self.register_manager.freezeRegsAssumeUnused(4, regs, &regs_locks);
2186 defer for (regs_locks) |reg_locked| {
2187 self.register_manager.unfreezeReg(reg_locked);
2188 };
21752189
21762190 const src_reg = reg;
21772191 const dst_reg = regs[0];
......@@ -2197,8 +2211,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
21972211 .stack_argument_offset,
21982212 => {
21992213 const reg = try self.register_manager.allocReg(null);
2200 self.register_manager.freezeRegs(&.{reg});
2201 defer self.register_manager.unfreezeRegs(&.{reg});
2214 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
2215 defer self.register_manager.unfreezeReg(reg_lock);
22022216
22032217 try self.genSetReg(ptr_ty, reg, ptr);
22042218 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
......@@ -2252,8 +2266,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
22522266 try self.genSetStack(value_ty, off, value);
22532267 },
22542268 .register => |addr_reg| {
2255 self.register_manager.freezeRegs(&.{addr_reg});
2256 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2269 const addr_reg_lock = self.register_manager.freezeReg(addr_reg);
2270 defer if (addr_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
22572271
22582272 switch (value) {
22592273 .dead => unreachable,
......@@ -2264,15 +2278,18 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
22642278 else => {
22652279 if (elem_size <= 4) {
22662280 const tmp_reg = try self.register_manager.allocReg(null);
2267 self.register_manager.freezeRegs(&.{tmp_reg});
2268 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
2281 const tmp_reg_lock = self.register_manager.freezeRegAssumeUnused(tmp_reg);
2282 defer self.register_manager.unfreezeReg(tmp_reg_lock);
22692283
22702284 try self.genSetReg(value_ty, tmp_reg, value);
22712285 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
22722286 } else {
22732287 const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2274 self.register_manager.freezeRegs(&regs);
2275 defer self.register_manager.unfreezeRegs(&regs);
2288 var regs_locks: [4]RegisterLock = undefined;
2289 self.register_manager.freezeRegsAssumeUnused(4, regs, &regs_locks);
2290 defer for (regs_locks) |reg| {
2291 self.register_manager.unfreezeReg(reg);
2292 };
22762293
22772294 const src_reg = regs[0];
22782295 const dst_reg = addr_reg;
......@@ -2356,12 +2373,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
23562373 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
23572374 .immediate = struct_field_offset,
23582375 });
2359 self.register_manager.freezeRegs(&.{offset_reg});
2360 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2376 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
2377 defer self.register_manager.unfreezeReg(offset_reg_lock);
23612378
23622379 const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
2363 self.register_manager.freezeRegs(&.{addr_reg});
2364 defer self.register_manager.unfreezeRegs(&.{addr_reg});
2380 const addr_reg_lock = self.register_manager.freezeRegAssumeUnused(addr_reg);
2381 defer self.register_manager.unfreezeReg(addr_reg_lock);
23652382
23662383 const dest = try self.binOp(
23672384 .add,
......@@ -2477,8 +2494,11 @@ fn binOpRegister(
24772494 const lhs_is_register = lhs == .register;
24782495 const rhs_is_register = rhs == .register;
24792496
2480 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
2481 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
2497 const lhs_lock: ?RegisterLock = if (lhs_is_register)
2498 self.register_manager.freezeReg(lhs.register)
2499 else
2500 null;
2501 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
24822502
24832503 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
24842504
......@@ -2489,13 +2509,13 @@ fn binOpRegister(
24892509 } else null;
24902510
24912511 const reg = try self.register_manager.allocReg(track_inst);
2492 self.register_manager.freezeRegs(&.{reg});
24932512
24942513 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
24952514
24962515 break :blk reg;
24972516 };
2498 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
2517 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
2518 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
24992519
25002520 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
25012521 const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
......@@ -2504,13 +2524,13 @@ fn binOpRegister(
25042524 } else null;
25052525
25062526 const reg = try self.register_manager.allocReg(track_inst);
2507 self.register_manager.freezeRegs(&.{reg});
25082527
25092528 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
25102529
25112530 break :blk reg;
25122531 };
2513 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
2532 const new_rhs_lock = self.register_manager.freezeReg(rhs_reg);
2533 defer if (new_rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
25142534
25152535 const dest_reg = switch (mir_tag) {
25162536 .cmp => .r0, // cmp has no destination regardless
......@@ -2593,7 +2613,11 @@ fn binOpImmediate(
25932613) !MCValue {
25942614 const lhs_is_register = lhs == .register;
25952615
2596 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
2616 const lhs_lock: ?RegisterLock = if (lhs_is_register)
2617 self.register_manager.freezeReg(lhs.register)
2618 else
2619 null;
2620 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
25972621
25982622 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
25992623
......@@ -2606,13 +2630,13 @@ fn binOpImmediate(
26062630 } else null;
26072631
26082632 const reg = try self.register_manager.allocReg(track_inst);
2609 self.register_manager.freezeRegs(&.{reg});
26102633
26112634 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
26122635
26132636 break :blk reg;
26142637 };
2615 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
2638 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
2639 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
26162640
26172641 const dest_reg = switch (mir_tag) {
26182642 .cmp => .r0, // cmp has no destination reg
src/arch/riscv64/CodeGen.zig+12-8
......@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
2323const build_options = @import("build_options");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
2525const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
26const RegisterLock = RegisterManager.RegisterLock;
2627
2728const FnResult = @import("../../codegen.zig").FnResult;
2829const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
......@@ -937,8 +938,11 @@ fn binOpRegister(
937938 const lhs_is_register = lhs == .register;
938939 const rhs_is_register = rhs == .register;
939940
940 if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
941 if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
941 const lhs_lock: ?RegisterLock = if (lhs_is_register)
942 self.register_manager.freezeReg(lhs.register)
943 else
944 null;
945 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
942946
943947 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
944948
......@@ -949,13 +953,13 @@ fn binOpRegister(
949953 } else null;
950954
951955 const reg = try self.register_manager.allocReg(track_inst);
952 self.register_manager.freezeRegs(&.{reg});
953956
954957 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
955958
956959 break :blk reg;
957960 };
958 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
961 const new_lhs_lock = self.register_manager.freezeReg(lhs_reg);
962 defer if (new_lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
959963
960964 const rhs_reg = if (rhs_is_register) rhs.register else blk: {
961965 const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
......@@ -964,13 +968,13 @@ fn binOpRegister(
964968 } else null;
965969
966970 const reg = try self.register_manager.allocReg(track_inst);
967 self.register_manager.freezeRegs(&.{reg});
968971
969972 if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
970973
971974 break :blk reg;
972975 };
973 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
976 const new_rhs_lock = self.register_manager.freezeReg(rhs_reg);
977 defer if (new_rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
974978
975979 const dest_reg = if (maybe_inst) |inst| blk: {
976980 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
......@@ -1448,8 +1452,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
14481452 .stack_offset,
14491453 => {
14501454 const reg = try self.register_manager.allocReg(null);
1451 self.register_manager.freezeRegs(&.{reg});
1452 defer self.register_manager.unfreezeRegs(&.{reg});
1455 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
1456 defer self.register_manager.unfreezeReg(reg_lock);
14531457
14541458 try self.genSetReg(ptr_ty, reg, ptr);
14551459 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
src/arch/x86_64/CodeGen.zig+422-241
......@@ -22,6 +22,8 @@ const Liveness = @import("../../Liveness.zig");
2222const Mir = @import("Mir.zig");
2323const Module = @import("../../Module.zig");
2424const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
26const RegisterLock = RegisterManager.RegisterLock;
2527const Target = std.Target;
2628const Type = @import("../../type.zig").Type;
2729const TypedValue = @import("../../TypedValue.zig");
......@@ -42,8 +44,6 @@ const InnerError = error{
4244 OutOfRegisters,
4345};
4446
45const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
46
4747gpa: Allocator,
4848air: Air,
4949liveness: Liveness,
......@@ -211,40 +211,6 @@ pub const MCValue = union(enum) {
211211 else => false,
212212 };
213213 }
214
215 fn freezeIfRegister(mcv: MCValue, mgr: *RegisterManager) void {
216 switch (mcv) {
217 .register,
218 .register_overflow_signed,
219 .register_overflow_unsigned,
220 => |reg| {
221 mgr.freezeRegs(&.{reg});
222 },
223 else => {},
224 }
225 }
226
227 fn unfreezeIfRegister(mcv: MCValue, mgr: *RegisterManager) void {
228 switch (mcv) {
229 .register,
230 .register_overflow_signed,
231 .register_overflow_unsigned,
232 => |reg| {
233 mgr.unfreezeRegs(&.{reg});
234 },
235 else => {},
236 }
237 }
238
239 fn asRegister(mcv: MCValue) ?Register {
240 return switch (mcv) {
241 .register,
242 .register_overflow_signed,
243 .register_overflow_unsigned,
244 => |reg| reg,
245 else => null,
246 };
247 }
248214};
249215
250216const Branch = struct {
......@@ -876,15 +842,20 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
876842 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
877843 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
878844
879 if (result.asRegister()) |reg| {
880 // In some cases (such as bitcast), an operand
881 // may be the same MCValue as the result. If
882 // that operand died and was a register, it
883 // was freed by processDeath. We have to
884 // "re-allocate" the register.
885 if (self.register_manager.isRegFree(reg)) {
886 self.register_manager.getRegAssumeFree(reg, inst);
887 }
845 switch (result) {
846 .register,
847 .register_overflow_signed,
848 .register_overflow_unsigned,
849 => |reg| {
850 // In some cases (such as bitcast), an operand
851 // may be the same MCValue as the result. If
852 // that operand died and was a register, it
853 // was freed by processDeath. We have to
854 // "re-allocate" the register.
855 if (self.register_manager.isRegFree(reg)) {
856 self.register_manager.getRegAssumeFree(reg, inst);
857 }
858 },
888859 }
889860 }
890861 self.finishAirBookkeeping();
......@@ -955,7 +926,15 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
955926 const stack_mcv = try self.allocRegOrMem(inst, false);
956927 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
957928 const reg_mcv = self.getResolvedInstValue(inst);
958 assert(reg.to64() == reg_mcv.asRegister().?.to64());
929 switch (reg_mcv) {
930 .register,
931 .register_overflow_unsigned,
932 .register_overflow_signed,
933 => |other| {
934 assert(reg.to64() == other.to64());
935 },
936 else => {},
937 }
959938 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
960939 try branch.inst_table.put(self.gpa, inst, stack_mcv);
961940 try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
......@@ -1043,8 +1022,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
10431022 return self.fail("TODO implement intCast for abi sizes larger than 8", .{});
10441023 }
10451024
1046 operand.freezeIfRegister(&self.register_manager);
1047 defer operand.unfreezeIfRegister(&self.register_manager);
1025 const operand_lock: ?RegisterLock = switch (operand) {
1026 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1027 else => null,
1028 };
1029 defer if (operand_lock) |reg| self.register_manager.unfreezeReg(reg);
10481030
10491031 const reg = try self.register_manager.allocReg(inst);
10501032 try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
......@@ -1071,8 +1053,11 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
10711053 return self.fail("TODO implement trunc for abi sizes larger than 8", .{});
10721054 }
10731055
1074 operand.freezeIfRegister(&self.register_manager);
1075 defer operand.unfreezeIfRegister(&self.register_manager);
1056 const operand_lock: ?RegisterLock = switch (operand) {
1057 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1058 else => null,
1059 };
1060 defer if (operand_lock) |reg| self.register_manager.unfreezeReg(reg);
10761061
10771062 const reg: Register = blk: {
10781063 if (operand.isRegister()) {
......@@ -1156,16 +1141,22 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
11561141 // TODO improve by checking if any operand can be reused.
11571142 // TODO audit register allocation
11581143 const lhs = try self.resolveInst(bin_op.lhs);
1159 lhs.freezeIfRegister(&self.register_manager);
1160 defer lhs.unfreezeIfRegister(&self.register_manager);
1144 const lhs_lock: ?RegisterLock = switch (lhs) {
1145 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1146 else => null,
1147 };
1148 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
11611149
11621150 const lhs_reg = try self.copyToTmpRegister(ty, lhs);
1163 self.register_manager.freezeRegs(&.{lhs_reg});
1164 defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1151 const lhs_reg_lock = self.register_manager.freezeRegAssumeUnused(lhs_reg);
1152 defer self.register_manager.unfreezeReg(lhs_reg_lock);
11651153
11661154 const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32);
1167 rhs_mcv.freezeIfRegister(&self.register_manager);
1168 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1155 const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
1156 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1157 else => null,
1158 };
1159 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
11691160
11701161 try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);
11711162
......@@ -1200,8 +1191,11 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
12001191 const offset = try self.resolveInst(op_rhs);
12011192 const offset_ty = self.air.typeOf(op_rhs);
12021193
1203 offset.freezeIfRegister(&self.register_manager);
1204 defer offset.unfreezeIfRegister(&self.register_manager);
1194 const offset_lock: ?RegisterLock = switch (offset) {
1195 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1196 else => null,
1197 };
1198 defer if (offset_lock) |reg| self.register_manager.unfreezeReg(reg);
12051199
12061200 const dst_mcv = blk: {
12071201 if (self.reuseOperand(inst, op_lhs, 0, ptr)) {
......@@ -1210,8 +1204,11 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
12101204 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, ptr) };
12111205 };
12121206
1213 dst_mcv.freezeIfRegister(&self.register_manager);
1214 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
1207 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
1208 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1209 else => null,
1210 };
1211 defer if (dst_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
12151212
12161213 const offset_mcv = blk: {
12171214 if (self.reuseOperand(inst, op_rhs, 1, offset)) {
......@@ -1220,8 +1217,11 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
12201217 break :blk MCValue{ .register = try self.copyToTmpRegister(offset_ty, offset) };
12211218 };
12221219
1223 offset_mcv.freezeIfRegister(&self.register_manager);
1224 defer offset_mcv.unfreezeIfRegister(&self.register_manager);
1220 const offset_mcv_lock: ?RegisterLock = switch (offset_mcv) {
1221 .register => |reg| self.register_manager.freezeReg(reg),
1222 else => null,
1223 };
1224 defer if (offset_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
12251225
12261226 try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });
12271227
......@@ -1306,12 +1306,18 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
13061306 const dst_ty = self.air.typeOf(op_lhs);
13071307
13081308 const lhs = try self.resolveInst(op_lhs);
1309 lhs.freezeIfRegister(&self.register_manager);
1310 defer lhs.unfreezeIfRegister(&self.register_manager);
1309 const lhs_lock: ?RegisterLock = switch (lhs) {
1310 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1311 else => null,
1312 };
1313 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
13111314
13121315 const rhs = try self.resolveInst(op_rhs);
1313 rhs.freezeIfRegister(&self.register_manager);
1314 defer rhs.unfreezeIfRegister(&self.register_manager);
1316 const rhs_lock: ?RegisterLock = switch (rhs) {
1317 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1318 else => null,
1319 };
1320 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
13151321
13161322 const dst_mcv = blk: {
13171323 if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
......@@ -1319,17 +1325,21 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
13191325 }
13201326 break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);
13211327 };
1322
1323 dst_mcv.freezeIfRegister(&self.register_manager);
1324 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
1328 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
1329 .register => |reg| self.register_manager.freezeReg(reg),
1330 else => null,
1331 };
1332 defer if (dst_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
13251333
13261334 const rhs_mcv = blk: {
13271335 if (rhs.isMemory() or rhs.isRegister()) break :blk rhs;
13281336 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
13291337 };
1330
1331 rhs_mcv.freezeIfRegister(&self.register_manager);
1332 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1338 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
1339 .register => |reg| self.register_manager.freezeReg(reg),
1340 else => null,
1341 };
1342 defer if (rhs_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
13331343
13341344 try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv);
13351345
......@@ -1366,8 +1376,11 @@ fn airMul(self: *Self, inst: Air.Inst.Index) !void {
13661376 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
13671377 try self.register_manager.getReg(.rax, inst);
13681378 try self.register_manager.getReg(.rdx, null);
1369 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1370 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1379 var reg_locks: [2]RegisterLock = undefined;
1380 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1381 defer for (reg_locks) |reg| {
1382 self.register_manager.unfreezeReg(reg);
1383 };
13711384
13721385 const lhs = try self.resolveInst(bin_op.lhs);
13731386 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1477,8 +1490,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14771490 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
14781491 try self.register_manager.getReg(.rax, inst);
14791492 try self.register_manager.getReg(.rdx, null);
1480 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1481 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1493 var reg_locks: [2]RegisterLock = undefined;
1494 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1495 defer for (reg_locks) |reg| {
1496 self.register_manager.unfreezeReg(reg);
1497 };
14821498
14831499 const lhs = try self.resolveInst(bin_op.lhs);
14841500 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1504,21 +1520,28 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15041520 const lhs = try self.resolveInst(bin_op.lhs);
15051521 const rhs = try self.resolveInst(bin_op.rhs);
15061522
1507 rhs.freezeIfRegister(&self.register_manager);
1508 defer rhs.unfreezeIfRegister(&self.register_manager);
1523 const rhs_lock: ?RegisterLock = switch (rhs) {
1524 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1525 else => null,
1526 };
1527 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
15091528
15101529 const dst_reg: Register = blk: {
15111530 if (lhs.isRegister()) break :blk lhs.register;
15121531 break :blk try self.copyToTmpRegister(ty, lhs);
15131532 };
1514 self.register_manager.freezeRegs(&.{dst_reg});
1533 const dst_reg_lock = self.register_manager.freezeRegAssumeUnused(dst_reg);
1534 defer self.register_manager.unfreezeReg(dst_reg_lock);
15151535
15161536 const rhs_mcv = blk: {
15171537 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
15181538 break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };
15191539 };
1520 rhs_mcv.freezeIfRegister(&self.register_manager);
1521 defer rhs_mcv.unfreezeIfRegister(&self.register_manager);
1540 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
1541 .register => |reg| self.register_manager.freezeReg(reg),
1542 else => null,
1543 };
1544 defer if (rhs_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
15221545
15231546 try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
15241547
......@@ -1528,8 +1551,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15281551 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
15291552 try self.register_manager.getReg(.rax, null);
15301553 try self.register_manager.getReg(.rdx, null);
1531 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1532 defer self.register_manager.unfreezeRegs(&.{.rdx});
1554 var reg_locks: [2]RegisterLock = undefined;
1555 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1556 defer for (reg_locks) |reg| {
1557 self.register_manager.unfreezeReg(reg);
1558 };
15331559
15341560 const lhs = try self.resolveInst(bin_op.lhs);
15351561 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1540,7 +1566,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15401566 },
15411567 }
15421568 };
1543 defer self.register_manager.unfreezeRegs(&.{dst_reg});
1569 const dst_reg_lock = self.register_manager.freezeRegAssumeUnused(dst_reg);
1570 defer self.register_manager.unfreezeReg(dst_reg_lock);
15441571
15451572 const tuple_ty = self.air.typeOfIndex(inst);
15461573 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
......@@ -1554,8 +1581,11 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15541581 };
15551582
15561583 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1557 self.register_manager.freezeRegs(&temp_regs);
1558 defer self.register_manager.unfreezeRegs(&temp_regs);
1584 var temp_regs_locks: [3]RegisterLock = undefined;
1585 self.register_manager.freezeRegsAssumeUnused(3, temp_regs, &temp_regs_locks);
1586 defer for (temp_regs_locks) |reg| {
1587 self.register_manager.unfreezeReg(reg);
1588 };
15591589
15601590 const overflow_reg = temp_regs[0];
15611591 const flags: u2 = switch (int_info.signedness) {
......@@ -1703,14 +1733,15 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
17031733 .register => |reg| reg,
17041734 else => try self.copyToTmpRegister(ty, lhs),
17051735 };
1706 self.register_manager.freezeRegs(&.{dividend});
1736 const dividend_lock = self.register_manager.freezeReg(dividend);
1737 defer if (dividend_lock) |reg| self.register_manager.unfreezeReg(reg);
17071738
17081739 const divisor = switch (rhs) {
17091740 .register => |reg| reg,
17101741 else => try self.copyToTmpRegister(ty, rhs),
17111742 };
1712 self.register_manager.freezeRegs(&.{divisor});
1713 defer self.register_manager.unfreezeRegs(&.{ dividend, divisor });
1743 const divisor_lock = self.register_manager.freezeReg(divisor);
1744 defer if (divisor_lock) |reg| self.register_manager.unfreezeReg(reg);
17141745
17151746 try self.genIntMulDivOpMir(switch (signedness) {
17161747 .signed => .idiv,
......@@ -1779,20 +1810,30 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
17791810 };
17801811 try self.register_manager.getReg(.rax, track_rax);
17811812 try self.register_manager.getReg(.rdx, null);
1782 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1783 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1813 var reg_locks: [2]RegisterLock = undefined;
1814 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1815 defer for (reg_locks) |reg| {
1816 self.register_manager.unfreezeReg(reg);
1817 };
17841818
17851819 const lhs = try self.resolveInst(bin_op.lhs);
1786 lhs.freezeIfRegister(&self.register_manager);
1787 defer lhs.unfreezeIfRegister(&self.register_manager);
1820 const lhs_lock: ?RegisterLock = switch (lhs) {
1821 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1822 else => null,
1823 };
1824 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
17881825
17891826 const rhs = blk: {
17901827 const rhs = try self.resolveInst(bin_op.rhs);
17911828 if (signedness == .signed) {
17921829 switch (tag) {
17931830 .div_floor => {
1794 rhs.freezeIfRegister(&self.register_manager);
1795 defer rhs.unfreezeIfRegister(&self.register_manager);
1831 const rhs_lock: ?RegisterLock = switch (rhs) {
1832 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
1833 else => null,
1834 };
1835 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
1836
17961837 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
17971838 },
17981839 else => {},
......@@ -1800,8 +1841,11 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
18001841 }
18011842 break :blk rhs;
18021843 };
1803 rhs.freezeIfRegister(&self.register_manager);
1804 defer rhs.unfreezeIfRegister(&self.register_manager);
1844 const rhs_lock: ?RegisterLock = switch (rhs) {
1845 .register => |reg| self.register_manager.freezeReg(reg),
1846 else => null,
1847 };
1848 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
18051849
18061850 if (signedness == .unsigned) {
18071851 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);
......@@ -1835,8 +1879,11 @@ fn airRem(self: *Self, inst: Air.Inst.Index) !void {
18351879 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
18361880 try self.register_manager.getReg(.rax, null);
18371881 try self.register_manager.getReg(.rdx, inst);
1838 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1839 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1882 var reg_locks: [2]RegisterLock = undefined;
1883 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1884 defer for (reg_locks) |reg| {
1885 self.register_manager.unfreezeReg(reg);
1886 };
18401887
18411888 const lhs = try self.resolveInst(bin_op.lhs);
18421889 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1863,8 +1910,11 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {
18631910 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
18641911 try self.register_manager.getReg(.rax, null);
18651912 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);
1866 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1867 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1913 var reg_locks: [2]RegisterLock = undefined;
1914 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rdx }, &reg_locks);
1915 defer for (reg_locks) |reg| {
1916 self.register_manager.unfreezeReg(reg);
1917 };
18681918
18691919 const lhs = try self.resolveInst(bin_op.lhs);
18701920 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1954,12 +2004,15 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {
19542004 try self.register_manager.getReg(.rcx, null);
19552005 try self.genSetReg(shift_ty, .rcx, shift);
19562006 }
1957 self.register_manager.freezeRegs(&.{.rcx});
1958 defer self.register_manager.unfreezeRegs(&.{.rcx});
2007 const rcx_lock = self.register_manager.freezeRegAssumeUnused(.rcx);
2008 defer self.register_manager.unfreezeReg(rcx_lock);
19592009
19602010 const value = try self.resolveInst(bin_op.lhs);
1961 value.freezeIfRegister(&self.register_manager);
1962 defer value.unfreezeIfRegister(&self.register_manager);
2011 const value_lock: ?RegisterLock = switch (value) {
2012 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2013 else => null,
2014 };
2015 defer if (value_lock) |reg| self.register_manager.unfreezeReg(reg);
19632016
19642017 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value);
19652018 _ = try self.addInst(.{
......@@ -2055,8 +2108,11 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
20552108 const err_ty = err_union_ty.errorUnionSet();
20562109 const payload_ty = err_union_ty.errorUnionPayload();
20572110 const operand = try self.resolveInst(ty_op.operand);
2058 operand.freezeIfRegister(&self.register_manager);
2059 defer operand.unfreezeIfRegister(&self.register_manager);
2111 const operand_lock: ?RegisterLock = switch (operand) {
2112 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2113 else => null,
2114 };
2115 defer if (operand_lock) |reg| self.register_manager.unfreezeReg(reg);
20602116
20612117 const result: MCValue = result: {
20622118 if (!payload_ty.hasRuntimeBits()) break :result operand;
......@@ -2085,8 +2141,11 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
20852141 if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
20862142
20872143 const operand = try self.resolveInst(ty_op.operand);
2088 operand.freezeIfRegister(&self.register_manager);
2089 defer operand.unfreezeIfRegister(&self.register_manager);
2144 const operand_lock: ?RegisterLock = switch (operand) {
2145 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2146 else => null,
2147 };
2148 defer if (operand_lock) |reg| self.register_manager.unfreezeReg(reg);
20902149
20912150 const abi_align = err_union_ty.abiAlignment(self.target.*);
20922151 const err_ty = err_union_ty.errorUnionSet();
......@@ -2154,8 +2213,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
21542213
21552214 const optional_ty = self.air.typeOfIndex(inst);
21562215 const operand = try self.resolveInst(ty_op.operand);
2157 operand.freezeIfRegister(&self.register_manager);
2158 defer operand.unfreezeIfRegister(&self.register_manager);
2216 const operand_lock: ?RegisterLock = switch (operand) {
2217 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2218 else => null,
2219 };
2220 defer if (operand_lock) |reg| self.register_manager.unfreezeReg(reg);
21592221
21602222 if (optional_ty.isPtrLikeOptional()) {
21612223 // TODO should we check if we can reuse the operand?
......@@ -2288,8 +2350,11 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
22882350fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
22892351 const slice_ty = self.air.typeOf(lhs);
22902352 const slice_mcv = try self.resolveInst(lhs);
2291 slice_mcv.freezeIfRegister(&self.register_manager);
2292 defer slice_mcv.unfreezeIfRegister(&self.register_manager);
2353 const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) {
2354 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2355 else => null,
2356 };
2357 defer if (slice_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
22932358
22942359 const elem_ty = slice_ty.childType();
22952360 const elem_size = elem_ty.abiSize(self.target.*);
......@@ -2298,12 +2363,15 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
22982363
22992364 const index_ty = self.air.typeOf(rhs);
23002365 const index_mcv = try self.resolveInst(rhs);
2301 index_mcv.freezeIfRegister(&self.register_manager);
2302 defer index_mcv.unfreezeIfRegister(&self.register_manager);
2366 const index_mcv_lock: ?RegisterLock = switch (index_mcv) {
2367 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2368 else => null,
2369 };
2370 defer if (index_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
23032371
23042372 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);
2305 self.register_manager.freezeRegs(&.{offset_reg});
2306 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2373 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
2374 defer self.register_manager.unfreezeReg(offset_reg_lock);
23072375
23082376 const addr_reg = try self.register_manager.allocReg(null);
23092377 switch (slice_mcv) {
......@@ -2359,20 +2427,26 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
23592427 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
23602428 const array_ty = self.air.typeOf(bin_op.lhs);
23612429 const array = try self.resolveInst(bin_op.lhs);
2362 array.freezeIfRegister(&self.register_manager);
2363 defer array.unfreezeIfRegister(&self.register_manager);
2430 const array_lock: ?RegisterLock = switch (array) {
2431 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2432 else => null,
2433 };
2434 defer if (array_lock) |reg| self.register_manager.unfreezeReg(reg);
23642435
23652436 const elem_ty = array_ty.childType();
23662437 const elem_abi_size = elem_ty.abiSize(self.target.*);
23672438
23682439 const index_ty = self.air.typeOf(bin_op.rhs);
23692440 const index = try self.resolveInst(bin_op.rhs);
2370 index.freezeIfRegister(&self.register_manager);
2371 defer index.unfreezeIfRegister(&self.register_manager);
2441 const index_lock: ?RegisterLock = switch (index) {
2442 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2443 else => null,
2444 };
2445 defer if (index_lock) |reg| self.register_manager.unfreezeReg(reg);
23722446
23732447 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2374 self.register_manager.freezeRegs(&.{offset_reg});
2375 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2448 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
2449 defer self.register_manager.unfreezeReg(offset_reg_lock);
23762450
23772451 const addr_reg = try self.register_manager.allocReg(null);
23782452 switch (array) {
......@@ -2432,19 +2506,25 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
24322506
24332507 const ptr_ty = self.air.typeOf(bin_op.lhs);
24342508 const ptr = try self.resolveInst(bin_op.lhs);
2435 ptr.freezeIfRegister(&self.register_manager);
2436 defer ptr.unfreezeIfRegister(&self.register_manager);
2509 const ptr_lock: ?RegisterLock = switch (ptr) {
2510 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2511 else => null,
2512 };
2513 defer if (ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
24372514
24382515 const elem_ty = ptr_ty.elemType2();
24392516 const elem_abi_size = elem_ty.abiSize(self.target.*);
24402517 const index_ty = self.air.typeOf(bin_op.rhs);
24412518 const index = try self.resolveInst(bin_op.rhs);
2442 index.freezeIfRegister(&self.register_manager);
2443 defer index.unfreezeIfRegister(&self.register_manager);
2519 const index_lock: ?RegisterLock = switch (index) {
2520 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2521 else => null,
2522 };
2523 defer if (index_lock) |reg| self.register_manager.unfreezeReg(reg);
24442524
24452525 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2446 self.register_manager.freezeRegs(&.{offset_reg});
2447 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2526 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
2527 defer self.register_manager.unfreezeReg(offset_reg_lock);
24482528
24492529 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
24502530 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
......@@ -2473,19 +2553,25 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
24732553 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
24742554 const ptr_ty = self.air.typeOf(extra.lhs);
24752555 const ptr = try self.resolveInst(extra.lhs);
2476 ptr.freezeIfRegister(&self.register_manager);
2477 defer ptr.unfreezeIfRegister(&self.register_manager);
2556 const ptr_lock: ?RegisterLock = switch (ptr) {
2557 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2558 else => null,
2559 };
2560 defer if (ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
24782561
24792562 const elem_ty = ptr_ty.elemType2();
24802563 const elem_abi_size = elem_ty.abiSize(self.target.*);
24812564 const index_ty = self.air.typeOf(extra.rhs);
24822565 const index = try self.resolveInst(extra.rhs);
2483 index.freezeIfRegister(&self.register_manager);
2484 defer index.unfreezeIfRegister(&self.register_manager);
2566 const index_lock: ?RegisterLock = switch (index) {
2567 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2568 else => null,
2569 };
2570 defer if (index_lock) |reg| self.register_manager.unfreezeReg(reg);
24852571
24862572 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2487 self.register_manager.freezeRegs(&.{offset_reg});
2488 defer self.register_manager.unfreezeRegs(&.{offset_reg});
2573 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
2574 defer self.register_manager.unfreezeReg(offset_reg_lock);
24892575
24902576 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
24912577 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
......@@ -2506,12 +2592,18 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
25062592 }
25072593
25082594 const ptr = try self.resolveInst(bin_op.lhs);
2509 ptr.freezeIfRegister(&self.register_manager);
2510 defer ptr.unfreezeIfRegister(&self.register_manager);
2595 const ptr_lock: ?RegisterLock = switch (ptr) {
2596 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2597 else => null,
2598 };
2599 defer if (ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
25112600
25122601 const tag = try self.resolveInst(bin_op.rhs);
2513 tag.freezeIfRegister(&self.register_manager);
2514 defer tag.unfreezeIfRegister(&self.register_manager);
2602 const tag_lock: ?RegisterLock = switch (tag) {
2603 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2604 else => null,
2605 };
2606 defer if (tag_lock) |reg| self.register_manager.unfreezeReg(reg);
25152607
25162608 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
25172609 // TODO reusing the operand
......@@ -2541,8 +2633,11 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
25412633
25422634 // TODO reusing the operand
25432635 const operand = try self.resolveInst(ty_op.operand);
2544 operand.freezeIfRegister(&self.register_manager);
2545 defer operand.unfreezeIfRegister(&self.register_manager);
2636 const operand_lock: ?RegisterLock = switch (operand) {
2637 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
2638 else => null,
2639 };
2640 defer if (operand_lock) |reg| self.register_manager.unfreezeReg(reg);
25462641
25472642 const tag_abi_size = tag_ty.abiSize(self.target.*);
25482643 const dst_mcv: MCValue = blk: {
......@@ -2689,8 +2784,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
26892784 try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
26902785 },
26912786 .register => |reg| {
2692 self.register_manager.freezeRegs(&.{reg});
2693 defer self.register_manager.unfreezeRegs(&.{reg});
2787 const reg_lock = self.register_manager.freezeReg(reg);
2788 defer if (reg_lock) |locked_reg| self.register_manager.unfreezeReg(locked_reg);
26942789
26952790 switch (dst_mcv) {
26962791 .dead => unreachable,
......@@ -2815,8 +2910,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
28152910 try self.genSetStack(value_ty, off, value, .{});
28162911 },
28172912 .register => |reg| {
2818 self.register_manager.freezeRegs(&.{reg});
2819 defer self.register_manager.unfreezeRegs(&.{reg});
2913 const reg_lock = self.register_manager.freezeReg(reg);
2914 defer if (reg_lock) |locked_reg| self.register_manager.unfreezeReg(locked_reg);
28202915
28212916 switch (value) {
28222917 .none => unreachable,
......@@ -2906,12 +3001,15 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
29063001 .direct_load,
29073002 .memory,
29083003 => {
2909 value.freezeIfRegister(&self.register_manager);
2910 defer value.unfreezeIfRegister(&self.register_manager);
3004 const value_lock: ?RegisterLock = switch (value) {
3005 .register => |reg| self.register_manager.freezeReg(reg),
3006 else => null,
3007 };
3008 defer if (value_lock) |reg| self.register_manager.unfreezeReg(reg);
29113009
29123010 const addr_reg = try self.register_manager.allocReg(null);
2913 self.register_manager.freezeRegs(&.{addr_reg});
2914 defer self.register_manager.unfreezeRegs(&.{addr_reg});
3011 const addr_reg_lock = self.register_manager.freezeRegAssumeUnused(addr_reg);
3012 defer self.register_manager.unfreezeReg(addr_reg_lock);
29153013
29163014 try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
29173015
......@@ -2982,8 +3080,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
29823080 => {
29833081 if (abi_size <= 8) {
29843082 const tmp_reg = try self.register_manager.allocReg(null);
2985 self.register_manager.freezeRegs(&.{tmp_reg});
2986 defer self.register_manager.unfreezeRegs(&.{tmp_reg});
3083 const tmp_reg_lock = self.register_manager.freezeRegAssumeUnused(tmp_reg);
3084 defer self.register_manager.unfreezeReg(tmp_reg_lock);
29873085
29883086 try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
29893087
......@@ -3073,8 +3171,8 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
30733171 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
30743172 .immediate = struct_field_offset,
30753173 });
3076 self.register_manager.freezeRegs(&.{offset_reg});
3077 defer self.register_manager.unfreezeRegs(&.{offset_reg});
3174 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
3175 defer self.register_manager.unfreezeReg(offset_reg_lock);
30783176
30793177 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, mcv);
30803178 try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
......@@ -3085,24 +3183,27 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
30853183 break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };
30863184 },
30873185 .register => |reg| {
3186 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
3187 defer self.register_manager.unfreezeReg(reg_lock);
3188
30883189 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
30893190 .immediate = struct_field_offset,
30903191 });
3091 self.register_manager.freezeRegs(&.{offset_reg});
3092 defer self.register_manager.unfreezeRegs(&.{offset_reg});
3192 const offset_reg_lock = self.register_manager.freezeRegAssumeUnused(offset_reg);
3193 defer self.register_manager.unfreezeReg(offset_reg_lock);
30933194
30943195 const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
30953196 const result_reg = blk: {
30963197 if (can_reuse_operand) {
30973198 break :blk reg;
30983199 } else {
3099 self.register_manager.freezeRegs(&.{reg});
31003200 const result_reg = try self.register_manager.allocReg(inst);
31013201 try self.genSetReg(ptr_ty, result_reg, mcv);
31023202 break :blk result_reg;
31033203 }
31043204 };
3105 defer if (!can_reuse_operand) self.register_manager.unfreezeRegs(&.{reg});
3205 const result_reg_lock = self.register_manager.freezeReg(result_reg);
3206 defer if (result_reg_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
31063207
31073208 try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });
31083209 break :result MCValue{ .register = result_reg };
......@@ -3130,8 +3231,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
31303231 break :result MCValue{ .stack_offset = stack_offset };
31313232 },
31323233 .register => |reg| {
3133 self.register_manager.freezeRegs(&.{reg});
3134 defer self.register_manager.unfreezeRegs(&.{reg});
3234 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
3235 defer self.register_manager.unfreezeReg(reg_lock);
31353236
31363237 const dst_mcv = blk: {
31373238 if (self.reuseOperand(inst, operand, 0, mcv)) {
......@@ -3143,8 +3244,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
31433244 break :blk dst_mcv;
31443245 }
31453246 };
3146 dst_mcv.freezeIfRegister(&self.register_manager);
3147 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
3247 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3248 .register => |reg| self.register_manager.freezeReg(reg),
3249 else => null,
3250 };
3251 defer if (dst_mcv_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
31483252
31493253 // Shift by struct_field_offset.
31503254 const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
......@@ -3186,8 +3290,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
31863290 },
31873291 1 => {
31883292 // Get overflow bit.
3189 mcv.freezeIfRegister(&self.register_manager);
3190 defer mcv.unfreezeIfRegister(&self.register_manager);
3293 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
3294 defer self.register_manager.unfreezeReg(reg_lock);
31913295
31923296 const dst_reg = try self.register_manager.allocReg(inst);
31933297 const flags: u2 = switch (mcv) {
......@@ -3229,12 +3333,18 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
32293333 const dst_ty = self.air.typeOf(op_lhs);
32303334
32313335 const lhs = try self.resolveInst(op_lhs);
3232 lhs.freezeIfRegister(&self.register_manager);
3233 defer lhs.unfreezeIfRegister(&self.register_manager);
3336 const lhs_lock: ?RegisterLock = switch (lhs) {
3337 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
3338 else => null,
3339 };
3340 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
32343341
32353342 const rhs = try self.resolveInst(op_rhs);
3236 rhs.freezeIfRegister(&self.register_manager);
3237 defer rhs.unfreezeIfRegister(&self.register_manager);
3343 const rhs_lock: ?RegisterLock = switch (rhs) {
3344 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
3345 else => null,
3346 };
3347 defer if (rhs_lock) |reg| self.register_manager.unfreezeReg(reg);
32383348
32393349 var flipped: bool = false;
32403350 const dst_mcv = blk: {
......@@ -3247,16 +3357,22 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
32473357 }
32483358 break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);
32493359 };
3250 dst_mcv.freezeIfRegister(&self.register_manager);
3251 defer dst_mcv.unfreezeIfRegister(&self.register_manager);
3360 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3361 .register => |reg| self.register_manager.freezeReg(reg),
3362 else => null,
3363 };
3364 defer if (dst_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
32523365
32533366 const src_mcv = blk: {
32543367 const mcv = if (flipped) lhs else rhs;
32553368 if (mcv.isRegister() or mcv.isMemory()) break :blk mcv;
32563369 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) };
32573370 };
3258 src_mcv.freezeIfRegister(&self.register_manager);
3259 defer src_mcv.unfreezeIfRegister(&self.register_manager);
3371 const src_mcv_lock: ?RegisterLock = switch (src_mcv) {
3372 .register => |reg| self.register_manager.freezeReg(reg),
3373 else => null,
3374 };
3375 defer if (src_mcv_lock) |reg| self.register_manager.unfreezeReg(reg);
32603376
32613377 const tag = self.air.instructions.items(.tag)[inst];
32623378 switch (tag) {
......@@ -3287,8 +3403,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
32873403 .register_overflow_unsigned => unreachable,
32883404 .register_overflow_signed => unreachable,
32893405 .ptr_stack_offset => {
3290 self.register_manager.freezeRegs(&.{dst_reg});
3291 defer self.register_manager.unfreezeRegs(&.{dst_reg});
3406 const dst_reg_lock = self.register_manager.freezeReg(dst_reg);
3407 defer if (dst_reg_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
3408
32923409 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
32933410 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
32943411 },
......@@ -3318,8 +3435,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
33183435 .compare_flags_unsigned,
33193436 => {
33203437 assert(abi_size <= 8);
3321 self.register_manager.freezeRegs(&.{dst_reg});
3322 defer self.register_manager.unfreezeRegs(&.{dst_reg});
3438 const dst_reg_lock = self.register_manager.freezeReg(dst_reg);
3439 defer if (dst_reg_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
3440
33233441 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
33243442 return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
33253443 },
......@@ -3659,7 +3777,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
36593777 try self.register_manager.getReg(reg, null);
36603778 }
36613779
3662 if (info.return_value == .stack_offset) {
3780 const rdi_lock: ?RegisterLock = if (info.return_value == .stack_offset) blk: {
36633781 const ret_ty = fn_ty.fnReturnType();
36643782 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
36653783 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
......@@ -3668,11 +3786,13 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
36683786
36693787 try self.register_manager.getReg(.rdi, null);
36703788 try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });
3671 self.register_manager.freezeRegs(&.{.rdi});
3789 const rdi_lock = self.register_manager.freezeRegAssumeUnused(.rdi);
36723790
36733791 info.return_value.stack_offset = stack_offset;
3674 }
3675 defer if (info.return_value == .stack_offset) self.register_manager.unfreezeRegs(&.{.rdi});
3792
3793 break :blk rdi_lock;
3794 } else null;
3795 defer if (rdi_lock) |reg| self.register_manager.unfreezeReg(reg);
36763796
36773797 for (args) |arg, arg_i| {
36783798 const mc_arg = info.args[arg_i];
......@@ -3891,11 +4011,16 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
38914011 const ret_ty = self.fn_type.fnReturnType();
38924012 switch (self.ret_mcv) {
38934013 .stack_offset => {
3894 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3895 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
4014 var reg_locks: [2]RegisterLock = undefined;
4015 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rcx }, &reg_locks);
4016 defer for (reg_locks) |reg| {
4017 self.register_manager.unfreezeReg(reg);
4018 };
4019
38964020 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3897 self.register_manager.freezeRegs(&.{reg});
3898 defer self.register_manager.unfreezeRegs(&.{reg});
4021 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
4022 defer self.register_manager.unfreezeReg(reg_lock);
4023
38994024 try self.genSetStack(ret_ty, 0, operand, .{
39004025 .source_stack_base = .rbp,
39014026 .dest_stack_base = reg,
......@@ -3926,11 +4051,16 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
39264051 const elem_ty = ptr_ty.elemType();
39274052 switch (self.ret_mcv) {
39284053 .stack_offset => {
3929 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3930 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
4054 var reg_locks: [2]RegisterLock = undefined;
4055 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rcx }, &reg_locks);
4056 defer for (reg_locks) |reg| {
4057 self.register_manager.unfreezeReg(reg);
4058 };
4059
39314060 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3932 self.register_manager.freezeRegs(&.{reg});
3933 defer self.register_manager.unfreezeRegs(&.{reg});
4061 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
4062 defer self.register_manager.unfreezeReg(reg_lock);
4063
39344064 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
39354065 .source_stack_base = .rbp,
39364066 .dest_stack_base = reg,
......@@ -3980,12 +4110,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
39804110 // Source operand can be an immediate, 8 bits or 32 bits.
39814111 // TODO look into reusing the operand
39824112 const lhs = try self.resolveInst(bin_op.lhs);
3983 lhs.freezeIfRegister(&self.register_manager);
3984 defer lhs.unfreezeIfRegister(&self.register_manager);
4113 const lhs_lock: ?RegisterLock = switch (lhs) {
4114 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
4115 else => null,
4116 };
4117 defer if (lhs_lock) |reg| self.register_manager.unfreezeReg(reg);
39854118
39864119 const dst_reg = try self.copyToTmpRegister(ty, lhs);
3987 self.register_manager.freezeRegs(&.{dst_reg});
3988 defer self.register_manager.unfreezeRegs(&.{dst_reg});
4120 const dst_reg_lock = self.register_manager.freezeRegAssumeUnused(dst_reg);
4121 defer self.register_manager.unfreezeReg(dst_reg_lock);
39894122
39904123 const dst_mcv = MCValue{ .register = dst_reg };
39914124
......@@ -4448,8 +4581,13 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
44484581 const un_op = self.air.instructions.items(.data)[inst].un_op;
44494582 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
44504583 const operand_ptr = try self.resolveInst(un_op);
4451 operand_ptr.freezeIfRegister(&self.register_manager);
4452 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4584
4585 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4586 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
4587 else => null,
4588 };
4589 defer if (operand_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
4590
44534591 const operand: MCValue = blk: {
44544592 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
44554593 // The MCValue that holds the pointer can be re-used as the value.
......@@ -4479,8 +4617,13 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
44794617 const un_op = self.air.instructions.items(.data)[inst].un_op;
44804618 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
44814619 const operand_ptr = try self.resolveInst(un_op);
4482 operand_ptr.freezeIfRegister(&self.register_manager);
4483 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4620
4621 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4622 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
4623 else => null,
4624 };
4625 defer if (operand_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
4626
44844627 const operand: MCValue = blk: {
44854628 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
44864629 // The MCValue that holds the pointer can be re-used as the value.
......@@ -4510,8 +4653,13 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
45104653 const un_op = self.air.instructions.items(.data)[inst].un_op;
45114654 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
45124655 const operand_ptr = try self.resolveInst(un_op);
4513 operand_ptr.freezeIfRegister(&self.register_manager);
4514 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4656
4657 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4658 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
4659 else => null,
4660 };
4661 defer if (operand_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
4662
45154663 const operand: MCValue = blk: {
45164664 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
45174665 // The MCValue that holds the pointer can be re-used as the value.
......@@ -4541,8 +4689,13 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
45414689 const un_op = self.air.instructions.items(.data)[inst].un_op;
45424690 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
45434691 const operand_ptr = try self.resolveInst(un_op);
4544 operand_ptr.freezeIfRegister(&self.register_manager);
4545 defer operand_ptr.unfreezeIfRegister(&self.register_manager);
4692
4693 const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4694 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
4695 else => null,
4696 };
4697 defer if (operand_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
4698
45464699 const operand: MCValue = blk: {
45474700 if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
45484701 // The MCValue that holds the pointer can be re-used as the value.
......@@ -4610,8 +4763,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
46104763 .register => |cond_reg| {
46114764 try self.spillCompareFlagsIfOccupied();
46124765
4613 self.register_manager.freezeRegs(&.{cond_reg});
4614 defer self.register_manager.unfreezeRegs(&.{cond_reg});
4766 const cond_reg_lock = self.register_manager.freezeReg(cond_reg);
4767 defer if (cond_reg_lock) |reg| self.register_manager.unfreezeReg(reg);
46154768
46164769 switch (case) {
46174770 .none => unreachable,
......@@ -4670,8 +4823,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
46704823
46714824 if (abi_size <= 8) {
46724825 const reg = try self.copyToTmpRegister(ty, condition);
4673 self.register_manager.freezeRegs(&.{reg});
4674 defer self.register_manager.unfreezeRegs(&.{reg});
4826 const reg_lock = self.register_manager.freezeRegAssumeUnused(reg);
4827 defer self.register_manager.unfreezeReg(reg_lock);
46754828 return self.genCondSwitchMir(ty, .{ .register = reg }, case);
46764829 }
46774830
......@@ -5158,8 +5311,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
51585311 .register_overflow_unsigned,
51595312 .register_overflow_signed,
51605313 => |reg| {
5161 self.register_manager.freezeRegs(&.{reg});
5162 defer self.register_manager.unfreezeRegs(&.{reg});
5314 const reg_lock = self.register_manager.freezeReg(reg);
5315 defer if (reg_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
51635316
51645317 const wrapped_ty = ty.structFieldType(0);
51655318 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg }, .{});
......@@ -5260,8 +5413,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
52605413
52615414 const base_reg = opts.dest_stack_base orelse .rbp;
52625415 if (!math.isPowerOfTwo(abi_size)) {
5263 self.register_manager.freezeRegs(&.{reg});
5264 defer self.register_manager.unfreezeRegs(&.{reg});
5416 const reg_lock = self.register_manager.freezeReg(reg);
5417 defer if (reg_lock) |reg_locked| self.register_manager.unfreezeReg(reg_locked);
52655418
52665419 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
52675420
......@@ -5350,13 +5503,26 @@ fn genInlineMemcpy(
53505503 len: MCValue,
53515504 opts: InlineMemcpyOpts,
53525505) InnerError!void {
5353 self.register_manager.freezeRegs(&.{ .rax, .rcx });
5506 try self.register_manager.getReg(.rax, null);
5507 try self.register_manager.getReg(.rcx, null);
53545508
5355 if (opts.source_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
5356 defer if (opts.source_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
5509 var reg_locks: [2]RegisterLock = undefined;
5510 self.register_manager.freezeRegsAssumeUnused(2, .{ .rax, .rcx }, &reg_locks);
5511 defer for (reg_locks) |reg| {
5512 self.register_manager.unfreezeReg(reg);
5513 };
53575514
5358 if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
5359 defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
5515 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
5516 self.register_manager.freezeReg(reg)
5517 else
5518 null;
5519 defer if (ssbase_lock) |reg| self.register_manager.unfreezeReg(reg);
5520
5521 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
5522 self.register_manager.freezeReg(reg)
5523 else
5524 null;
5525 defer if (dsbase_lock) |reg| self.register_manager.unfreezeReg(reg);
53605526
53615527 const dst_addr_reg = try self.register_manager.allocReg(null);
53625528 switch (dst_ptr) {
......@@ -5390,8 +5556,8 @@ fn genInlineMemcpy(
53905556 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
53915557 },
53925558 }
5393 self.register_manager.freezeRegs(&.{dst_addr_reg});
5394 defer self.register_manager.unfreezeRegs(&.{dst_addr_reg});
5559 const dst_addr_reg_lock = self.register_manager.freezeRegAssumeUnused(dst_addr_reg);
5560 defer self.register_manager.unfreezeReg(dst_addr_reg_lock);
53955561
53965562 const src_addr_reg = try self.register_manager.allocReg(null);
53975563 switch (src_ptr) {
......@@ -5425,18 +5591,13 @@ fn genInlineMemcpy(
54255591 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
54265592 },
54275593 }
5428 self.register_manager.freezeRegs(&.{src_addr_reg});
5429 defer self.register_manager.unfreezeRegs(&.{src_addr_reg});
5594 const src_addr_reg_lock = self.register_manager.freezeRegAssumeUnused(src_addr_reg);
5595 defer self.register_manager.unfreezeReg(src_addr_reg_lock);
54305596
54315597 const regs = try self.register_manager.allocRegs(2, .{ null, null });
54325598 const count_reg = regs[0].to64();
54335599 const tmp_reg = regs[1].to8();
54345600
5435 self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
5436
5437 try self.register_manager.getReg(.rax, null);
5438 try self.register_manager.getReg(.rcx, null);
5439
54405601 try self.genSetReg(Type.usize, count_reg, len);
54415602
54425603 // mov rcx, 0
......@@ -5540,7 +5701,9 @@ fn genInlineMemset(
55405701 len: MCValue,
55415702 opts: InlineMemcpyOpts,
55425703) InnerError!void {
5543 self.register_manager.freezeRegs(&.{.rax});
5704 try self.register_manager.getReg(.rax, null);
5705 const rax_lock = self.register_manager.freezeRegAssumeUnused(.rax);
5706 defer self.register_manager.unfreezeReg(rax_lock);
55445707
55455708 const addr_reg = try self.register_manager.allocReg(null);
55465709 switch (dst_ptr) {
......@@ -5574,11 +5737,8 @@ fn genInlineMemset(
55745737 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
55755738 },
55765739 }
5577 self.register_manager.freezeRegs(&.{addr_reg});
5578 defer self.register_manager.unfreezeRegs(&.{addr_reg});
5579
5580 self.register_manager.unfreezeRegs(&.{.rax});
5581 try self.register_manager.getReg(.rax, null);
5740 const addr_reg_lock = self.register_manager.freezeRegAssumeUnused(addr_reg);
5741 defer self.register_manager.unfreezeReg(addr_reg_lock);
55825742
55835743 try self.genSetReg(Type.usize, .rax, len);
55845744 try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });
......@@ -6017,16 +6177,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
60176177 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
60186178
60196179 const dst_ptr = try self.resolveInst(pl_op.operand);
6020 dst_ptr.freezeIfRegister(&self.register_manager);
6021 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
6180 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
6181 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
6182 else => null,
6183 };
6184 defer if (dst_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
60226185
60236186 const src_val = try self.resolveInst(extra.lhs);
6024 src_val.freezeIfRegister(&self.register_manager);
6025 defer src_val.unfreezeIfRegister(&self.register_manager);
6187 const src_val_lock: ?RegisterLock = switch (src_val) {
6188 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
6189 else => null,
6190 };
6191 defer if (src_val_lock) |reg| self.register_manager.unfreezeReg(reg);
60266192
60276193 const len = try self.resolveInst(extra.rhs);
6028 len.freezeIfRegister(&self.register_manager);
6029 defer len.unfreezeIfRegister(&self.register_manager);
6194 const len_lock: ?RegisterLock = switch (len) {
6195 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
6196 else => null,
6197 };
6198 defer if (len_lock) |reg| self.register_manager.unfreezeReg(reg);
60306199
60316200 try self.genInlineMemset(dst_ptr, src_val, len, .{});
60326201
......@@ -6038,17 +6207,26 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
60386207 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
60396208
60406209 const dst_ptr = try self.resolveInst(pl_op.operand);
6041 dst_ptr.freezeIfRegister(&self.register_manager);
6042 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
6210 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
6211 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
6212 else => null,
6213 };
6214 defer if (dst_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
60436215
60446216 const src_ty = self.air.typeOf(extra.lhs);
60456217 const src_ptr = try self.resolveInst(extra.lhs);
6046 src_ptr.freezeIfRegister(&self.register_manager);
6047 defer src_ptr.unfreezeIfRegister(&self.register_manager);
6218 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
6219 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
6220 else => null,
6221 };
6222 defer if (src_ptr_lock) |reg| self.register_manager.unfreezeReg(reg);
60486223
60496224 const len = try self.resolveInst(extra.rhs);
6050 len.freezeIfRegister(&self.register_manager);
6051 defer len.unfreezeIfRegister(&self.register_manager);
6225 const len_lock: ?RegisterLock = switch (len) {
6226 .register => |reg| self.register_manager.freezeRegAssumeUnused(reg),
6227 else => null,
6228 };
6229 defer if (len_lock) |reg| self.register_manager.unfreezeReg(reg);
60526230
60536231 // TODO Is this the only condition for pointer dereference for memcpy?
60546232 const src: MCValue = blk: {
......@@ -6070,8 +6248,11 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
60706248 else => break :blk src_ptr,
60716249 }
60726250 };
6073 src.freezeIfRegister(&self.register_manager);
6074 defer src.unfreezeIfRegister(&self.register_manager);
6251 const src_lock: ?RegisterLock = switch (src) {
6252 .register => |reg| self.register_manager.freezeReg(reg),
6253 else => null,
6254 };
6255 defer if (src_lock) |reg| self.register_manager.unfreezeReg(reg);
60756256
60766257 try self.genInlineMemcpy(dst_ptr, src, len, .{});
60776258
src/register_manager.zig+49-22
......@@ -116,23 +116,52 @@ pub fn RegisterManager(
116116 return self.frozen_registers & mask != 0;
117117 }
118118
119 /// Prevents the registers from being allocated until they are
120 /// unfrozen again
121 pub fn freezeRegs(self: *Self, regs: []const Register) void {
122 for (regs) |reg| {
123 const mask = getRegisterMask(reg) orelse continue;
124 self.frozen_registers |= mask;
125 }
119 pub const RegisterLock = struct {
120 register: Register,
121 };
122
123 /// Prevents the register from being allocated until they are
124 /// unfrozen again.
125 /// Returns `RegisterLock` if the register was not already
126 /// frozen, or `null` otherwise.
127 /// Only the owner of the `RegisterLock` can unfreeze the
128 /// register later.
129 pub fn freezeReg(self: *Self, reg: Register) ?RegisterLock {
130 if (self.isRegFrozen(reg)) return null;
131 const mask = getRegisterMask(reg) orelse return null;
132 self.frozen_registers |= mask;
133 return RegisterLock{ .register = reg };
134 }
135
136 /// Like `freezeReg` but asserts the register was unused always
137 /// returning a valid lock.
138 pub fn freezeRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
139 assert(!self.isRegFrozen(reg));
140 const mask = getRegisterMask(reg) orelse unreachable;
141 self.frozen_registers |= mask;
142 return RegisterLock{ .register = reg };
126143 }
127144
128 /// Enables the allocation of the registers
129 pub fn unfreezeRegs(self: *Self, regs: []const Register) void {
130 for (regs) |reg| {
131 const mask = getRegisterMask(reg) orelse continue;
132 self.frozen_registers &= ~mask;
145 /// Like `freezeRegAssumeUnused` but locks multiple registers.
146 pub fn freezeRegsAssumeUnused(
147 self: *Self,
148 comptime count: comptime_int,
149 regs: [count]Register,
150 buf: *[count]RegisterLock,
151 ) void {
152 for (&regs) |reg, i| {
153 buf[i] = self.freezeRegAssumeUnused(reg);
133154 }
134155 }
135156
157 /// Unfreezes the register allowing its re-allocation and re-use.
158 /// Requires `RegisterLock` to unfreeze a register.
159 /// Call `freezeReg` to obtain the lock first.
160 pub fn unfreezeReg(self: *Self, lock: RegisterLock) void {
161 const mask = getRegisterMask(lock.register) orelse return;
162 self.frozen_registers &= ~mask;
163 }
164
136165 /// Returns true when at least one register is frozen
137166 pub fn frozenRegsExist(self: Self) bool {
138167 return self.frozen_registers != 0;
......@@ -419,8 +448,8 @@ test "allocReg: spilling" {
419448 // Frozen registers
420449 function.register_manager.freeReg(.r3);
421450 {
422 function.register_manager.freezeRegs(&.{.r2});
423 defer function.register_manager.unfreezeRegs(&.{.r2});
451 const lock = function.register_manager.freezeReg(.r2);
452 defer if (lock) |reg| function.register_manager.unfreezeReg(reg);
424453
425454 try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
426455 }
......@@ -447,8 +476,8 @@ test "tryAllocRegs" {
447476 function.register_manager.freeReg(.r2);
448477 function.register_manager.freeReg(.r3);
449478 {
450 function.register_manager.freezeRegs(&.{.r1});
451 defer function.register_manager.unfreezeRegs(&.{.r1});
479 const lock = function.register_manager.freezeReg(.r1);
480 defer if (lock) |reg| function.register_manager.unfreezeReg(reg);
452481
453482 try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
454483 }
......@@ -486,8 +515,8 @@ test "allocRegs: normal usage" {
486515 // contain any valuable data anymore and can be reused. For an
487516 // example of that, see `selectively reducing register
488517 // pressure`.
489 function.register_manager.freezeRegs(&.{result_reg});
490 defer function.register_manager.unfreezeRegs(&.{result_reg});
518 const lock = function.register_manager.freezeReg(result_reg);
519 defer if (lock) |reg| function.register_manager.unfreezeReg(reg);
491520
492521 const regs = try function.register_manager.allocRegs(2, .{ null, null });
493522 try function.genAdd(result_reg, regs[0], regs[1]);
......@@ -507,16 +536,14 @@ test "allocRegs: selectively reducing register pressure" {
507536 {
508537 const result_reg: MockRegister2 = .r1;
509538
510 function.register_manager.freezeRegs(&.{result_reg});
511 defer function.register_manager.unfreezeRegs(&.{result_reg});
539 const lock = function.register_manager.freezeReg(result_reg);
512540
513541 // Here, we don't defer unfreeze because we manually unfreeze
514542 // after genAdd
515543 const regs = try function.register_manager.allocRegs(2, .{ null, null });
516 function.register_manager.freezeRegs(&.{result_reg});
517544
518545 try function.genAdd(result_reg, regs[0], regs[1]);
519 function.register_manager.unfreezeRegs(&regs);
546 function.register_manager.unfreezeReg(lock.?);
520547
521548 const extra_summand_reg = try function.register_manager.allocReg(null);
522549 try function.genAdd(result_reg, result_reg, extra_summand_reg);
test/behavior/align.zig+1
......@@ -7,6 +7,7 @@ var foo: u8 align(4) = 100;
77
88test "global variable alignment" {
99 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1011
1112 comptime try expect(@typeInfo(@TypeOf(&foo)).Pointer.alignment == 4);
1213 comptime try expect(@TypeOf(&foo) == *align(4) u8);