Commit 9416b4d993e5e386c9208e9a52e4e881eee8cf72 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2022-05-08 11:20:14+02:00
committer	GitHub <noreply@github.com> 2022-05-08 11:20:14+02:00
log	9416b4d993e5e386c9208e9a52e4e881eee8cf72
tree	118a802c038879f57888c8d3643c233f031c449c
parent	aa05cd48097b42d0c52ff5e0f3dd1ece68d1cdcd
parent	6bf67eada47b7c6f4819a759268503658c97e9ec
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11608 from ziglang/stage2-regalloc

7 files changed, 1092 insertions(+), 686 deletions(-)

src
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - x86_64
    CodeGen.zig
- register_manager.zig
test
- behavior
  - align.zig
  - byval_arg_var.zig

src/arch/aarch64/CodeGen.zig+151-68

 @@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
 const build_options = @import("build_options");
 const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
 const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
+const RegisterLock = RegisterManager.RegisterLock;
 const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
 const FnResult = @import("../../codegen.zig").FnResult;
 @@ -727,7 +728,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             // zig fmt: on
         }
-        assert(!self.register_manager.frozenRegsExist());
+        assert(!self.register_manager.lockedRegsExist());
         if (std.debug.runtime_safety) {
             if (self.air_bookkeeping < old_air_bookkeeping + 1) {
 @@ -910,16 +911,16 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
 fn spillCompareFlagsIfOccupied(self: *Self) !void {
     if (self.compare_flags_inst) |inst_to_save| {
         const mcv = self.getResolvedInstValue(inst_to_save);
-        switch (mcv) {
+        const new_mcv = switch (mcv) {
             .compare_flags_signed,
             .compare_flags_unsigned,
+            => try self.allocRegOrMem(inst_to_save, true),
             .register_c_flag,
             .register_v_flag,
-            => {},
+            => try self.allocRegOrMem(inst_to_save, false),
             else => unreachable, // mcv doesn't occupy the compare flags
-        }
+        };
-        const new_mcv = try self.allocRegOrMem(inst_to_save, true);
         try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
         log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
 @@ -927,6 +928,15 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
         try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
         self.compare_flags_inst = null;
+        // TODO consolidate with register manager and spillInstruction
+        // this call should really belong in the register manager!
+        switch (mcv) {
+            .register_c_flag,
+            .register_v_flag,
+            => |reg| self.register_manager.freeReg(reg),
+            else => {},
+        }
     }
 }
 @@ -1048,8 +1058,8 @@ fn trunc(
                 }
             },
         };
-        self.register_manager.freezeRegs(&.{operand_reg});
+        const lock = self.register_manager.lockReg(operand_reg);
-        defer self.register_manager.unfreezeRegs(&.{operand_reg});
+        defer if (lock) |reg| self.register_manager.unlockReg(reg);
         const dest_reg = if (maybe_inst) |inst| blk: {
             const ty_op = self.air.instructions.items(.data)[inst].ty_op;
 @@ -1135,8 +1145,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
                             .register => |r| r,
                             else => try self.copyToTmpRegister(operand_ty, operand),
                         };
-                        self.register_manager.freezeRegs(&.{op_reg});
+                        const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
-                        defer self.register_manager.unfreezeRegs(&.{op_reg});
+                        defer self.register_manager.unlockReg(reg_lock);
                         const dest_reg = blk: {
                             if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
 @@ -1168,8 +1178,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
                                 .register => |r| r,
                                 else => try self.copyToTmpRegister(operand_ty, operand),
                             };
-                            self.register_manager.freezeRegs(&.{op_reg});
+                            const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
-                            defer self.register_manager.unfreezeRegs(&.{op_reg});
+                            defer self.register_manager.unlockReg(reg_lock);
                             const dest_reg = blk: {
                                 if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
 @@ -1257,8 +1267,17 @@ fn binOpRegister(
     const lhs_is_register = lhs == .register;
     const rhs_is_register = rhs == .register;
-    if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
+    const lhs_lock: ?RegisterLock = if (lhs_is_register)
-    if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
+        self.register_manager.lockReg(lhs.register)
+    else
+        null;
+    defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
+    const rhs_lock: ?RegisterLock = if (rhs_is_register)
+        self.register_manager.lockReg(rhs.register)
+    else
+        null;
+    defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
 @@ -1270,13 +1289,13 @@ fn binOpRegister(
         const raw_reg = try self.register_manager.allocReg(track_inst);
         const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
-        self.register_manager.freezeRegs(&.{reg});
         if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
         break :blk reg;
     };
-    defer self.register_manager.unfreezeRegs(&.{lhs_reg});
+    const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
+    defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
     const rhs_reg = if (rhs_is_register) rhs.register else blk: {
         const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
 @@ -1286,13 +1305,13 @@ fn binOpRegister(
         const raw_reg = try self.register_manager.allocReg(track_inst);
         const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
-        self.register_manager.freezeRegs(&.{reg});
         if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
         break :blk reg;
     };
-    defer self.register_manager.unfreezeRegs(&.{rhs_reg});
+    const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
+    defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
     const dest_reg = switch (mir_tag) {
         .cmp_shifted_register => undefined, // cmp has no destination register
 @@ -1394,7 +1413,11 @@ fn binOpImmediate(
 ) !MCValue {
     const lhs_is_register = lhs == .register;
-    if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
+    const lhs_lock: ?RegisterLock = if (lhs_is_register)
+        self.register_manager.lockReg(lhs.register)
+    else
+        null;
+    defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
 @@ -1408,13 +1431,13 @@ fn binOpImmediate(
         const raw_reg = try self.register_manager.allocReg(track_inst);
         const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
-        self.register_manager.freezeRegs(&.{reg});
         if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
         break :blk reg;
     };
-    defer self.register_manager.unfreezeRegs(&.{lhs_reg});
+    const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
+    defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
     const dest_reg = switch (mir_tag) {
         .cmp_immediate => undefined, // cmp has no destination register
 @@ -1758,7 +1781,10 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void {
     const lhs_ty = self.air.typeOf(bin_op.lhs);
     const rhs_ty = self.air.typeOf(bin_op.rhs);
-    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
+    const result: MCValue = if (self.liveness.isUnused(inst))
+        .dead
+    else
+        try self.binOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 @@ -1815,13 +1841,13 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
                         };
                         const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
                         const dest_reg = dest.register;
-                        self.register_manager.freezeRegs(&.{dest_reg});
+                        const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
-                        defer self.register_manager.unfreezeRegs(&.{dest_reg});
+                        defer self.register_manager.unlockReg(dest_reg_lock);
                         const raw_truncated_reg = try self.register_manager.allocReg(null);
                         const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*));
-                        self.register_manager.freezeRegs(&.{truncated_reg});
+                        const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
-                        defer self.register_manager.unfreezeRegs(&.{truncated_reg});
+                        defer self.register_manager.unlockReg(truncated_reg_lock);
                         // sbfx/ubfx truncated, dest, #0, #bits
                         try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
 @@ -1922,12 +1948,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                     const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
                     const dest_reg = dest.register;
-                    self.register_manager.freezeRegs(&.{dest_reg});
+                    const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
-                    defer self.register_manager.unfreezeRegs(&.{dest_reg});
+                    defer self.register_manager.unlockReg(dest_reg_lock);
                     const truncated_reg = try self.register_manager.allocReg(null);
-                    self.register_manager.freezeRegs(&.{truncated_reg});
+                    const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
-                    defer self.register_manager.unfreezeRegs(&.{truncated_reg});
+                    defer self.register_manager.unlockReg(truncated_reg_lock);
                     try self.truncRegister(
                         dest_reg.to32(),
 @@ -1977,36 +2003,44 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                     const lhs_is_register = lhs == .register;
                     const rhs_is_register = rhs == .register;
-                    if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
+                    const lhs_lock: ?RegisterLock = if (lhs_is_register)
-                    if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
+                        self.register_manager.lockRegAssumeUnused(lhs.register)
+                    else
+                        null;
+                    defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
+                    const rhs_lock: ?RegisterLock = if (rhs_is_register)
+                        self.register_manager.lockRegAssumeUnused(rhs.register)
+                    else
+                        null;
+                    defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
                     const lhs_reg = if (lhs_is_register) lhs.register else blk: {
                         const raw_reg = try self.register_manager.allocReg(null);
                         const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
-                        self.register_manager.freezeRegs(&.{reg});
                         break :blk reg;
                     };
-                    defer self.register_manager.unfreezeRegs(&.{lhs_reg});
+                    const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
+                    defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
                     const rhs_reg = if (rhs_is_register) rhs.register else blk: {
                         const raw_reg = try self.register_manager.allocReg(null);
                         const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*));
-                        self.register_manager.freezeRegs(&.{reg});
                         break :blk reg;
                     };
-                    defer self.register_manager.unfreezeRegs(&.{rhs_reg});
+                    const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
+                    defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
                     if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
                     if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
-                    // TODO reuse operands
                     const dest_reg = blk: {
                         const raw_reg = try self.register_manager.allocReg(null);
                         const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
-                        self.register_manager.freezeRegs(&.{reg});
                         break :blk reg;
                     };
-                    defer self.register_manager.unfreezeRegs(&.{dest_reg});
+                    const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
+                    defer self.register_manager.unlockReg(dest_reg_lock);
                     switch (int_info.signedness) {
                         .signed => {
 @@ -2021,8 +2055,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                             });
                             const dest_high_reg = try self.register_manager.allocReg(null);
-                            self.register_manager.freezeRegs(&.{dest_high_reg});
+                            const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg);
-                            defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
+                            defer self.register_manager.unlockReg(dest_high_reg_lock);
                             // smulh dest_high, lhs, rhs
                             _ = try self.addInst(.{
 @@ -2071,8 +2105,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                         },
                         .unsigned => {
                             const dest_high_reg = try self.register_manager.allocReg(null);
-                            self.register_manager.freezeRegs(&.{dest_high_reg});
+                            const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg);
-                            defer self.register_manager.unfreezeRegs(&.{dest_high_reg});
+                            defer self.register_manager.unlockReg(dest_high_reg_lock);
                             // umulh dest_high, lhs, rhs
                             _ = try self.addInst(.{
 @@ -2127,8 +2161,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                     }
                     const truncated_reg = try self.register_manager.allocReg(null);
-                    self.register_manager.freezeRegs(&.{truncated_reg});
+                    const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
-                    defer self.register_manager.unfreezeRegs(&.{truncated_reg});
+                    defer self.register_manager.unlockReg(truncated_reg_lock);
                     try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
 @@ -2168,14 +2202,20 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                 if (int_info.bits <= 64) {
                     const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
-                    if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});
+                    const lhs_lock: ?RegisterLock = if (lhs == .register)
-                    defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});
+                        self.register_manager.lockRegAssumeUnused(lhs.register)
+                    else
+                        null;
+                    defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
                     try self.spillCompareFlagsIfOccupied();
                     self.compare_flags_inst = null;
                     // lsl dest, lhs, rhs
                     const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
+                    const dest_reg = dest.register;
+                    const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
+                    defer self.register_manager.unlockReg(dest_reg_lock);
                     // asr/lsr reconstructed, dest, rhs
                     const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);
 @@ -2184,7 +2224,9 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                     _ = try self.binOp(.cmp_eq, null, lhs, reconstructed, lhs_ty, lhs_ty);
                     try self.genSetStack(lhs_ty, stack_offset, dest);
-                    try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ .compare_flags_unsigned = .neq });
+                    try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
+                        .compare_flags_unsigned = .neq,
+                    });
                     break :result MCValue{ .stack_offset = stack_offset };
                 } else {
 @@ -2411,14 +2453,18 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
         var buf: Type.SlicePtrFieldTypeBuffer = undefined;
         const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
-        if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register});
+        const index_lock: ?RegisterLock = if (index_is_register)
-        defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register});
+            self.register_manager.lockRegAssumeUnused(index_mcv.register)
+        else
+            null;
+        defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
         const base_mcv: MCValue = switch (slice_mcv) {
             .stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
             else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
         };
-        self.register_manager.freezeRegs(&.{base_mcv.register});
+        const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register);
+        defer self.register_manager.unlockReg(base_lock);
         switch (elem_size) {
             else => {
 @@ -2559,8 +2605,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
         .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
         .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
         .register => |addr_reg| {
-            self.register_manager.freezeRegs(&.{addr_reg});
+            const addr_reg_lock = self.register_manager.lockReg(addr_reg);
-            defer self.register_manager.unfreezeRegs(&.{addr_reg});
+            defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
             switch (dst_mcv) {
                 .dead => unreachable,
 @@ -2573,16 +2619,18 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
                     if (elem_size <= 8) {
                         const raw_tmp_reg = try self.register_manager.allocReg(null);
                         const tmp_reg = registerAlias(raw_tmp_reg, elem_size);
-                        self.register_manager.freezeRegs(&.{tmp_reg});
+                        const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
-                        defer self.register_manager.unfreezeRegs(&.{tmp_reg});
+                        defer self.register_manager.unlockReg(tmp_reg_lock);
                         try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
                         try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
                     } else {
                         // TODO optimize the register allocation
                         const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
-                        self.register_manager.freezeRegs(&regs);
+                        const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
-                        defer self.register_manager.unfreezeRegs(&regs);
+                        defer for (regs_locks) |reg| {
+                            self.register_manager.unlockReg(reg);
+                        };
                         const src_reg = addr_reg;
                         const dst_reg = regs[0];
 @@ -2784,8 +2832,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             try self.genSetStack(value_ty, off, value);
         },
         .register => |addr_reg| {
-            self.register_manager.freezeRegs(&.{addr_reg});
+            const addr_reg_lock = self.register_manager.lockReg(addr_reg);
-            defer self.register_manager.unfreezeRegs(&.{addr_reg});
+            defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
             switch (value) {
                 .register => |value_reg| {
 @@ -2795,8 +2843,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                     if (abi_size <= 8) {
                         const raw_tmp_reg = try self.register_manager.allocReg(null);
                         const tmp_reg = registerAlias(raw_tmp_reg, abi_size);
-                        self.register_manager.freezeRegs(&.{tmp_reg});
+                        const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
-                        defer self.register_manager.unfreezeRegs(&.{tmp_reg});
+                        defer self.register_manager.unlockReg(tmp_reg_lock);
                         try self.genSetReg(value_ty, tmp_reg, value);
                         try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
 @@ -2856,12 +2904,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
                 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
                     .immediate = struct_field_offset,
                 });
-                self.register_manager.freezeRegs(&.{offset_reg});
+                const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
-                defer self.register_manager.unfreezeRegs(&.{offset_reg});
+                defer self.register_manager.unlockReg(offset_reg_lock);
                 const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
-                self.register_manager.freezeRegs(&.{addr_reg});
+                const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
-                defer self.register_manager.unfreezeRegs(&.{addr_reg});
+                defer self.register_manager.unlockReg(addr_reg_lock);
                 const dest = try self.binOp(
                     .add,
 @@ -3369,6 +3417,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
     const parent_compare_flags_inst = self.compare_flags_inst;
     try self.branch_stack.append(.{});
+    errdefer {
+        _ = self.branch_stack.pop();
+    }
     try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
     for (liveness_condbr.then_deaths) |operand| {
 @@ -3955,8 +4006,38 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
         },
         .register_c_flag,
         .register_v_flag,
-        => {
+        => |reg| {
-            return self.fail("TODO implement genSetStack {}", .{mcv});
+            const reg_lock = self.register_manager.lockReg(reg);
+            defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
+            const wrapped_ty = ty.structFieldType(0);
+            try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg });
+            const overflow_bit_ty = ty.structFieldType(1);
+            const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
+            const raw_cond_reg = try self.register_manager.allocReg(null);
+            const cond_reg = registerAlias(
+                raw_cond_reg,
+                @intCast(u32, overflow_bit_ty.abiSize(self.target.*)),
+            );
+            // C flag: cset reg, cs
+            // V flag: cset reg, vs
+            _ = try self.addInst(.{
+                .tag = .cset,
+                .data = .{ .r_cond = .{
+                    .rd = cond_reg,
+                    .cond = switch (mcv) {
+                        .register_c_flag => .cs,
+                        .register_v_flag => .vs,
+                        else => unreachable,
+                    },
+                } },
+            });
+            try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
+                .register = cond_reg,
+            });
         },
         .got_load,
         .direct_load,
 @@ -3983,8 +4064,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
                 // TODO call extern memcpy
                 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null });
-                self.register_manager.freezeRegs(&regs);
+                const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs);
-                defer self.register_manager.unfreezeRegs(&regs);
+                defer for (regs_locks) |reg| {
+                    self.register_manager.unlockReg(reg);
+                };
                 const src_reg = regs[0];
                 const dst_reg = regs[1];

src/arch/arm/CodeGen.zig+146-82

...	@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);	...	@@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
23	const build_options = @import("build_options");	23	const build_options = @import("build_options");
24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;	24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
25	const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);	25	const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
		26	const RegisterLock = RegisterManager.RegisterLock;
26		27
27	const FnResult = @import("../../codegen.zig").FnResult;	28	const FnResult = @import("../../codegen.zig").FnResult;
28	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;	29	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
...	@@ -734,7 +735,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -734,7 +735,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
734	// zig fmt: on	735	// zig fmt: on
735	}	736	}
736		737
737	assert(!self.register_manager.frozenRegsExist());	738	assert(!self.register_manager.lockedRegsExist());
738		739
739	if (std.debug.runtime_safety) {	740	if (std.debug.runtime_safety) {
740	if (self.air_bookkeeping < old_air_bookkeeping + 1) {	741	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
...	@@ -897,16 +898,16 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void	...	@@ -897,16 +898,16 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
897	fn spillCompareFlagsIfOccupied(self: *Self) !void {	898	fn spillCompareFlagsIfOccupied(self: *Self) !void {
898	if (self.compare_flags_inst) \|inst_to_save\| {	899	if (self.compare_flags_inst) \|inst_to_save\| {
899	const mcv = self.getResolvedInstValue(inst_to_save);	900	const mcv = self.getResolvedInstValue(inst_to_save);
900	switch (mcv) {	901	const new_mcv = switch (mcv) {
901	.compare_flags_signed,	902	.compare_flags_signed,
902	.compare_flags_unsigned,	903	.compare_flags_unsigned,
		904	=> try self.allocRegOrMem(inst_to_save, true),
903	.register_c_flag,	905	.register_c_flag,
904	.register_v_flag,	906	.register_v_flag,
905	=> {},	907	=> try self.allocRegOrMem(inst_to_save, false),
906	else => unreachable, // mcv doesn't occupy the compare flags	908	else => unreachable, // mcv doesn't occupy the compare flags
907	}	909	};
908		910
909	const new_mcv = try self.allocRegOrMem(inst_to_save, true);
910	try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);	911	try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
911	log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });	912	log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
912		913
...	@@ -914,6 +915,15 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {	...	@@ -914,6 +915,15 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void {
914	try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);	915	try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
915		916
916	self.compare_flags_inst = null;	917	self.compare_flags_inst = null;
		918
		919	// TODO consolidate with register manager and spillInstruction
		920	// this call should really belong in the register manager!
		921	switch (mcv) {
		922	.register_c_flag,
		923	.register_v_flag,
		924	=> \|reg\| self.register_manager.freeReg(reg),
		925	else => {},
		926	}
917	}	927	}
918	}	928	}
919		929
...	@@ -1038,8 +1048,8 @@ fn trunc(	...	@@ -1038,8 +1048,8 @@ fn trunc(
1038	}	1048	}
1039	},	1049	},
1040	};	1050	};
1041	self.register_manager.freezeRegs(&.{operand_reg});	1051	const operand_reg_lock = self.register_manager.lockReg(operand_reg);
1042	defer self.register_manager.unfreezeRegs(&.{operand_reg});	1052	defer if (operand_reg_lock) \|reg\| self.register_manager.unlockReg(reg);
1043		1053
1044	const dest_reg = if (maybe_inst) \|inst\| blk: {	1054	const dest_reg = if (maybe_inst) \|inst\| blk: {
1045	const ty_op = self.air.instructions.items(.data)[inst].ty_op;	1055	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
...	@@ -1127,8 +1137,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1127,8 +1137,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1127	.register => \|r\| r,	1137	.register => \|r\| r,
1128	else => try self.copyToTmpRegister(operand_ty, operand),	1138	else => try self.copyToTmpRegister(operand_ty, operand),
1129	};	1139	};
1130	self.register_manager.freezeRegs(&.{op_reg});	1140	const op_reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1131	defer self.register_manager.unfreezeRegs(&.{op_reg});	1141	defer self.register_manager.unlockReg(op_reg_lock);
1132		1142
1133	const dest_reg = blk: {	1143	const dest_reg = blk: {
1134	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {	1144	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
...	@@ -1157,8 +1167,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1157,8 +1167,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1157	.register => \|r\| r,	1167	.register => \|r\| r,
1158	else => try self.copyToTmpRegister(operand_ty, operand),	1168	else => try self.copyToTmpRegister(operand_ty, operand),
1159	};	1169	};
1160	self.register_manager.freezeRegs(&.{op_reg});	1170	const op_reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
1161	defer self.register_manager.unfreezeRegs(&.{op_reg});	1171	defer self.register_manager.unlockReg(op_reg_lock);
1162		1172
1163	const dest_reg = blk: {	1173	const dest_reg = blk: {
1164	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {	1174	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
...	@@ -1218,15 +1228,15 @@ fn minMax(	...	@@ -1218,15 +1228,15 @@ fn minMax(
1218	.register => \|r\| r,	1228	.register => \|r\| r,
1219	else => try self.copyToTmpRegister(lhs_ty, lhs),	1229	else => try self.copyToTmpRegister(lhs_ty, lhs),
1220	};	1230	};
1221	self.register_manager.freezeRegs(&.{lhs_reg});	1231	const lhs_reg_lock = self.register_manager.lockReg(lhs_reg);
1222	defer self.register_manager.unfreezeRegs(&.{lhs_reg});	1232	defer if (lhs_reg_lock) \|reg\| self.register_manager.unlockReg(reg);
1223		1233
1224	const rhs_reg = switch (rhs) {	1234	const rhs_reg = switch (rhs) {
1225	.register => \|r\| r,	1235	.register => \|r\| r,
1226	else => try self.copyToTmpRegister(rhs_ty, rhs),	1236	else => try self.copyToTmpRegister(rhs_ty, rhs),
1227	};	1237	};
1228	self.register_manager.freezeRegs(&.{rhs_reg});	1238	const rhs_reg_lock = self.register_manager.lockReg(rhs_reg);
1229	defer self.register_manager.unfreezeRegs(&.{rhs_reg});	1239	defer if (rhs_reg_lock) \|reg\| self.register_manager.unlockReg(reg);
1230		1240
1231	const dest_reg = if (maybe_inst) \|inst\| blk: {	1241	const dest_reg = if (maybe_inst) \|inst\| blk: {
1232	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1242	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
...	@@ -1392,12 +1402,12 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1392,12 +1402,12 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void {
1392	};	1402	};
1393	const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);	1403	const dest = try self.binOp(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
1394	const dest_reg = dest.register;	1404	const dest_reg = dest.register;
1395	self.register_manager.freezeRegs(&.{dest_reg});	1405	const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1396	defer self.register_manager.unfreezeRegs(&.{dest_reg});	1406	defer self.register_manager.unlockReg(dest_reg_lock);
1397		1407
1398	const truncated_reg = try self.register_manager.allocReg(null);	1408	const truncated_reg = try self.register_manager.allocReg(null);
1399	self.register_manager.freezeRegs(&.{truncated_reg});	1409	const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1400	defer self.register_manager.unfreezeRegs(&.{truncated_reg});	1410	defer self.register_manager.unlockReg(truncated_reg_lock);
1401		1411
1402	// sbfx/ubfx truncated, dest, #0, #bits	1412	// sbfx/ubfx truncated, dest, #0, #bits
1403	try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);	1413	try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
...	@@ -1493,12 +1503,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1493,12 +1503,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1493		1503
1494	const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);	1504	const dest = try self.binOpRegister(base_tag, null, lhs, rhs, lhs_ty, rhs_ty);
1495	const dest_reg = dest.register;	1505	const dest_reg = dest.register;
1496	self.register_manager.freezeRegs(&.{dest_reg});	1506	const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1497	defer self.register_manager.unfreezeRegs(&.{dest_reg});	1507	defer self.register_manager.unlockReg(dest_reg_lock);
1498		1508
1499	const truncated_reg = try self.register_manager.allocReg(null);	1509	const truncated_reg = try self.register_manager.allocReg(null);
1500	self.register_manager.freezeRegs(&.{truncated_reg});	1510	const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1501	defer self.register_manager.unfreezeRegs(&.{truncated_reg});	1511	defer self.register_manager.unlockReg(truncated_reg_lock);
1502		1512
1503	// sbfx/ubfx truncated, dest, #0, #bits	1513	// sbfx/ubfx truncated, dest, #0, #bits
1504	try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);	1514	try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);
...	@@ -1526,28 +1536,31 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1526,28 +1536,31 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1526	const lhs_is_register = lhs == .register;	1536	const lhs_is_register = lhs == .register;
1527	const rhs_is_register = rhs == .register;	1537	const rhs_is_register = rhs == .register;
1528		1538
1529	if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});	1539	const lhs_lock: ?RegisterLock = if (lhs_is_register)
1530	if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});	1540	self.register_manager.lockReg(lhs.register)
1531		1541	else
1532	const lhs_reg = if (lhs_is_register) lhs.register else blk: {	1542	null;
1533	const reg = try self.register_manager.allocReg(null);	1543	defer if (lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
1534	self.register_manager.freezeRegs(&.{reg});
1535
1536	break :blk reg;
1537	};
1538	defer self.register_manager.unfreezeRegs(&.{lhs_reg});
1539		1544
1540	const rhs_reg = if (rhs_is_register) rhs.register else blk: {	1545	const lhs_reg = if (lhs_is_register)
1541	const reg = try self.register_manager.allocReg(null);	1546	lhs.register
1542	self.register_manager.freezeRegs(&.{reg});	1547	else
		1548	try self.register_manager.allocReg(null);
		1549	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
		1550	defer if (new_lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
1543		1551
1544	break :blk reg;	1552	const rhs_reg = if (rhs_is_register)
1545	};	1553	rhs.register
1546	defer self.register_manager.unfreezeRegs(&.{rhs_reg});	1554	else
		1555	try self.register_manager.allocReg(null);
		1556	const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
		1557	defer if (new_rhs_lock) \|reg\| self.register_manager.unlockReg(reg);
1547		1558
1548	const dest_regs = try self.register_manager.allocRegs(2, .{ null, null });	1559	const dest_regs = try self.register_manager.allocRegs(2, .{ null, null });
1549	self.register_manager.freezeRegs(&dest_regs);	1560	const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs);
1550	defer self.register_manager.unfreezeRegs(&dest_regs);	1561	defer for (dest_regs_locks) \|reg\| {
		1562	self.register_manager.unlockReg(reg);
		1563	};
1551	const rdlo = dest_regs[0];	1564	const rdlo = dest_regs[0];
1552	const rdhi = dest_regs[1];	1565	const rdhi = dest_regs[1];
1553		1566
...	@@ -1555,8 +1568,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1555,8 +1568,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1555	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);	1568	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
1556		1569
1557	const truncated_reg = try self.register_manager.allocReg(null);	1570	const truncated_reg = try self.register_manager.allocReg(null);
1558	self.register_manager.freezeRegs(&.{truncated_reg});	1571	const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1559	defer self.register_manager.unfreezeRegs(&.{truncated_reg});	1572	defer self.register_manager.unlockReg(truncated_reg_lock);
1560		1573
1561	_ = try self.addInst(.{	1574	_ = try self.addInst(.{
1562	.tag = base_tag,	1575	.tag = base_tag,
...	@@ -1648,14 +1661,20 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1648,14 +1661,20 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1648	if (int_info.bits <= 32) {	1661	if (int_info.bits <= 32) {
1649	const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);	1662	const stack_offset = try self.allocMem(inst, tuple_size, tuple_align);
1650		1663
1651	if (lhs == .register) self.register_manager.freezeRegs(&.{lhs.register});	1664	const lhs_lock: ?RegisterLock = if (lhs == .register)
1652	defer if (lhs == .register) self.register_manager.unfreezeRegs(&.{lhs.register});	1665	self.register_manager.lockRegAssumeUnused(lhs.register)
		1666	else
		1667	null;
		1668	defer if (lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
1653		1669
1654	try self.spillCompareFlagsIfOccupied();	1670	try self.spillCompareFlagsIfOccupied();
1655	self.compare_flags_inst = null;	1671	self.compare_flags_inst = null;
1656		1672
1657	// lsl dest, lhs, rhs	1673	// lsl dest, lhs, rhs
1658	const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);	1674	const dest = try self.binOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
		1675	const dest_reg = dest.register;
		1676	const dest_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
		1677	defer self.register_manager.unlockReg(dest_lock);
1659		1678
1660	// asr/lsr reconstructed, dest, rhs	1679	// asr/lsr reconstructed, dest, rhs
1661	const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);	1680	const reconstructed = try self.binOp(.shr, null, dest, rhs, lhs_ty, rhs_ty);
...	@@ -1939,8 +1958,11 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1939,8 +1958,11 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1939	var buf: Type.SlicePtrFieldTypeBuffer = undefined;	1958	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
1940	const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);	1959	const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
1941		1960
1942	if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register});	1961	const index_lock: ?RegisterLock = if (index_is_register)
1943	defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register});	1962	self.register_manager.lockRegAssumeUnused(index_mcv.register)
		1963	else
		1964	null;
		1965	defer if (index_lock) \|reg\| self.register_manager.unlockReg(reg);
1944		1966
1945	const base_mcv = slicePtr(slice_mcv);	1967	const base_mcv = slicePtr(slice_mcv);
1946		1968
...	@@ -1950,20 +1972,20 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1950,20 +1972,20 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1950	.register => \|r\| r,	1972	.register => \|r\| r,
1951	else => try self.copyToTmpRegister(slice_ptr_field_type, base_mcv),	1973	else => try self.copyToTmpRegister(slice_ptr_field_type, base_mcv),
1952	};	1974	};
1953	self.register_manager.freezeRegs(&.{base_reg});	1975	const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg);
1954	defer self.register_manager.unfreezeRegs(&.{base_reg});	1976	defer self.register_manager.unlockReg(base_reg_lock);
1955		1977
1956	const dst_reg = try self.register_manager.allocReg(inst);	1978	const dst_reg = try self.register_manager.allocReg(inst);
1957	const dst_mcv = MCValue{ .register = dst_reg };	1979	const dst_mcv = MCValue{ .register = dst_reg };
1958	self.register_manager.freezeRegs(&.{dst_reg});	1980	const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1959	defer self.register_manager.unfreezeRegs(&.{dst_reg});	1981	defer self.register_manager.unlockReg(dst_reg_lock);
1960		1982
1961	const index_reg: Register = switch (index_mcv) {	1983	const index_reg: Register = switch (index_mcv) {
1962	.register => \|reg\| reg,	1984	.register => \|reg\| reg,
1963	else => try self.copyToTmpRegister(Type.usize, index_mcv),	1985	else => try self.copyToTmpRegister(Type.usize, index_mcv),
1964	};	1986	};
1965	self.register_manager.freezeRegs(&.{index_reg});	1987	const index_reg_lock = self.register_manager.lockReg(index_reg);
1966	defer self.register_manager.unfreezeRegs(&.{index_reg});	1988	defer if (index_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
1967		1989
1968	const tag: Mir.Inst.Tag = switch (elem_size) {	1990	const tag: Mir.Inst.Tag = switch (elem_size) {
1969	1 => .ldrb,	1991	1 => .ldrb,
...	@@ -2149,8 +2171,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2149,8 +2171,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2149	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),	2171	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
2150	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),	2172	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
2151	.register => \|reg\| {	2173	.register => \|reg\| {
2152	self.register_manager.freezeRegs(&.{reg});	2174	const reg_lock = self.register_manager.lockReg(reg);
2153	defer self.register_manager.unfreezeRegs(&.{reg});	2175	defer if (reg_lock) \|reg_locked\| self.register_manager.unlockReg(reg_locked);
2154		2176
2155	switch (dst_mcv) {	2177	switch (dst_mcv) {
2156	.dead => unreachable,	2178	.dead => unreachable,
...	@@ -2162,16 +2184,18 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2162,16 +2184,18 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2162	.stack_offset => \|off\| {	2184	.stack_offset => \|off\| {
2163	if (elem_size <= 4) {	2185	if (elem_size <= 4) {
2164	const tmp_reg = try self.register_manager.allocReg(null);	2186	const tmp_reg = try self.register_manager.allocReg(null);
2165	self.register_manager.freezeRegs(&.{tmp_reg});	2187	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2166	defer self.register_manager.unfreezeRegs(&.{tmp_reg});	2188	defer self.register_manager.unlockReg(tmp_reg_lock);
2167		2189
2168	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);	2190	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
2169	try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });	2191	try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
2170	} else {	2192	} else {
2171	// TODO optimize the register allocation	2193	// TODO optimize the register allocation
2172	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });	2194	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2173	self.register_manager.freezeRegs(&regs);	2195	const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2174	defer self.register_manager.unfreezeRegs(&regs);	2196	defer for (regs_locks) \|reg_locked\| {
		2197	self.register_manager.unlockReg(reg_locked);
		2198	};
2175		2199
2176	const src_reg = reg;	2200	const src_reg = reg;
2177	const dst_reg = regs[0];	2201	const dst_reg = regs[0];
...	@@ -2197,8 +2221,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2197,8 +2221,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2197	.stack_argument_offset,	2221	.stack_argument_offset,
2198	=> {	2222	=> {
2199	const reg = try self.register_manager.allocReg(null);	2223	const reg = try self.register_manager.allocReg(null);
2200	self.register_manager.freezeRegs(&.{reg});	2224	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
2201	defer self.register_manager.unfreezeRegs(&.{reg});	2225	defer self.register_manager.unlockReg(reg_lock);
2202		2226
2203	try self.genSetReg(ptr_ty, reg, ptr);	2227	try self.genSetReg(ptr_ty, reg, ptr);
2204	try self.load(dst_mcv, .{ .register = reg }, ptr_ty);	2228	try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
...	@@ -2252,8 +2276,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2252,8 +2276,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2252	try self.genSetStack(value_ty, off, value);	2276	try self.genSetStack(value_ty, off, value);
2253	},	2277	},
2254	.register => \|addr_reg\| {	2278	.register => \|addr_reg\| {
2255	self.register_manager.freezeRegs(&.{addr_reg});	2279	const addr_reg_lock = self.register_manager.lockReg(addr_reg);
2256	defer self.register_manager.unfreezeRegs(&.{addr_reg});	2280	defer if (addr_reg_lock) \|reg\| self.register_manager.unlockReg(reg);
2257		2281
2258	switch (value) {	2282	switch (value) {
2259	.dead => unreachable,	2283	.dead => unreachable,
...	@@ -2264,15 +2288,17 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2264,15 +2288,17 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2264	else => {	2288	else => {
2265	if (elem_size <= 4) {	2289	if (elem_size <= 4) {
2266	const tmp_reg = try self.register_manager.allocReg(null);	2290	const tmp_reg = try self.register_manager.allocReg(null);
2267	self.register_manager.freezeRegs(&.{tmp_reg});	2291	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2268	defer self.register_manager.unfreezeRegs(&.{tmp_reg});	2292	defer self.register_manager.unlockReg(tmp_reg_lock);
2269		2293
2270	try self.genSetReg(value_ty, tmp_reg, value);	2294	try self.genSetReg(value_ty, tmp_reg, value);
2271	try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);	2295	try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2272	} else {	2296	} else {
2273	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });	2297	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null });
2274	self.register_manager.freezeRegs(&regs);	2298	const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
2275	defer self.register_manager.unfreezeRegs(&regs);	2299	defer for (regs_locks) \|reg\| {
		2300	self.register_manager.unlockReg(reg);
		2301	};
2276		2302
2277	const src_reg = regs[0];	2303	const src_reg = regs[0];
2278	const dst_reg = addr_reg;	2304	const dst_reg = addr_reg;
...	@@ -2356,12 +2382,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde	...	@@ -2356,12 +2382,12 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
2356	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{	2382	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
2357	.immediate = struct_field_offset,	2383	.immediate = struct_field_offset,
2358	});	2384	});
2359	self.register_manager.freezeRegs(&.{offset_reg});	2385	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2360	defer self.register_manager.unfreezeRegs(&.{offset_reg});	2386	defer self.register_manager.unlockReg(offset_reg_lock);
2361		2387
2362	const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);	2388	const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
2363	self.register_manager.freezeRegs(&.{addr_reg});	2389	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
2364	defer self.register_manager.unfreezeRegs(&.{addr_reg});	2390	defer self.register_manager.unlockReg(addr_reg_lock);
2365		2391
2366	const dest = try self.binOp(	2392	const dest = try self.binOp(
2367	.add,	2393	.add,
...	@@ -2477,8 +2503,11 @@ fn binOpRegister(	...	@@ -2477,8 +2503,11 @@ fn binOpRegister(
2477	const lhs_is_register = lhs == .register;	2503	const lhs_is_register = lhs == .register;
2478	const rhs_is_register = rhs == .register;	2504	const rhs_is_register = rhs == .register;
2479		2505
2480	if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});	2506	const lhs_lock: ?RegisterLock = if (lhs_is_register)
2481	if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});	2507	self.register_manager.lockReg(lhs.register)
		2508	else
		2509	null;
		2510	defer if (lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
2482		2511
2483	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	2512	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
2484		2513
...	@@ -2489,13 +2518,13 @@ fn binOpRegister(	...	@@ -2489,13 +2518,13 @@ fn binOpRegister(
2489	} else null;	2518	} else null;
2490		2519
2491	const reg = try self.register_manager.allocReg(track_inst);	2520	const reg = try self.register_manager.allocReg(track_inst);
2492	self.register_manager.freezeRegs(&.{reg});
2493		2521
2494	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	2522	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
2495		2523
2496	break :blk reg;	2524	break :blk reg;
2497	};	2525	};
2498	defer self.register_manager.unfreezeRegs(&.{lhs_reg});	2526	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
		2527	defer if (new_lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
2499		2528
2500	const rhs_reg = if (rhs_is_register) rhs.register else blk: {	2529	const rhs_reg = if (rhs_is_register) rhs.register else blk: {
2501	const track_inst: ?Air.Inst.Index = if (maybe_inst) \|inst\| inst: {	2530	const track_inst: ?Air.Inst.Index = if (maybe_inst) \|inst\| inst: {
...	@@ -2504,13 +2533,13 @@ fn binOpRegister(	...	@@ -2504,13 +2533,13 @@ fn binOpRegister(
2504	} else null;	2533	} else null;
2505		2534
2506	const reg = try self.register_manager.allocReg(track_inst);	2535	const reg = try self.register_manager.allocReg(track_inst);
2507	self.register_manager.freezeRegs(&.{reg});
2508		2536
2509	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	2537	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
2510		2538
2511	break :blk reg;	2539	break :blk reg;
2512	};	2540	};
2513	defer self.register_manager.unfreezeRegs(&.{rhs_reg});	2541	const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
		2542	defer if (new_rhs_lock) \|reg\| self.register_manager.unlockReg(reg);
2514		2543
2515	const dest_reg = switch (mir_tag) {	2544	const dest_reg = switch (mir_tag) {
2516	.cmp => .r0, // cmp has no destination regardless	2545	.cmp => .r0, // cmp has no destination regardless
...	@@ -2593,7 +2622,11 @@ fn binOpImmediate(	...	@@ -2593,7 +2622,11 @@ fn binOpImmediate(
2593	) !MCValue {	2622	) !MCValue {
2594	const lhs_is_register = lhs == .register;	2623	const lhs_is_register = lhs == .register;
2595		2624
2596	if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});	2625	const lhs_lock: ?RegisterLock = if (lhs_is_register)
		2626	self.register_manager.lockReg(lhs.register)
		2627	else
		2628	null;
		2629	defer if (lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
2597		2630
2598	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	2631	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
2599		2632
...	@@ -2606,13 +2639,13 @@ fn binOpImmediate(	...	@@ -2606,13 +2639,13 @@ fn binOpImmediate(
2606	} else null;	2639	} else null;
2607		2640
2608	const reg = try self.register_manager.allocReg(track_inst);	2641	const reg = try self.register_manager.allocReg(track_inst);
2609	self.register_manager.freezeRegs(&.{reg});
2610		2642
2611	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });	2643	if (track_inst) \|inst\| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
2612		2644
2613	break :blk reg;	2645	break :blk reg;
2614	};	2646	};
2615	defer self.register_manager.unfreezeRegs(&.{lhs_reg});	2647	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
		2648	defer if (new_lhs_lock) \|reg\| self.register_manager.unlockReg(reg);
2616		2649
2617	const dest_reg = switch (mir_tag) {	2650	const dest_reg = switch (mir_tag) {
2618	.cmp => .r0, // cmp has no destination reg	2651	.cmp => .r0, // cmp has no destination reg
...	@@ -3656,6 +3689,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3656,6 +3689,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
3656	const parent_compare_flags_inst = self.compare_flags_inst;	3689	const parent_compare_flags_inst = self.compare_flags_inst;
3657		3690
3658	try self.branch_stack.append(.{});	3691	try self.branch_stack.append(.{});
		3692	errdefer {
		3693	_ = self.branch_stack.pop();
		3694	}
3659		3695
3660	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);	3696	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
3661	for (liveness_condbr.then_deaths) \|operand\| {	3697	for (liveness_condbr.then_deaths) \|operand\| {
...	@@ -4264,8 +4300,36 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro	...	@@ -4264,8 +4300,36 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
4264	},	4300	},
4265	.register_c_flag,	4301	.register_c_flag,
4266	.register_v_flag,	4302	.register_v_flag,
4267	=> {	4303	=> \|reg\| {
4268	return self.fail("TODO implement genSetStack {}", .{mcv});	4304	const reg_lock = self.register_manager.lockReg(reg);
		4305	defer if (reg_lock) \|locked_reg\| self.register_manager.unlockReg(locked_reg);
		4306
		4307	const wrapped_ty = ty.structFieldType(0);
		4308	try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg });
		4309
		4310	const overflow_bit_ty = ty.structFieldType(1);
		4311	const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*));
		4312	const cond_reg = try self.register_manager.allocReg(null);
		4313
		4314	// C flag: movcs reg, #1
		4315	// V flag: movvs reg, #1
		4316	_ = try self.addInst(.{
		4317	.tag = .mov,
		4318	.cond = switch (mcv) {
		4319	.register_c_flag => .cs,
		4320	.register_v_flag => .vs,
		4321	else => unreachable,
		4322	},
		4323	.data = .{ .rr_op = .{
		4324	.rd = cond_reg,
		4325	.rn = .r0,
		4326	.op = Instruction.Operand.fromU32(1).?,
		4327	} },
		4328	});
		4329
		4330	try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
		4331	.register = cond_reg,
		4332	});
4269	},	4333	},
4270	.memory,	4334	.memory,
4271	.stack_argument_offset,	4335	.stack_argument_offset,

src/arch/riscv64/CodeGen.zig+12-8

 @@ -23,6 +23,7 @@ const log = std.log.scoped(.codegen);
 const build_options = @import("build_options");
 const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
 const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
+const RegisterLock = RegisterManager.RegisterLock;
 const FnResult = @import("../../codegen.zig").FnResult;
 const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
 @@ -937,8 +938,11 @@ fn binOpRegister(
     const lhs_is_register = lhs == .register;
     const rhs_is_register = rhs == .register;
-    if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register});
+    const lhs_lock: ?RegisterLock = if (lhs_is_register)
-    if (rhs_is_register) self.register_manager.freezeRegs(&.{rhs.register});
+        self.register_manager.lockReg(lhs.register)
+    else
+        null;
+    defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
     const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
 @@ -949,13 +953,13 @@ fn binOpRegister(
         } else null;
         const reg = try self.register_manager.allocReg(track_inst);
-        self.register_manager.freezeRegs(&.{reg});
         if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
         break :blk reg;
     };
-    defer self.register_manager.unfreezeRegs(&.{lhs_reg});
+    const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
+    defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
     const rhs_reg = if (rhs_is_register) rhs.register else blk: {
         const track_inst: ?Air.Inst.Index = if (maybe_inst) |inst| inst: {
 @@ -964,13 +968,13 @@ fn binOpRegister(
         } else null;
         const reg = try self.register_manager.allocReg(track_inst);
-        self.register_manager.freezeRegs(&.{reg});
         if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg });
         break :blk reg;
     };
-    defer self.register_manager.unfreezeRegs(&.{rhs_reg});
+    const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
+    defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
     const dest_reg = if (maybe_inst) |inst| blk: {
         const bin_op = self.air.instructions.items(.data)[inst].bin_op;
 @@ -1448,8 +1452,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
         .stack_offset,
         => {
             const reg = try self.register_manager.allocReg(null);
-            self.register_manager.freezeRegs(&.{reg});
+            const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
-            defer self.register_manager.unfreezeRegs(&.{reg});
+            defer self.register_manager.unlockReg(reg_lock);
             try self.genSetReg(ptr_ty, reg, ptr);
             try self.load(dst_mcv, .{ .register = reg }, ptr_ty);

src/arch/x86_64/CodeGen.zig+701-482

...	@@ -22,6 +22,8 @@ const Liveness = @import("../../Liveness.zig");	...	@@ -22,6 +22,8 @@ const Liveness = @import("../../Liveness.zig");
22	const Mir = @import("Mir.zig");	22	const Mir = @import("Mir.zig");
23	const Module = @import("../../Module.zig");	23	const Module = @import("../../Module.zig");
24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;	24	const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager;
		25	const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
		26	const RegisterLock = RegisterManager.RegisterLock;
25	const Target = std.Target;	27	const Target = std.Target;
26	const Type = @import("../../type.zig").Type;	28	const Type = @import("../../type.zig").Type;
27	const TypedValue = @import("../../TypedValue.zig");	29	const TypedValue = @import("../../TypedValue.zig");
...	@@ -42,8 +44,6 @@ const InnerError = error{	...	@@ -42,8 +44,6 @@ const InnerError = error{
42	OutOfRegisters,	44	OutOfRegisters,
43	};	45	};
44		46
45	const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
46
47	gpa: Allocator,	47	gpa: Allocator,
48	air: Air,	48	air: Air,
49	liveness: Liveness,	49	liveness: Liveness,
...	@@ -191,60 +191,12 @@ pub const MCValue = union(enum) {	...	@@ -191,60 +191,12 @@ pub const MCValue = union(enum) {
191	};	191	};
192	}	192	}
193		193
194	fn usesCompareFlags(mcv: MCValue) bool {
195	return switch (mcv) {
196	.compare_flags_unsigned,
197	.compare_flags_signed,
198	.register_overflow_unsigned,
199	.register_overflow_signed,
200	=> true,
201	else => false,
202	};
203	}
204
205	fn isRegister(mcv: MCValue) bool {	194	fn isRegister(mcv: MCValue) bool {
206	return switch (mcv) {	195	return switch (mcv) {
207	.register,	196	.register => true,
208	.register_overflow_unsigned,
209	.register_overflow_signed,
210	=> true,
211	else => false,	197	else => false,
212	};	198	};
213	}	199	}
214
215	fn freezeIfRegister(mcv: MCValue, mgr: *RegisterManager) void {
216	switch (mcv) {
217	.register,
218	.register_overflow_signed,
219	.register_overflow_unsigned,
220	=> \|reg\| {
221	mgr.freezeRegs(&.{reg});
222	},
223	else => {},
224	}
225	}
226
227	fn unfreezeIfRegister(mcv: MCValue, mgr: *RegisterManager) void {
228	switch (mcv) {
229	.register,
230	.register_overflow_signed,
231	.register_overflow_unsigned,
232	=> \|reg\| {
233	mgr.unfreezeRegs(&.{reg});
234	},
235	else => {},
236	}
237	}
238
239	fn asRegister(mcv: MCValue) ?Register {
240	return switch (mcv) {
241	.register,
242	.register_overflow_signed,
243	.register_overflow_unsigned,
244	=> \|reg\| reg,
245	else => null,
246	};
247	}
248	};	200	};
249		201
250	const Branch = struct {	202	const Branch = struct {
...	@@ -819,7 +771,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -819,7 +771,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
819	// zig fmt: on	771	// zig fmt: on
820	}	772	}
821		773
822	assert(!self.register_manager.frozenRegsExist());	774	assert(!self.register_manager.lockedRegsExist());
823		775
824	if (std.debug.runtime_safety) {	776	if (std.debug.runtime_safety) {
825	if (self.air_bookkeeping < old_air_bookkeeping + 1) {	777	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
...	@@ -876,15 +828,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live	...	@@ -876,15 +828,21 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
876	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	828	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
877	branch.inst_table.putAssumeCapacityNoClobber(inst, result);	829	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
878		830
879	if (result.asRegister()) \|reg\| {	831	switch (result) {
880	// In some cases (such as bitcast), an operand	832	.register,
881	// may be the same MCValue as the result. If	833	.register_overflow_signed,
882	// that operand died and was a register, it	834	.register_overflow_unsigned,
883	// was freed by processDeath. We have to	835	=> \|reg\| {
884	// "re-allocate" the register.	836	// In some cases (such as bitcast), an operand
885	if (self.register_manager.isRegFree(reg)) {	837	// may be the same MCValue as the result. If
886	self.register_manager.getRegAssumeFree(reg, inst);	838	// that operand died and was a register, it
887	}	839	// was freed by processDeath. We have to
		840	// "re-allocate" the register.
		841	if (self.register_manager.isRegFree(reg)) {
		842	self.register_manager.getRegAssumeFree(reg, inst);
		843	}
		844	},
		845	else => {},
888	}	846	}
889	}	847	}
890	self.finishAirBookkeeping();	848	self.finishAirBookkeeping();
...	@@ -955,7 +913,15 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void	...	@@ -955,7 +913,15 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
955	const stack_mcv = try self.allocRegOrMem(inst, false);	913	const stack_mcv = try self.allocRegOrMem(inst, false);
956	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });	914	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
957	const reg_mcv = self.getResolvedInstValue(inst);	915	const reg_mcv = self.getResolvedInstValue(inst);
958	assert(reg.to64() == reg_mcv.asRegister().?.to64());	916	switch (reg_mcv) {
		917	.register,
		918	.register_overflow_unsigned,
		919	.register_overflow_signed,
		920	=> \|other\| {
		921	assert(reg.to64() == other.to64());
		922	},
		923	else => {},
		924	}
959	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	925	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
960	try branch.inst_table.put(self.gpa, inst, stack_mcv);	926	try branch.inst_table.put(self.gpa, inst, stack_mcv);
961	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});	927	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{});
...	@@ -964,18 +930,32 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void	...	@@ -964,18 +930,32 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
964	pub fn spillCompareFlagsIfOccupied(self: *Self) !void {	930	pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
965	if (self.compare_flags_inst) \|inst_to_save\| {	931	if (self.compare_flags_inst) \|inst_to_save\| {
966	const mcv = self.getResolvedInstValue(inst_to_save);	932	const mcv = self.getResolvedInstValue(inst_to_save);
967	assert(mcv.usesCompareFlags());	933	const new_mcv = switch (mcv) {
		934	.register_overflow_signed,
		935	.register_overflow_unsigned,
		936	=> try self.allocRegOrMem(inst_to_save, false),
		937	.compare_flags_signed,
		938	.compare_flags_unsigned,
		939	=> try self.allocRegOrMem(inst_to_save, true),
		940	else => unreachable,
		941	};
968		942
969	const new_mcv = try self.allocRegOrMem(inst_to_save, !mcv.isRegister());
970	try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);	943	try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv);
971	log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });	944	log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
		945
972	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	946	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
973	try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);	947	try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
974		948
975	self.compare_flags_inst = null;	949	self.compare_flags_inst = null;
		950
976	// TODO consolidate with register manager and spillInstruction	951	// TODO consolidate with register manager and spillInstruction
977	// this call should really belong in the register manager!	952	// this call should really belong in the register manager!
978	if (mcv.isRegister()) self.register_manager.freeReg(mcv.asRegister().?);	953	switch (mcv) {
		954	.register_overflow_signed,
		955	.register_overflow_unsigned,
		956	=> \|reg\| self.register_manager.freeReg(reg),
		957	else => {},
		958	}
979	}	959	}
980	}	960	}
981		961
...	@@ -1043,8 +1023,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1043,8 +1023,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1043	return self.fail("TODO implement intCast for abi sizes larger than 8", .{});	1023	return self.fail("TODO implement intCast for abi sizes larger than 8", .{});
1044	}	1024	}
1045		1025
1046	operand.freezeIfRegister(&self.register_manager);	1026	const operand_lock: ?RegisterLock = switch (operand) {
1047	defer operand.unfreezeIfRegister(&self.register_manager);	1027	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1028	else => null,
		1029	};
		1030	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
1048		1031
1049	const reg = try self.register_manager.allocReg(inst);	1032	const reg = try self.register_manager.allocReg(inst);
1050	try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });	1033	try self.genSetReg(dest_ty, reg, .{ .immediate = 0 });
...	@@ -1071,8 +1054,11 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1071,8 +1054,11 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1071	return self.fail("TODO implement trunc for abi sizes larger than 8", .{});	1054	return self.fail("TODO implement trunc for abi sizes larger than 8", .{});
1072	}	1055	}
1073		1056
1074	operand.freezeIfRegister(&self.register_manager);	1057	const operand_lock: ?RegisterLock = switch (operand) {
1075	defer operand.unfreezeIfRegister(&self.register_manager);	1058	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1059	else => null,
		1060	};
		1061	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
1076		1062
1077	const reg: Register = blk: {	1063	const reg: Register = blk: {
1078	if (operand.isRegister()) {	1064	if (operand.isRegister()) {
...	@@ -1156,16 +1142,22 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1156,16 +1142,22 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1156	// TODO improve by checking if any operand can be reused.	1142	// TODO improve by checking if any operand can be reused.
1157	// TODO audit register allocation	1143	// TODO audit register allocation
1158	const lhs = try self.resolveInst(bin_op.lhs);	1144	const lhs = try self.resolveInst(bin_op.lhs);
1159	lhs.freezeIfRegister(&self.register_manager);	1145	const lhs_lock: ?RegisterLock = switch (lhs) {
1160	defer lhs.unfreezeIfRegister(&self.register_manager);	1146	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1147	else => null,
		1148	};
		1149	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1161		1150
1162	const lhs_reg = try self.copyToTmpRegister(ty, lhs);	1151	const lhs_reg = try self.copyToTmpRegister(ty, lhs);
1163	self.register_manager.freezeRegs(&.{lhs_reg});	1152	const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);
1164	defer self.register_manager.unfreezeRegs(&.{lhs_reg});	1153	defer self.register_manager.unlockReg(lhs_reg_lock);
1165		1154
1166	const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32);	1155	const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32);
1167	rhs_mcv.freezeIfRegister(&self.register_manager);	1156	const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
1168	defer rhs_mcv.unfreezeIfRegister(&self.register_manager);	1157	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1158	else => null,
		1159	};
		1160	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1169		1161
1170	try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);	1162	try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);
1171		1163
...	@@ -1200,28 +1192,37 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r	...	@@ -1200,28 +1192,37 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
1200	const offset = try self.resolveInst(op_rhs);	1192	const offset = try self.resolveInst(op_rhs);
1201	const offset_ty = self.air.typeOf(op_rhs);	1193	const offset_ty = self.air.typeOf(op_rhs);
1202		1194
1203	offset.freezeIfRegister(&self.register_manager);	1195	const offset_lock: ?RegisterLock = switch (offset) {
1204	defer offset.unfreezeIfRegister(&self.register_manager);	1196	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1197	else => null,
		1198	};
		1199	defer if (offset_lock) \|lock\| self.register_manager.unlockReg(lock);
1205		1200
1206	const dst_mcv = blk: {	1201	const dst_mcv: MCValue = blk: {
1207	if (self.reuseOperand(inst, op_lhs, 0, ptr)) {	1202	if (self.reuseOperand(inst, op_lhs, 0, ptr)) {
1208	if (ptr.isMemory() or ptr.isRegister()) break :blk ptr;	1203	if (ptr.isMemory() or ptr.isRegister()) break :blk ptr;
1209	}	1204	}
1210	break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, ptr) };	1205	break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, ptr) };
1211	};	1206	};
1212		1207
1213	dst_mcv.freezeIfRegister(&self.register_manager);	1208	const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
1214	defer dst_mcv.unfreezeIfRegister(&self.register_manager);	1209	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1210	else => null,
		1211	};
		1212	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1215		1213
1216	const offset_mcv = blk: {	1214	const offset_mcv: MCValue = blk: {
1217	if (self.reuseOperand(inst, op_rhs, 1, offset)) {	1215	if (self.reuseOperand(inst, op_rhs, 1, offset)) {
1218	if (offset.isRegister()) break :blk offset;	1216	if (offset.isRegister()) break :blk offset;
1219	}	1217	}
1220	break :blk MCValue{ .register = try self.copyToTmpRegister(offset_ty, offset) };	1218	break :blk MCValue{ .register = try self.copyToTmpRegister(offset_ty, offset) };
1221	};	1219	};
1222		1220
1223	offset_mcv.freezeIfRegister(&self.register_manager);	1221	const offset_mcv_lock: ?RegisterLock = switch (offset_mcv) {
1224	defer offset_mcv.unfreezeIfRegister(&self.register_manager);	1222	.register => \|reg\| self.register_manager.lockReg(reg),
		1223	else => null,
		1224	};
		1225	defer if (offset_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1225		1226
1226	try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });	1227	try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });
1227		1228
...	@@ -1306,30 +1307,40 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air	...	@@ -1306,30 +1307,40 @@ fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1306	const dst_ty = self.air.typeOf(op_lhs);	1307	const dst_ty = self.air.typeOf(op_lhs);
1307		1308
1308	const lhs = try self.resolveInst(op_lhs);	1309	const lhs = try self.resolveInst(op_lhs);
1309	lhs.freezeIfRegister(&self.register_manager);	1310	const lhs_lock: ?RegisterLock = switch (lhs) {
1310	defer lhs.unfreezeIfRegister(&self.register_manager);	1311	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1312	else => null,
		1313	};
		1314	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1311		1315
1312	const rhs = try self.resolveInst(op_rhs);	1316	const rhs = try self.resolveInst(op_rhs);
1313	rhs.freezeIfRegister(&self.register_manager);	1317	const rhs_lock: ?RegisterLock = switch (rhs) {
1314	defer rhs.unfreezeIfRegister(&self.register_manager);	1318	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1319	else => null,
		1320	};
		1321	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1315		1322
1316	const dst_mcv = blk: {	1323	const dst_mcv: MCValue = blk: {
1317	if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {	1324	if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
1318	break :blk lhs;	1325	break :blk lhs;
1319	}	1326	}
1320	break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);	1327	break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);
1321	};	1328	};
		1329	const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
		1330	.register => \|reg\| self.register_manager.lockReg(reg),
		1331	else => null,
		1332	};
		1333	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1322		1334
1323	dst_mcv.freezeIfRegister(&self.register_manager);	1335	const rhs_mcv: MCValue = blk: {
1324	defer dst_mcv.unfreezeIfRegister(&self.register_manager);
1325
1326	const rhs_mcv = blk: {
1327	if (rhs.isMemory() or rhs.isRegister()) break :blk rhs;	1336	if (rhs.isMemory() or rhs.isRegister()) break :blk rhs;
1328	break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };	1337	break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
1329	};	1338	};
1330		1339	const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
1331	rhs_mcv.freezeIfRegister(&self.register_manager);	1340	.register => \|reg\| self.register_manager.lockReg(reg),
1332	defer rhs_mcv.unfreezeIfRegister(&self.register_manager);	1341	else => null,
		1342	};
		1343	defer if (rhs_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1333		1344
1334	try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv);	1345	try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv);
1335		1346
...	@@ -1366,8 +1377,10 @@ fn airMul(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1366,8 +1377,10 @@ fn airMul(self: *Self, inst: Air.Inst.Index) !void {
1366	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.	1377	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1367	try self.register_manager.getReg(.rax, inst);	1378	try self.register_manager.getReg(.rax, inst);
1368	try self.register_manager.getReg(.rdx, null);	1379	try self.register_manager.getReg(.rdx, null);
1369	self.register_manager.freezeRegs(&.{ .rax, .rdx });	1380	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1370	defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });	1381	defer for (reg_locks) \|reg\| {
		1382	self.register_manager.unlockReg(reg);
		1383	};
1371		1384
1372	const lhs = try self.resolveInst(bin_op.lhs);	1385	const lhs = try self.resolveInst(bin_op.lhs);
1373	const rhs = try self.resolveInst(bin_op.rhs);	1386	const rhs = try self.resolveInst(bin_op.rhs);
...	@@ -1458,9 +1471,13 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1458,9 +1471,13 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1458	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	1471	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1459	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;	1472	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1460	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;	1473	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1461	const result = if (self.liveness.isUnused(inst)) .dead else result: {
1462	const ty = self.air.typeOf(bin_op.lhs);
1463		1474
		1475	if (self.liveness.isUnused(inst)) {
		1476	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
		1477	}
		1478
		1479	const ty = self.air.typeOf(bin_op.lhs);
		1480	const result: MCValue = result: {
1464	switch (ty.zigTypeTag()) {	1481	switch (ty.zigTypeTag()) {
1465	.Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),	1482	.Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
1466	.Int => {	1483	.Int => {
...	@@ -1477,8 +1494,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1477,8 +1494,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1477	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.	1494	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1478	try self.register_manager.getReg(.rax, inst);	1495	try self.register_manager.getReg(.rax, inst);
1479	try self.register_manager.getReg(.rdx, null);	1496	try self.register_manager.getReg(.rdx, null);
1480	self.register_manager.freezeRegs(&.{ .rax, .rdx });	1497	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1481	defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });	1498	defer for (reg_locks) \|reg\| {
		1499	self.register_manager.unlockReg(reg);
		1500	};
1482		1501
1483	const lhs = try self.resolveInst(bin_op.lhs);	1502	const lhs = try self.resolveInst(bin_op.lhs);
1484	const rhs = try self.resolveInst(bin_op.rhs);	1503	const rhs = try self.resolveInst(bin_op.rhs);
...	@@ -1504,21 +1523,28 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1504,21 +1523,28 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1504	const lhs = try self.resolveInst(bin_op.lhs);	1523	const lhs = try self.resolveInst(bin_op.lhs);
1505	const rhs = try self.resolveInst(bin_op.rhs);	1524	const rhs = try self.resolveInst(bin_op.rhs);
1506		1525
1507	rhs.freezeIfRegister(&self.register_manager);	1526	const rhs_lock: ?RegisterLock = switch (rhs) {
1508	defer rhs.unfreezeIfRegister(&self.register_manager);	1527	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1528	else => null,
		1529	};
		1530	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1509		1531
1510	const dst_reg: Register = blk: {	1532	const dst_reg: Register = blk: {
1511	if (lhs.isRegister()) break :blk lhs.register;	1533	if (lhs.isRegister()) break :blk lhs.register;
1512	break :blk try self.copyToTmpRegister(ty, lhs);	1534	break :blk try self.copyToTmpRegister(ty, lhs);
1513	};	1535	};
1514	self.register_manager.freezeRegs(&.{dst_reg});	1536	const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
		1537	defer self.register_manager.unlockReg(dst_reg_lock);
1515		1538
1516	const rhs_mcv = blk: {	1539	const rhs_mcv: MCValue = blk: {
1517	if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;	1540	if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
1518	break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };	1541	break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };
1519	};	1542	};
1520	rhs_mcv.freezeIfRegister(&self.register_manager);	1543	const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
1521	defer rhs_mcv.unfreezeIfRegister(&self.register_manager);	1544	.register => \|reg\| self.register_manager.lockReg(reg),
		1545	else => null,
		1546	};
		1547	defer if (rhs_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1522		1548
1523	try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);	1549	try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
1524		1550
...	@@ -1528,8 +1554,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1528,8 +1554,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1528	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.	1554	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1529	try self.register_manager.getReg(.rax, null);	1555	try self.register_manager.getReg(.rax, null);
1530	try self.register_manager.getReg(.rdx, null);	1556	try self.register_manager.getReg(.rdx, null);
1531	self.register_manager.freezeRegs(&.{ .rax, .rdx });	1557	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1532	defer self.register_manager.unfreezeRegs(&.{.rdx});	1558	defer for (reg_locks) \|reg\| {
		1559	self.register_manager.unlockReg(reg);
		1560	};
1533		1561
1534	const lhs = try self.resolveInst(bin_op.lhs);	1562	const lhs = try self.resolveInst(bin_op.lhs);
1535	const rhs = try self.resolveInst(bin_op.rhs);	1563	const rhs = try self.resolveInst(bin_op.rhs);
...	@@ -1540,7 +1568,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1540,7 +1568,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1540	},	1568	},
1541	}	1569	}
1542	};	1570	};
1543	defer self.register_manager.unfreezeRegs(&.{dst_reg});	1571	const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
		1572	defer self.register_manager.unlockReg(dst_reg_lock);
1544		1573
1545	const tuple_ty = self.air.typeOfIndex(inst);	1574	const tuple_ty = self.air.typeOfIndex(inst);
1546	const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));	1575	const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));
...	@@ -1554,8 +1583,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1554,8 +1583,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1554	};	1583	};
1555		1584
1556	const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });	1585	const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1557	self.register_manager.freezeRegs(&temp_regs);	1586	const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
1558	defer self.register_manager.unfreezeRegs(&temp_regs);	1587	defer for (temp_regs_locks) \|reg\| {
		1588	self.register_manager.unlockReg(reg);
		1589	};
1559		1590
1560	const overflow_reg = temp_regs[0];	1591	const overflow_reg = temp_regs[0];
1561	const flags: u2 = switch (int_info.signedness) {	1592	const flags: u2 = switch (int_info.signedness) {
...	@@ -1699,18 +1730,19 @@ fn genIntMulDivOpMir(	...	@@ -1699,18 +1730,19 @@ fn genIntMulDivOpMir(
1699	/// Clobbers .rax and .rdx registers.	1730	/// Clobbers .rax and .rdx registers.
1700	fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {	1731	fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
1701	const signedness = ty.intInfo(self.target.*).signedness;	1732	const signedness = ty.intInfo(self.target.*).signedness;
1702	const dividend = switch (lhs) {	1733	const dividend: Register = switch (lhs) {
1703	.register => \|reg\| reg,	1734	.register => \|reg\| reg,
1704	else => try self.copyToTmpRegister(ty, lhs),	1735	else => try self.copyToTmpRegister(ty, lhs),
1705	};	1736	};
1706	self.register_manager.freezeRegs(&.{dividend});	1737	const dividend_lock = self.register_manager.lockReg(dividend);
		1738	defer if (dividend_lock) \|lock\| self.register_manager.unlockReg(lock);
1707		1739
1708	const divisor = switch (rhs) {	1740	const divisor: Register = switch (rhs) {
1709	.register => \|reg\| reg,	1741	.register => \|reg\| reg,
1710	else => try self.copyToTmpRegister(ty, rhs),	1742	else => try self.copyToTmpRegister(ty, rhs),
1711	};	1743	};
1712	self.register_manager.freezeRegs(&.{divisor});	1744	const divisor_lock = self.register_manager.lockReg(divisor);
1713	defer self.register_manager.unfreezeRegs(&.{ dividend, divisor });	1745	defer if (divisor_lock) \|lock\| self.register_manager.unlockReg(lock);
1714		1746
1715	try self.genIntMulDivOpMir(switch (signedness) {	1747	try self.genIntMulDivOpMir(switch (signedness) {
1716	.signed => .idiv,	1748	.signed => .idiv,
...	@@ -1755,54 +1787,71 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -1755,54 +1787,71 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
1755		1787
1756	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {	1788	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1757	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1789	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1758	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1759	const tag = self.air.instructions.items(.tag)[inst];
1760	const ty = self.air.typeOfIndex(inst);
1761		1790
1762	if (ty.zigTypeTag() != .Int) {	1791	if (self.liveness.isUnused(inst)) {
1763	return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });	1792	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1764	}	1793	}
		1794
		1795	const tag = self.air.instructions.items(.tag)[inst];
		1796	const ty = self.air.typeOfIndex(inst);
1765		1797
1766	if (tag == .div_float) {	1798	if (ty.zigTypeTag() != .Int) {
1767	return self.fail("TODO implement {}", .{tag});	1799	return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });
		1800	}
		1801
		1802	if (tag == .div_float) {
		1803	return self.fail("TODO implement {}", .{tag});
		1804	}
		1805
		1806	const signedness = ty.intInfo(self.target.*).signedness;
		1807
		1808	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
		1809	const track_rax: ?Air.Inst.Index = blk: {
		1810	if (signedness == .unsigned) break :blk inst;
		1811	switch (tag) {
		1812	.div_exact, .div_trunc => break :blk inst,
		1813	else => break :blk null,
1768	}	1814	}
		1815	};
		1816	try self.register_manager.getReg(.rax, track_rax);
		1817	try self.register_manager.getReg(.rdx, null);
		1818	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
		1819	defer for (reg_locks) \|reg\| {
		1820	self.register_manager.unlockReg(reg);
		1821	};
1769		1822
1770	const signedness = ty.intInfo(self.target.*).signedness;	1823	const lhs = try self.resolveInst(bin_op.lhs);
		1824	const lhs_lock: ?RegisterLock = switch (lhs) {
		1825	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		1826	else => null,
		1827	};
		1828	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1771		1829
1772	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.	1830	const rhs: MCValue = blk: {
1773	const track_rax: ?Air.Inst.Index = blk: {	1831	const rhs = try self.resolveInst(bin_op.rhs);
1774	if (signedness == .unsigned) break :blk inst;	1832	if (signedness == .signed) {
1775	switch (tag) {	1833	switch (tag) {
1776	.div_exact, .div_trunc => break :blk inst,	1834	.div_floor => {
1777	else => break :blk null,	1835	const rhs_lock: ?RegisterLock = switch (rhs) {
1778	}	1836	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
1779	};	1837	else => null,
1780	try self.register_manager.getReg(.rax, track_rax);	1838	};
1781	try self.register_manager.getReg(.rdx, null);	1839	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1782	self.register_manager.freezeRegs(&.{ .rax, .rdx });
1783	defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1784		1840
1785	const lhs = try self.resolveInst(bin_op.lhs);	1841	break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1786	lhs.freezeIfRegister(&self.register_manager);	1842	},
1787	defer lhs.unfreezeIfRegister(&self.register_manager);	1843	else => {},
1788
1789	const rhs = blk: {
1790	const rhs = try self.resolveInst(bin_op.rhs);
1791	if (signedness == .signed) {
1792	switch (tag) {
1793	.div_floor => {
1794	rhs.freezeIfRegister(&self.register_manager);
1795	defer rhs.unfreezeIfRegister(&self.register_manager);
1796	break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1797	},
1798	else => {},
1799	}
1800	}	1844	}
1801	break :blk rhs;	1845	}
1802	};	1846	break :blk rhs;
1803	rhs.freezeIfRegister(&self.register_manager);	1847	};
1804	defer rhs.unfreezeIfRegister(&self.register_manager);	1848	const rhs_lock: ?RegisterLock = switch (rhs) {
		1849	.register => \|reg\| self.register_manager.lockReg(reg),
		1850	else => null,
		1851	};
		1852	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1805		1853
		1854	const result: MCValue = result: {
1806	if (signedness == .unsigned) {	1855	if (signedness == .unsigned) {
1807	try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);	1856	try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);
1808	break :result MCValue{ .register = .rax };	1857	break :result MCValue{ .register = .rax };
...	@@ -1822,53 +1871,67 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1822,53 +1871,67 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1822	else => unreachable,	1871	else => unreachable,
1823	}	1872	}
1824	};	1873	};
		1874
1825	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1875	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1826	}	1876	}
1827		1877
1828	fn airRem(self: *Self, inst: Air.Inst.Index) !void {	1878	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
1829	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1879	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1830	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1831	const ty = self.air.typeOfIndex(inst);
1832	if (ty.zigTypeTag() != .Int) {
1833	return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
1834	}
1835	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1836	try self.register_manager.getReg(.rax, null);
1837	try self.register_manager.getReg(.rdx, inst);
1838	self.register_manager.freezeRegs(&.{ .rax, .rdx });
1839	defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1840
1841	const lhs = try self.resolveInst(bin_op.lhs);
1842	const rhs = try self.resolveInst(bin_op.rhs);
1843		1880
1844	const signedness = ty.intInfo(self.target.*).signedness;	1881	if (self.liveness.isUnused(inst)) {
1845	try self.genIntMulDivOpMir(switch (signedness) {	1882	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1846	.signed => .idiv,	1883	}
1847	.unsigned => .div,	1884	const ty = self.air.typeOfIndex(inst);
1848	}, ty, signedness, lhs, rhs);	1885	if (ty.zigTypeTag() != .Int) {
1849	break :result MCValue{ .register = .rdx };	1886	return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
		1887	}
		1888	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
		1889	try self.register_manager.getReg(.rax, null);
		1890	try self.register_manager.getReg(.rdx, inst);
		1891	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
		1892	defer for (reg_locks) \|reg\| {
		1893	self.register_manager.unlockReg(reg);
1850	};	1894	};
		1895
		1896	const lhs = try self.resolveInst(bin_op.lhs);
		1897	const rhs = try self.resolveInst(bin_op.rhs);
		1898
		1899	const signedness = ty.intInfo(self.target.*).signedness;
		1900	try self.genIntMulDivOpMir(switch (signedness) {
		1901	.signed => .idiv,
		1902	.unsigned => .div,
		1903	}, ty, signedness, lhs, rhs);
		1904
		1905	const result: MCValue = .{ .register = .rdx };
		1906
1851	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1907	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1852	}	1908	}
1853		1909
1854	fn airMod(self: *Self, inst: Air.Inst.Index) !void {	1910	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1855	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1911	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1856	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1857	const ty = self.air.typeOfIndex(inst);
1858	if (ty.zigTypeTag() != .Int) {
1859	return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
1860	}
1861	const signedness = ty.intInfo(self.target.*).signedness;
1862		1912
1863	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.	1913	if (self.liveness.isUnused(inst)) {
1864	try self.register_manager.getReg(.rax, null);	1914	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1865	try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);	1915	}
1866	self.register_manager.freezeRegs(&.{ .rax, .rdx });
1867	defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1868		1916
1869	const lhs = try self.resolveInst(bin_op.lhs);	1917	const ty = self.air.typeOfIndex(inst);
1870	const rhs = try self.resolveInst(bin_op.rhs);	1918	if (ty.zigTypeTag() != .Int) {
		1919	return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
		1920	}
		1921	const signedness = ty.intInfo(self.target.*).signedness;
		1922
		1923	// Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
		1924	try self.register_manager.getReg(.rax, null);
		1925	try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);
		1926	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
		1927	defer for (reg_locks) \|reg\| {
		1928	self.register_manager.unlockReg(reg);
		1929	};
		1930
		1931	const lhs = try self.resolveInst(bin_op.lhs);
		1932	const rhs = try self.resolveInst(bin_op.rhs);
1871		1933
		1934	const result: MCValue = result: {
1872	switch (signedness) {	1935	switch (signedness) {
1873	.unsigned => {	1936	.unsigned => {
1874	try self.genIntMulDivOpMir(switch (signedness) {	1937	try self.genIntMulDivOpMir(switch (signedness) {
...	@@ -1888,6 +1951,7 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1888,6 +1951,7 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1888	},	1951	},
1889	}	1952	}
1890	};	1953	};
		1954
1891	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1955	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1892	}	1956	}
1893		1957
...	@@ -1954,12 +2018,15 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1954,12 +2018,15 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void {
1954	try self.register_manager.getReg(.rcx, null);	2018	try self.register_manager.getReg(.rcx, null);
1955	try self.genSetReg(shift_ty, .rcx, shift);	2019	try self.genSetReg(shift_ty, .rcx, shift);
1956	}	2020	}
1957	self.register_manager.freezeRegs(&.{.rcx});	2021	const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
1958	defer self.register_manager.unfreezeRegs(&.{.rcx});	2022	defer self.register_manager.unlockReg(rcx_lock);
1959		2023
1960	const value = try self.resolveInst(bin_op.lhs);	2024	const value = try self.resolveInst(bin_op.lhs);
1961	value.freezeIfRegister(&self.register_manager);	2025	const value_lock: ?RegisterLock = switch (value) {
1962	defer value.unfreezeIfRegister(&self.register_manager);	2026	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2027	else => null,
		2028	};
		2029	defer if (value_lock) \|lock\| self.register_manager.unlockReg(lock);
1963		2030
1964	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value);	2031	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value);
1965	_ = try self.addInst(.{	2032	_ = try self.addInst(.{
...	@@ -2055,8 +2122,11 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2055,8 +2122,11 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
2055	const err_ty = err_union_ty.errorUnionSet();	2122	const err_ty = err_union_ty.errorUnionSet();
2056	const payload_ty = err_union_ty.errorUnionPayload();	2123	const payload_ty = err_union_ty.errorUnionPayload();
2057	const operand = try self.resolveInst(ty_op.operand);	2124	const operand = try self.resolveInst(ty_op.operand);
2058	operand.freezeIfRegister(&self.register_manager);	2125	const operand_lock: ?RegisterLock = switch (operand) {
2059	defer operand.unfreezeIfRegister(&self.register_manager);	2126	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2127	else => null,
		2128	};
		2129	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
2060		2130
2061	const result: MCValue = result: {	2131	const result: MCValue = result: {
2062	if (!payload_ty.hasRuntimeBits()) break :result operand;	2132	if (!payload_ty.hasRuntimeBits()) break :result operand;
...	@@ -2085,8 +2155,11 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2085,8 +2155,11 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
2085	if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;	2155	if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
2086		2156
2087	const operand = try self.resolveInst(ty_op.operand);	2157	const operand = try self.resolveInst(ty_op.operand);
2088	operand.freezeIfRegister(&self.register_manager);	2158	const operand_lock: ?RegisterLock = switch (operand) {
2089	defer operand.unfreezeIfRegister(&self.register_manager);	2159	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2160	else => null,
		2161	};
		2162	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
2090		2163
2091	const abi_align = err_union_ty.abiAlignment(self.target.*);	2164	const abi_align = err_union_ty.abiAlignment(self.target.*);
2092	const err_ty = err_union_ty.errorUnionSet();	2165	const err_ty = err_union_ty.errorUnionSet();
...	@@ -2154,8 +2227,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2154,8 +2227,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2154		2227
2155	const optional_ty = self.air.typeOfIndex(inst);	2228	const optional_ty = self.air.typeOfIndex(inst);
2156	const operand = try self.resolveInst(ty_op.operand);	2229	const operand = try self.resolveInst(ty_op.operand);
2157	operand.freezeIfRegister(&self.register_manager);	2230	const operand_lock: ?RegisterLock = switch (operand) {
2158	defer operand.unfreezeIfRegister(&self.register_manager);	2231	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2232	else => null,
		2233	};
		2234	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
2159		2235
2160	if (optional_ty.isPtrLikeOptional()) {	2236	if (optional_ty.isPtrLikeOptional()) {
2161	// TODO should we check if we can reuse the operand?	2237	// TODO should we check if we can reuse the operand?
...	@@ -2288,8 +2364,11 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi	...	@@ -2288,8 +2364,11 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
2288	fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	2364	fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2289	const slice_ty = self.air.typeOf(lhs);	2365	const slice_ty = self.air.typeOf(lhs);
2290	const slice_mcv = try self.resolveInst(lhs);	2366	const slice_mcv = try self.resolveInst(lhs);
2291	slice_mcv.freezeIfRegister(&self.register_manager);	2367	const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) {
2292	defer slice_mcv.unfreezeIfRegister(&self.register_manager);	2368	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2369	else => null,
		2370	};
		2371	defer if (slice_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
2293		2372
2294	const elem_ty = slice_ty.childType();	2373	const elem_ty = slice_ty.childType();
2295	const elem_size = elem_ty.abiSize(self.target.*);	2374	const elem_size = elem_ty.abiSize(self.target.*);
...	@@ -2298,12 +2377,15 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	...	@@ -2298,12 +2377,15 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2298		2377
2299	const index_ty = self.air.typeOf(rhs);	2378	const index_ty = self.air.typeOf(rhs);
2300	const index_mcv = try self.resolveInst(rhs);	2379	const index_mcv = try self.resolveInst(rhs);
2301	index_mcv.freezeIfRegister(&self.register_manager);	2380	const index_mcv_lock: ?RegisterLock = switch (index_mcv) {
2302	defer index_mcv.unfreezeIfRegister(&self.register_manager);	2381	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2382	else => null,
		2383	};
		2384	defer if (index_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
2303		2385
2304	const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);	2386	const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);
2305	self.register_manager.freezeRegs(&.{offset_reg});	2387	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2306	defer self.register_manager.unfreezeRegs(&.{offset_reg});	2388	defer self.register_manager.unlockReg(offset_reg_lock);
2307		2389
2308	const addr_reg = try self.register_manager.allocReg(null);	2390	const addr_reg = try self.register_manager.allocReg(null);
2309	switch (slice_mcv) {	2391	switch (slice_mcv) {
...	@@ -2356,98 +2438,119 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2356,98 +2438,119 @@ fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2356		2438
2357	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	2439	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2358	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	2440	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2359	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2360	const array_ty = self.air.typeOf(bin_op.lhs);
2361	const array = try self.resolveInst(bin_op.lhs);
2362	array.freezeIfRegister(&self.register_manager);
2363	defer array.unfreezeIfRegister(&self.register_manager);
2364
2365	const elem_ty = array_ty.childType();
2366	const elem_abi_size = elem_ty.abiSize(self.target.*);
2367		2441
2368	const index_ty = self.air.typeOf(bin_op.rhs);	2442	if (self.liveness.isUnused(inst)) {
2369	const index = try self.resolveInst(bin_op.rhs);	2443	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2370	index.freezeIfRegister(&self.register_manager);	2444	}
2371	defer index.unfreezeIfRegister(&self.register_manager);
2372		2445
2373	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);	2446	const array_ty = self.air.typeOf(bin_op.lhs);
2374	self.register_manager.freezeRegs(&.{offset_reg});	2447	const array = try self.resolveInst(bin_op.lhs);
2375	defer self.register_manager.unfreezeRegs(&.{offset_reg});	2448	const array_lock: ?RegisterLock = switch (array) {
		2449	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2450	else => null,
		2451	};
		2452	defer if (array_lock) \|lock\| self.register_manager.unlockReg(lock);
2376		2453
2377	const addr_reg = try self.register_manager.allocReg(null);	2454	const elem_ty = array_ty.childType();
2378	switch (array) {	2455	const elem_abi_size = elem_ty.abiSize(self.target.*);
2379	.register => {
2380	const off = @intCast(i32, try self.allocMem(
2381	inst,
2382	@intCast(u32, array_ty.abiSize(self.target.*)),
2383	array_ty.abiAlignment(self.target.*),
2384	));
2385	try self.genSetStack(array_ty, off, array, .{});
2386	// lea reg, [rbp]
2387	_ = try self.addInst(.{
2388	.tag = .lea,
2389	.ops = (Mir.Ops{
2390	.reg1 = addr_reg.to64(),
2391	.reg2 = .rbp,
2392	}).encode(),
2393	.data = .{ .imm = @bitCast(u32, -off) },
2394	});
2395	},
2396	.stack_offset => \|off\| {
2397	// lea reg, [rbp]
2398	_ = try self.addInst(.{
2399	.tag = .lea,
2400	.ops = (Mir.Ops{
2401	.reg1 = addr_reg.to64(),
2402	.reg2 = .rbp,
2403	}).encode(),
2404	.data = .{ .imm = @bitCast(u32, -off) },
2405	});
2406	},
2407	.memory,
2408	.got_load,
2409	.direct_load,
2410	=> {
2411	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
2412	},
2413	else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2414	}
2415		2456
2416	// TODO we could allocate register here, but need to expect addr register and potentially	2457	const index_ty = self.air.typeOf(bin_op.rhs);
2417	// offset register.	2458	const index = try self.resolveInst(bin_op.rhs);
2418	const dst_mcv = try self.allocRegOrMem(inst, false);	2459	const index_lock: ?RegisterLock = switch (index) {
2419	try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });	2460	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
2420	try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);	2461	else => null,
2421	break :result dst_mcv;
2422	};	2462	};
2423	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2463	defer if (index_lock) \|lock\| self.register_manager.unlockReg(lock);
		2464
		2465	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
		2466	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
		2467	defer self.register_manager.unlockReg(offset_reg_lock);
		2468
		2469	const addr_reg = try self.register_manager.allocReg(null);
		2470	switch (array) {
		2471	.register => {
		2472	const off = @intCast(i32, try self.allocMem(
		2473	inst,
		2474	@intCast(u32, array_ty.abiSize(self.target.*)),
		2475	array_ty.abiAlignment(self.target.*),
		2476	));
		2477	try self.genSetStack(array_ty, off, array, .{});
		2478	// lea reg, [rbp]
		2479	_ = try self.addInst(.{
		2480	.tag = .lea,
		2481	.ops = (Mir.Ops{
		2482	.reg1 = addr_reg.to64(),
		2483	.reg2 = .rbp,
		2484	}).encode(),
		2485	.data = .{ .imm = @bitCast(u32, -off) },
		2486	});
		2487	},
		2488	.stack_offset => \|off\| {
		2489	// lea reg, [rbp]
		2490	_ = try self.addInst(.{
		2491	.tag = .lea,
		2492	.ops = (Mir.Ops{
		2493	.reg1 = addr_reg.to64(),
		2494	.reg2 = .rbp,
		2495	}).encode(),
		2496	.data = .{ .imm = @bitCast(u32, -off) },
		2497	});
		2498	},
		2499	.memory,
		2500	.got_load,
		2501	.direct_load,
		2502	=> {
		2503	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
		2504	},
		2505	else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
		2506	}
		2507
		2508	// TODO we could allocate register here, but need to expect addr register and potentially
		2509	// offset register.
		2510	const dst_mcv = try self.allocRegOrMem(inst, false);
		2511	try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
		2512	try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);
		2513
		2514	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2424	}	2515	}
2425		2516
2426	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	2517	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2427	const is_volatile = false; // TODO	2518	const is_volatile = false; // TODO
2428	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	2519	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2429	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else result: {	2520
2430	// this is identical to the `airPtrElemPtr` codegen expect here an	2521	if (!is_volatile and self.liveness.isUnused(inst)) {
2431	// additional `mov` is needed at the end to get the actual value	2522	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2432		2523	}
2433	const ptr_ty = self.air.typeOf(bin_op.lhs);	2524
2434	const ptr = try self.resolveInst(bin_op.lhs);	2525	// this is identical to the `airPtrElemPtr` codegen expect here an
2435	ptr.freezeIfRegister(&self.register_manager);	2526	// additional `mov` is needed at the end to get the actual value
2436	defer ptr.unfreezeIfRegister(&self.register_manager);	2527
2437		2528	const ptr_ty = self.air.typeOf(bin_op.lhs);
2438	const elem_ty = ptr_ty.elemType2();	2529	const ptr = try self.resolveInst(bin_op.lhs);
2439	const elem_abi_size = elem_ty.abiSize(self.target.*);	2530	const ptr_lock: ?RegisterLock = switch (ptr) {
2440	const index_ty = self.air.typeOf(bin_op.rhs);	2531	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
2441	const index = try self.resolveInst(bin_op.rhs);	2532	else => null,
2442	index.freezeIfRegister(&self.register_manager);	2533	};
2443	defer index.unfreezeIfRegister(&self.register_manager);	2534	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
2444		2535
2445	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);	2536	const elem_ty = ptr_ty.elemType2();
2446	self.register_manager.freezeRegs(&.{offset_reg});	2537	const elem_abi_size = elem_ty.abiSize(self.target.*);
2447	defer self.register_manager.unfreezeRegs(&.{offset_reg});	2538	const index_ty = self.air.typeOf(bin_op.rhs);
2448		2539	const index = try self.resolveInst(bin_op.rhs);
2449	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);	2540	const index_lock: ?RegisterLock = switch (index) {
2450	try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });	2541	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2542	else => null,
		2543	};
		2544	defer if (index_lock) \|lock\| self.register_manager.unlockReg(lock);
		2545
		2546	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
		2547	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
		2548	defer self.register_manager.unlockReg(offset_reg_lock);
		2549
		2550	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
		2551	try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
		2552
		2553	const result: MCValue = result: {
2451	if (elem_abi_size > 8) {	2554	if (elem_abi_size > 8) {
2452	return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});	2555	return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});
2453	} else {	2556	} else {
...	@@ -2464,34 +2567,44 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2464,34 +2567,44 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2464	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };	2567	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2465	}	2568	}
2466	};	2569	};
		2570
2467	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	2571	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2468	}	2572	}
2469		2573
2470	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {	2574	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2471	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;	2575	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2472	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;	2576	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2473	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	2577
2474	const ptr_ty = self.air.typeOf(extra.lhs);	2578	if (self.liveness.isUnused(inst)) {
2475	const ptr = try self.resolveInst(extra.lhs);	2579	return self.finishAir(inst, .dead, .{ extra.lhs, extra.rhs, .none });
2476	ptr.freezeIfRegister(&self.register_manager);	2580	}
2477	defer ptr.unfreezeIfRegister(&self.register_manager);	2581
2478		2582	const ptr_ty = self.air.typeOf(extra.lhs);
2479	const elem_ty = ptr_ty.elemType2();	2583	const ptr = try self.resolveInst(extra.lhs);
2480	const elem_abi_size = elem_ty.abiSize(self.target.*);	2584	const ptr_lock: ?RegisterLock = switch (ptr) {
2481	const index_ty = self.air.typeOf(extra.rhs);	2585	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
2482	const index = try self.resolveInst(extra.rhs);	2586	else => null,
2483	index.freezeIfRegister(&self.register_manager);
2484	defer index.unfreezeIfRegister(&self.register_manager);
2485
2486	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
2487	self.register_manager.freezeRegs(&.{offset_reg});
2488	defer self.register_manager.unfreezeRegs(&.{offset_reg});
2489
2490	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
2491	try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
2492	break :result dst_mcv;
2493	};	2587	};
2494	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });	2588	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
		2589
		2590	const elem_ty = ptr_ty.elemType2();
		2591	const elem_abi_size = elem_ty.abiSize(self.target.*);
		2592	const index_ty = self.air.typeOf(extra.rhs);
		2593	const index = try self.resolveInst(extra.rhs);
		2594	const index_lock: ?RegisterLock = switch (index) {
		2595	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2596	else => null,
		2597	};
		2598	defer if (index_lock) \|lock\| self.register_manager.unlockReg(lock);
		2599
		2600	const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
		2601	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
		2602	defer self.register_manager.unlockReg(offset_reg_lock);
		2603
		2604	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
		2605	try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
		2606
		2607	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
2495	}	2608	}
2496		2609
2497	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	2610	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -2506,12 +2619,18 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2506,12 +2619,18 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2506	}	2619	}
2507		2620
2508	const ptr = try self.resolveInst(bin_op.lhs);	2621	const ptr = try self.resolveInst(bin_op.lhs);
2509	ptr.freezeIfRegister(&self.register_manager);	2622	const ptr_lock: ?RegisterLock = switch (ptr) {
2510	defer ptr.unfreezeIfRegister(&self.register_manager);	2623	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2624	else => null,
		2625	};
		2626	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
2511		2627
2512	const tag = try self.resolveInst(bin_op.rhs);	2628	const tag = try self.resolveInst(bin_op.rhs);
2513	tag.freezeIfRegister(&self.register_manager);	2629	const tag_lock: ?RegisterLock = switch (tag) {
2514	defer tag.unfreezeIfRegister(&self.register_manager);	2630	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2631	else => null,
		2632	};
		2633	defer if (tag_lock) \|lock\| self.register_manager.unlockReg(lock);
2515		2634
2516	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {	2635	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
2517	// TODO reusing the operand	2636	// TODO reusing the operand
...	@@ -2541,8 +2660,11 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2541,8 +2660,11 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2541		2660
2542	// TODO reusing the operand	2661	// TODO reusing the operand
2543	const operand = try self.resolveInst(ty_op.operand);	2662	const operand = try self.resolveInst(ty_op.operand);
2544	operand.freezeIfRegister(&self.register_manager);	2663	const operand_lock: ?RegisterLock = switch (operand) {
2545	defer operand.unfreezeIfRegister(&self.register_manager);	2664	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		2665	else => null,
		2666	};
		2667	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
2546		2668
2547	const tag_abi_size = tag_ty.abiSize(self.target.*);	2669	const tag_abi_size = tag_ty.abiSize(self.target.*);
2548	const dst_mcv: MCValue = blk: {	2670	const dst_mcv: MCValue = blk: {
...	@@ -2689,8 +2811,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2689,8 +2811,8 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2689	try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });	2811	try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
2690	},	2812	},
2691	.register => \|reg\| {	2813	.register => \|reg\| {
2692	self.register_manager.freezeRegs(&.{reg});	2814	const reg_lock = self.register_manager.lockReg(reg);
2693	defer self.register_manager.unfreezeRegs(&.{reg});	2815	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
2694		2816
2695	switch (dst_mcv) {	2817	switch (dst_mcv) {
2696	.dead => unreachable,	2818	.dead => unreachable,
...	@@ -2815,8 +2937,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2815,8 +2937,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2815	try self.genSetStack(value_ty, off, value, .{});	2937	try self.genSetStack(value_ty, off, value, .{});
2816	},	2938	},
2817	.register => \|reg\| {	2939	.register => \|reg\| {
2818	self.register_manager.freezeRegs(&.{reg});	2940	const reg_lock = self.register_manager.lockReg(reg);
2819	defer self.register_manager.unfreezeRegs(&.{reg});	2941	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
2820		2942
2821	switch (value) {	2943	switch (value) {
2822	.none => unreachable,	2944	.none => unreachable,
...	@@ -2906,12 +3028,15 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2906,12 +3028,15 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2906	.direct_load,	3028	.direct_load,
2907	.memory,	3029	.memory,
2908	=> {	3030	=> {
2909	value.freezeIfRegister(&self.register_manager);	3031	const value_lock: ?RegisterLock = switch (value) {
2910	defer value.unfreezeIfRegister(&self.register_manager);	3032	.register => \|reg\| self.register_manager.lockReg(reg),
		3033	else => null,
		3034	};
		3035	defer if (value_lock) \|lock\| self.register_manager.unlockReg(lock);
2911		3036
2912	const addr_reg = try self.register_manager.allocReg(null);	3037	const addr_reg = try self.register_manager.allocReg(null);
2913	self.register_manager.freezeRegs(&.{addr_reg});	3038	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
2914	defer self.register_manager.unfreezeRegs(&.{addr_reg});	3039	defer self.register_manager.unlockReg(addr_reg_lock);
2915		3040
2916	try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);	3041	try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
2917		3042
...	@@ -2982,8 +3107,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2982,8 +3107,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2982	=> {	3107	=> {
2983	if (abi_size <= 8) {	3108	if (abi_size <= 8) {
2984	const tmp_reg = try self.register_manager.allocReg(null);	3109	const tmp_reg = try self.register_manager.allocReg(null);
2985	self.register_manager.freezeRegs(&.{tmp_reg});	3110	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2986	defer self.register_manager.unfreezeRegs(&.{tmp_reg});	3111	defer self.register_manager.unlockReg(tmp_reg_lock);
2987		3112
2988	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);	3113	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
2989		3114
...	@@ -3073,8 +3198,8 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde	...	@@ -3073,8 +3198,8 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
3073	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{	3198	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
3074	.immediate = struct_field_offset,	3199	.immediate = struct_field_offset,
3075	});	3200	});
3076	self.register_manager.freezeRegs(&.{offset_reg});	3201	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3077	defer self.register_manager.unfreezeRegs(&.{offset_reg});	3202	defer self.register_manager.unlockReg(offset_reg_lock);
3078		3203
3079	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, mcv);	3204	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, mcv);
3080	try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });	3205	try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
...	@@ -3085,24 +3210,27 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde	...	@@ -3085,24 +3210,27 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
3085	break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };	3210	break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };
3086	},	3211	},
3087	.register => \|reg\| {	3212	.register => \|reg\| {
		3213	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
		3214	defer self.register_manager.unlockReg(reg_lock);
		3215
3088	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{	3216	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
3089	.immediate = struct_field_offset,	3217	.immediate = struct_field_offset,
3090	});	3218	});
3091	self.register_manager.freezeRegs(&.{offset_reg});	3219	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3092	defer self.register_manager.unfreezeRegs(&.{offset_reg});	3220	defer self.register_manager.unlockReg(offset_reg_lock);
3093		3221
3094	const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);	3222	const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
3095	const result_reg = blk: {	3223	const result_reg: Register = blk: {
3096	if (can_reuse_operand) {	3224	if (can_reuse_operand) {
3097	break :blk reg;	3225	break :blk reg;
3098	} else {	3226	} else {
3099	self.register_manager.freezeRegs(&.{reg});
3100	const result_reg = try self.register_manager.allocReg(inst);	3227	const result_reg = try self.register_manager.allocReg(inst);
3101	try self.genSetReg(ptr_ty, result_reg, mcv);	3228	try self.genSetReg(ptr_ty, result_reg, mcv);
3102	break :blk result_reg;	3229	break :blk result_reg;
3103	}	3230	}
3104	};	3231	};
3105	defer if (!can_reuse_operand) self.register_manager.unfreezeRegs(&.{reg});	3232	const result_reg_lock = self.register_manager.lockReg(result_reg);
		3233	defer if (result_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
3106		3234
3107	try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });	3235	try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });
3108	break :result MCValue{ .register = result_reg };	3236	break :result MCValue{ .register = result_reg };
...	@@ -3118,22 +3246,27 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3118,22 +3246,27 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3118	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;	3246	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
3119	const operand = extra.struct_operand;	3247	const operand = extra.struct_operand;
3120	const index = extra.field_index;	3248	const index = extra.field_index;
3121	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
3122	const mcv = try self.resolveInst(operand);
3123	const struct_ty = self.air.typeOf(operand);
3124	const struct_field_offset = struct_ty.structFieldOffset(index, self.target.*);
3125	const struct_field_ty = struct_ty.structFieldType(index);
3126		3249
		3250	if (self.liveness.isUnused(inst)) {
		3251	return self.finishAir(inst, .dead, .{ extra.struct_operand, .none, .none });
		3252	}
		3253
		3254	const mcv = try self.resolveInst(operand);
		3255	const struct_ty = self.air.typeOf(operand);
		3256	const struct_field_offset = struct_ty.structFieldOffset(index, self.target.*);
		3257	const struct_field_ty = struct_ty.structFieldType(index);
		3258
		3259	const result: MCValue = result: {
3127	switch (mcv) {	3260	switch (mcv) {
3128	.stack_offset => \|off\| {	3261	.stack_offset => \|off\| {
3129	const stack_offset = off - @intCast(i32, struct_field_offset);	3262	const stack_offset = off - @intCast(i32, struct_field_offset);
3130	break :result MCValue{ .stack_offset = stack_offset };	3263	break :result MCValue{ .stack_offset = stack_offset };
3131	},	3264	},
3132	.register => \|reg\| {	3265	.register => \|reg\| {
3133	self.register_manager.freezeRegs(&.{reg});	3266	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3134	defer self.register_manager.unfreezeRegs(&.{reg});	3267	defer self.register_manager.unlockReg(reg_lock);
3135		3268
3136	const dst_mcv = blk: {	3269	const dst_mcv: MCValue = blk: {
3137	if (self.reuseOperand(inst, operand, 0, mcv)) {	3270	if (self.reuseOperand(inst, operand, 0, mcv)) {
3138	break :blk mcv;	3271	break :blk mcv;
3139	} else {	3272	} else {
...	@@ -3143,8 +3276,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3143,8 +3276,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3143	break :blk dst_mcv;	3276	break :blk dst_mcv;
3144	}	3277	}
3145	};	3278	};
3146	dst_mcv.freezeIfRegister(&self.register_manager);	3279	const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3147	defer dst_mcv.unfreezeIfRegister(&self.register_manager);	3280	.register => \|a_reg\| self.register_manager.lockReg(a_reg),
		3281	else => null,
		3282	};
		3283	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
3148		3284
3149	// Shift by struct_field_offset.	3285	// Shift by struct_field_offset.
3150	const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));	3286	const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
...	@@ -3186,8 +3322,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3186,8 +3322,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3186	},	3322	},
3187	1 => {	3323	1 => {
3188	// Get overflow bit.	3324	// Get overflow bit.
3189	mcv.freezeIfRegister(&self.register_manager);	3325	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3190	defer mcv.unfreezeIfRegister(&self.register_manager);	3326	defer self.register_manager.unlockReg(reg_lock);
3191		3327
3192	const dst_reg = try self.register_manager.allocReg(inst);	3328	const dst_reg = try self.register_manager.allocReg(inst);
3193	const flags: u2 = switch (mcv) {	3329	const flags: u2 = switch (mcv) {
...	@@ -3229,15 +3365,21 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:	...	@@ -3229,15 +3365,21 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
3229	const dst_ty = self.air.typeOf(op_lhs);	3365	const dst_ty = self.air.typeOf(op_lhs);
3230		3366
3231	const lhs = try self.resolveInst(op_lhs);	3367	const lhs = try self.resolveInst(op_lhs);
3232	lhs.freezeIfRegister(&self.register_manager);	3368	const lhs_lock: ?RegisterLock = switch (lhs) {
3233	defer lhs.unfreezeIfRegister(&self.register_manager);	3369	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		3370	else => null,
		3371	};
		3372	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3234		3373
3235	const rhs = try self.resolveInst(op_rhs);	3374	const rhs = try self.resolveInst(op_rhs);
3236	rhs.freezeIfRegister(&self.register_manager);	3375	const rhs_lock: ?RegisterLock = switch (rhs) {
3237	defer rhs.unfreezeIfRegister(&self.register_manager);	3376	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		3377	else => null,
		3378	};
		3379	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3238		3380
3239	var flipped: bool = false;	3381	var flipped: bool = false;
3240	const dst_mcv = blk: {	3382	const dst_mcv: MCValue = blk: {
3241	if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {	3383	if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) {
3242	break :blk lhs;	3384	break :blk lhs;
3243	}	3385	}
...	@@ -3247,16 +3389,22 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:	...	@@ -3247,16 +3389,22 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
3247	}	3389	}
3248	break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);	3390	break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs);
3249	};	3391	};
3250	dst_mcv.freezeIfRegister(&self.register_manager);	3392	const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
3251	defer dst_mcv.unfreezeIfRegister(&self.register_manager);	3393	.register => \|reg\| self.register_manager.lockReg(reg),
		3394	else => null,
		3395	};
		3396	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
3252		3397
3253	const src_mcv = blk: {	3398	const src_mcv: MCValue = blk: {
3254	const mcv = if (flipped) lhs else rhs;	3399	const mcv = if (flipped) lhs else rhs;
3255	if (mcv.isRegister() or mcv.isMemory()) break :blk mcv;	3400	if (mcv.isRegister() or mcv.isMemory()) break :blk mcv;
3256	break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) };	3401	break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) };
3257	};	3402	};
3258	src_mcv.freezeIfRegister(&self.register_manager);	3403	const src_mcv_lock: ?RegisterLock = switch (src_mcv) {
3259	defer src_mcv.unfreezeIfRegister(&self.register_manager);	3404	.register => \|reg\| self.register_manager.lockReg(reg),
		3405	else => null,
		3406	};
		3407	defer if (src_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
3260		3408
3261	const tag = self.air.instructions.items(.tag)[inst];	3409	const tag = self.air.instructions.items(.tag)[inst];
3262	switch (tag) {	3410	switch (tag) {
...	@@ -3287,8 +3435,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC	...	@@ -3287,8 +3435,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
3287	.register_overflow_unsigned => unreachable,	3435	.register_overflow_unsigned => unreachable,
3288	.register_overflow_signed => unreachable,	3436	.register_overflow_signed => unreachable,
3289	.ptr_stack_offset => {	3437	.ptr_stack_offset => {
3290	self.register_manager.freezeRegs(&.{dst_reg});	3438	const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3291	defer self.register_manager.unfreezeRegs(&.{dst_reg});	3439	defer if (dst_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
		3440
3292	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);	3441	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3293	return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });	3442	return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
3294	},	3443	},
...	@@ -3318,8 +3467,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC	...	@@ -3318,8 +3467,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
3318	.compare_flags_unsigned,	3467	.compare_flags_unsigned,
3319	=> {	3468	=> {
3320	assert(abi_size <= 8);	3469	assert(abi_size <= 8);
3321	self.register_manager.freezeRegs(&.{dst_reg});	3470	const dst_reg_lock = self.register_manager.lockReg(dst_reg);
3322	defer self.register_manager.unfreezeRegs(&.{dst_reg});	3471	defer if (dst_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
		3472
3323	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);	3473	const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3324	return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });	3474	return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
3325	},	3475	},
...	@@ -3659,20 +3809,25 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.	...	@@ -3659,20 +3809,25 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
3659	try self.register_manager.getReg(reg, null);	3809	try self.register_manager.getReg(reg, null);
3660	}	3810	}
3661		3811
3662	if (info.return_value == .stack_offset) {	3812	const rdi_lock: ?RegisterLock = blk: {
3663	const ret_ty = fn_ty.fnReturnType();	3813	if (info.return_value == .stack_offset) {
3664	const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));	3814	const ret_ty = fn_ty.fnReturnType();
3665	const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));	3815	const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3666	const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));	3816	const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
3667	log.debug("airCall: return value on stack at offset {}", .{stack_offset});	3817	const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
		3818	log.debug("airCall: return value on stack at offset {}", .{stack_offset});
3668		3819
3669	try self.register_manager.getReg(.rdi, null);	3820	try self.register_manager.getReg(.rdi, null);
3670	try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });	3821	try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });
3671	self.register_manager.freezeRegs(&.{.rdi});	3822	const rdi_lock = self.register_manager.lockRegAssumeUnused(.rdi);
3672		3823
3673	info.return_value.stack_offset = stack_offset;	3824	info.return_value.stack_offset = stack_offset;
3674	}	3825
3675	defer if (info.return_value == .stack_offset) self.register_manager.unfreezeRegs(&.{.rdi});	3826	break :blk rdi_lock;
		3827	}
		3828	break :blk null;
		3829	};
		3830	defer if (rdi_lock) \|lock\| self.register_manager.unlockReg(lock);
3676		3831
3677	for (args) \|arg, arg_i\| {	3832	for (args) \|arg, arg_i\| {
3678	const mc_arg = info.args[arg_i];	3833	const mc_arg = info.args[arg_i];
...	@@ -3891,11 +4046,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3891,11 +4046,10 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
3891	const ret_ty = self.fn_type.fnReturnType();	4046	const ret_ty = self.fn_type.fnReturnType();
3892	switch (self.ret_mcv) {	4047	switch (self.ret_mcv) {
3893	.stack_offset => {	4048	.stack_offset => {
3894	self.register_manager.freezeRegs(&.{ .rax, .rcx });
3895	defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3896	const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);	4049	const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3897	self.register_manager.freezeRegs(&.{reg});	4050	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3898	defer self.register_manager.unfreezeRegs(&.{reg});	4051	defer self.register_manager.unlockReg(reg_lock);
		4052
3899	try self.genSetStack(ret_ty, 0, operand, .{	4053	try self.genSetStack(ret_ty, 0, operand, .{
3900	.source_stack_base = .rbp,	4054	.source_stack_base = .rbp,
3901	.dest_stack_base = reg,	4055	.dest_stack_base = reg,
...	@@ -3926,11 +4080,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3926,11 +4080,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
3926	const elem_ty = ptr_ty.elemType();	4080	const elem_ty = ptr_ty.elemType();
3927	switch (self.ret_mcv) {	4081	switch (self.ret_mcv) {
3928	.stack_offset => {	4082	.stack_offset => {
3929	self.register_manager.freezeRegs(&.{ .rax, .rcx });
3930	defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3931	const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);	4083	const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3932	self.register_manager.freezeRegs(&.{reg});	4084	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
3933	defer self.register_manager.unfreezeRegs(&.{reg});	4085	defer self.register_manager.unlockReg(reg_lock);
		4086
3934	try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{	4087	try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
3935	.source_stack_base = .rbp,	4088	.source_stack_base = .rbp,
3936	.dest_stack_base = reg,	4089	.dest_stack_base = reg,
...	@@ -3980,12 +4133,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -3980,12 +4133,15 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
3980	// Source operand can be an immediate, 8 bits or 32 bits.	4133	// Source operand can be an immediate, 8 bits or 32 bits.
3981	// TODO look into reusing the operand	4134	// TODO look into reusing the operand
3982	const lhs = try self.resolveInst(bin_op.lhs);	4135	const lhs = try self.resolveInst(bin_op.lhs);
3983	lhs.freezeIfRegister(&self.register_manager);	4136	const lhs_lock: ?RegisterLock = switch (lhs) {
3984	defer lhs.unfreezeIfRegister(&self.register_manager);	4137	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		4138	else => null,
		4139	};
		4140	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3985		4141
3986	const dst_reg = try self.copyToTmpRegister(ty, lhs);	4142	const dst_reg = try self.copyToTmpRegister(ty, lhs);
3987	self.register_manager.freezeRegs(&.{dst_reg});	4143	const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3988	defer self.register_manager.unfreezeRegs(&.{dst_reg});	4144	defer self.register_manager.unlockReg(dst_reg_lock);
3989		4145
3990	const dst_mcv = MCValue{ .register = dst_reg };	4146	const dst_mcv = MCValue{ .register = dst_reg };
3991		4147
...	@@ -4446,22 +4602,31 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4446,22 +4602,31 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
4446		4602
4447	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {	4603	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4448	const un_op = self.air.instructions.items(.data)[inst].un_op;	4604	const un_op = self.air.instructions.items(.data)[inst].un_op;
4449	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	4605
4450	const operand_ptr = try self.resolveInst(un_op);	4606	if (self.liveness.isUnused(inst)) {
4451	operand_ptr.freezeIfRegister(&self.register_manager);	4607	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4452	defer operand_ptr.unfreezeIfRegister(&self.register_manager);	4608	}
4453	const operand: MCValue = blk: {	4609
4454	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	4610	const operand_ptr = try self.resolveInst(un_op);
4455	// The MCValue that holds the pointer can be re-used as the value.	4611	const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4456	break :blk operand_ptr;	4612	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
4457	} else {	4613	else => null,
4458	break :blk try self.allocRegOrMem(inst, true);	4614	};
4459	}	4615	defer if (operand_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
4460	};	4616
4461	const ptr_ty = self.air.typeOf(un_op);	4617	const operand: MCValue = blk: {
4462	try self.load(operand, operand_ptr, ptr_ty);	4618	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4463	break :result try self.isNull(inst, ptr_ty.elemType(), operand);	4619	// The MCValue that holds the pointer can be re-used as the value.
		4620	break :blk operand_ptr;
		4621	} else {
		4622	break :blk try self.allocRegOrMem(inst, true);
		4623	}
4464	};	4624	};
		4625	const ptr_ty = self.air.typeOf(un_op);
		4626	try self.load(operand, operand_ptr, ptr_ty);
		4627
		4628	const result = try self.isNull(inst, ptr_ty.elemType(), operand);
		4629
4465	return self.finishAir(inst, result, .{ un_op, .none, .none });	4630	return self.finishAir(inst, result, .{ un_op, .none, .none });
4466	}	4631	}
4467		4632
...	@@ -4477,22 +4642,31 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4477,22 +4642,31 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
4477		4642
4478	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {	4643	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4479	const un_op = self.air.instructions.items(.data)[inst].un_op;	4644	const un_op = self.air.instructions.items(.data)[inst].un_op;
4480	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	4645
4481	const operand_ptr = try self.resolveInst(un_op);	4646	if (self.liveness.isUnused(inst)) {
4482	operand_ptr.freezeIfRegister(&self.register_manager);	4647	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4483	defer operand_ptr.unfreezeIfRegister(&self.register_manager);	4648	}
4484	const operand: MCValue = blk: {	4649
4485	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	4650	const operand_ptr = try self.resolveInst(un_op);
4486	// The MCValue that holds the pointer can be re-used as the value.	4651	const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4487	break :blk operand_ptr;	4652	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
4488	} else {	4653	else => null,
4489	break :blk try self.allocRegOrMem(inst, true);	4654	};
4490	}	4655	defer if (operand_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
4491	};	4656
4492	const ptr_ty = self.air.typeOf(un_op);	4657	const operand: MCValue = blk: {
4493	try self.load(operand, operand_ptr, ptr_ty);	4658	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4494	break :result try self.isNonNull(inst, ptr_ty.elemType(), operand);	4659	// The MCValue that holds the pointer can be re-used as the value.
		4660	break :blk operand_ptr;
		4661	} else {
		4662	break :blk try self.allocRegOrMem(inst, true);
		4663	}
4495	};	4664	};
		4665	const ptr_ty = self.air.typeOf(un_op);
		4666	try self.load(operand, operand_ptr, ptr_ty);
		4667
		4668	const result = try self.isNonNull(inst, ptr_ty.elemType(), operand);
		4669
4496	return self.finishAir(inst, result, .{ un_op, .none, .none });	4670	return self.finishAir(inst, result, .{ un_op, .none, .none });
4497	}	4671	}
4498		4672
...	@@ -4508,22 +4682,31 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4508,22 +4682,31 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
4508		4682
4509	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {	4683	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4510	const un_op = self.air.instructions.items(.data)[inst].un_op;	4684	const un_op = self.air.instructions.items(.data)[inst].un_op;
4511	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	4685
4512	const operand_ptr = try self.resolveInst(un_op);	4686	if (self.liveness.isUnused(inst)) {
4513	operand_ptr.freezeIfRegister(&self.register_manager);	4687	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4514	defer operand_ptr.unfreezeIfRegister(&self.register_manager);	4688	}
4515	const operand: MCValue = blk: {	4689
4516	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	4690	const operand_ptr = try self.resolveInst(un_op);
4517	// The MCValue that holds the pointer can be re-used as the value.	4691	const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4518	break :blk operand_ptr;	4692	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
4519	} else {	4693	else => null,
4520	break :blk try self.allocRegOrMem(inst, true);
4521	}
4522	};
4523	const ptr_ty = self.air.typeOf(un_op);
4524	try self.load(operand, operand_ptr, ptr_ty);
4525	break :result try self.isErr(inst, ptr_ty.elemType(), operand);
4526	};	4694	};
		4695	defer if (operand_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
		4696
		4697	const operand: MCValue = blk: {
		4698	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
		4699	// The MCValue that holds the pointer can be re-used as the value.
		4700	break :blk operand_ptr;
		4701	} else {
		4702	break :blk try self.allocRegOrMem(inst, true);
		4703	}
		4704	};
		4705	const ptr_ty = self.air.typeOf(un_op);
		4706	try self.load(operand, operand_ptr, ptr_ty);
		4707
		4708	const result = try self.isErr(inst, ptr_ty.elemType(), operand);
		4709
4527	return self.finishAir(inst, result, .{ un_op, .none, .none });	4710	return self.finishAir(inst, result, .{ un_op, .none, .none });
4528	}	4711	}
4529		4712
...	@@ -4539,22 +4722,31 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4539,22 +4722,31 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
4539		4722
4540	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {	4723	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
4541	const un_op = self.air.instructions.items(.data)[inst].un_op;	4724	const un_op = self.air.instructions.items(.data)[inst].un_op;
4542	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {	4725
4543	const operand_ptr = try self.resolveInst(un_op);	4726	if (self.liveness.isUnused(inst)) {
4544	operand_ptr.freezeIfRegister(&self.register_manager);	4727	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4545	defer operand_ptr.unfreezeIfRegister(&self.register_manager);	4728	}
4546	const operand: MCValue = blk: {	4729
4547	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {	4730	const operand_ptr = try self.resolveInst(un_op);
4548	// The MCValue that holds the pointer can be re-used as the value.	4731	const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4549	break :blk operand_ptr;	4732	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
4550	} else {	4733	else => null,
4551	break :blk try self.allocRegOrMem(inst, true);	4734	};
4552	}	4735	defer if (operand_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
4553	};	4736
4554	const ptr_ty = self.air.typeOf(un_op);	4737	const operand: MCValue = blk: {
4555	try self.load(operand, operand_ptr, ptr_ty);	4738	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
4556	break :result try self.isNonErr(inst, ptr_ty.elemType(), operand);	4739	// The MCValue that holds the pointer can be re-used as the value.
		4740	break :blk operand_ptr;
		4741	} else {
		4742	break :blk try self.allocRegOrMem(inst, true);
		4743	}
4557	};	4744	};
		4745	const ptr_ty = self.air.typeOf(un_op);
		4746	try self.load(operand, operand_ptr, ptr_ty);
		4747
		4748	const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);
		4749
4558	return self.finishAir(inst, result, .{ un_op, .none, .none });	4750	return self.finishAir(inst, result, .{ un_op, .none, .none });
4559	}	4751	}
4560		4752
...	@@ -4610,8 +4802,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4610,8 +4802,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4610	.register => \|cond_reg\| {	4802	.register => \|cond_reg\| {
4611	try self.spillCompareFlagsIfOccupied();	4803	try self.spillCompareFlagsIfOccupied();
4612		4804
4613	self.register_manager.freezeRegs(&.{cond_reg});	4805	const cond_reg_lock = self.register_manager.lockReg(cond_reg);
4614	defer self.register_manager.unfreezeRegs(&.{cond_reg});	4806	defer if (cond_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
4615		4807
4616	switch (case) {	4808	switch (case) {
4617	.none => unreachable,	4809	.none => unreachable,
...	@@ -4670,8 +4862,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4670,8 +4862,8 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4670		4862
4671	if (abi_size <= 8) {	4863	if (abi_size <= 8) {
4672	const reg = try self.copyToTmpRegister(ty, condition);	4864	const reg = try self.copyToTmpRegister(ty, condition);
4673	self.register_manager.freezeRegs(&.{reg});	4865	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4674	defer self.register_manager.unfreezeRegs(&.{reg});	4866	defer self.register_manager.unlockReg(reg_lock);
4675	return self.genCondSwitchMir(ty, .{ .register = reg }, case);	4867	return self.genCondSwitchMir(ty, .{ .register = reg }, case);
4676	}	4868	}
4677		4869
...	@@ -5158,8 +5350,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5158,8 +5350,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5158	.register_overflow_unsigned,	5350	.register_overflow_unsigned,
5159	.register_overflow_signed,	5351	.register_overflow_signed,
5160	=> \|reg\| {	5352	=> \|reg\| {
5161	self.register_manager.freezeRegs(&.{reg});	5353	const reg_lock = self.register_manager.lockReg(reg);
5162	defer self.register_manager.unfreezeRegs(&.{reg});	5354	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
5163		5355
5164	const wrapped_ty = ty.structFieldType(0);	5356	const wrapped_ty = ty.structFieldType(0);
5165	try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg }, .{});	5357	try self.genSetStack(wrapped_ty, stack_offset, .{ .register = reg }, .{});
...	@@ -5260,8 +5452,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5260,8 +5452,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5260		5452
5261	const base_reg = opts.dest_stack_base orelse .rbp;	5453	const base_reg = opts.dest_stack_base orelse .rbp;
5262	if (!math.isPowerOfTwo(abi_size)) {	5454	if (!math.isPowerOfTwo(abi_size)) {
5263	self.register_manager.freezeRegs(&.{reg});	5455	const reg_lock = self.register_manager.lockReg(reg);
5264	defer self.register_manager.unfreezeRegs(&.{reg});	5456	defer if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
5265		5457
5266	const tmp_reg = try self.copyToTmpRegister(ty, mcv);	5458	const tmp_reg = try self.copyToTmpRegister(ty, mcv);
5267		5459
...	@@ -5350,13 +5542,25 @@ fn genInlineMemcpy(	...	@@ -5350,13 +5542,25 @@ fn genInlineMemcpy(
5350	len: MCValue,	5542	len: MCValue,
5351	opts: InlineMemcpyOpts,	5543	opts: InlineMemcpyOpts,
5352	) InnerError!void {	5544	) InnerError!void {
5353	self.register_manager.freezeRegs(&.{ .rax, .rcx });	5545	try self.register_manager.getReg(.rax, null);
		5546	try self.register_manager.getReg(.rcx, null);
5354		5547
5355	if (opts.source_stack_base) \|reg\| self.register_manager.freezeRegs(&.{reg});	5548	const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rcx });
5356	defer if (opts.source_stack_base) \|reg\| self.register_manager.unfreezeRegs(&.{reg});	5549	defer for (reg_locks) \|lock\| {
		5550	self.register_manager.unlockReg(lock);
		5551	};
		5552
		5553	const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) \|reg\|
		5554	self.register_manager.lockReg(reg)
		5555	else
		5556	null;
		5557	defer if (ssbase_lock) \|reg\| self.register_manager.unlockReg(reg);
5357		5558
5358	if (opts.dest_stack_base) \|reg\| self.register_manager.freezeRegs(&.{reg});	5559	const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) \|reg\|
5359	defer if (opts.dest_stack_base) \|reg\| self.register_manager.unfreezeRegs(&.{reg});	5560	self.register_manager.lockReg(reg)
		5561	else
		5562	null;
		5563	defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
5360		5564
5361	const dst_addr_reg = try self.register_manager.allocReg(null);	5565	const dst_addr_reg = try self.register_manager.allocReg(null);
5362	switch (dst_ptr) {	5566	switch (dst_ptr) {
...	@@ -5390,8 +5594,8 @@ fn genInlineMemcpy(	...	@@ -5390,8 +5594,8 @@ fn genInlineMemcpy(
5390	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});	5594	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
5391	},	5595	},
5392	}	5596	}
5393	self.register_manager.freezeRegs(&.{dst_addr_reg});	5597	const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5394	defer self.register_manager.unfreezeRegs(&.{dst_addr_reg});	5598	defer self.register_manager.unlockReg(dst_addr_reg_lock);
5395		5599
5396	const src_addr_reg = try self.register_manager.allocReg(null);	5600	const src_addr_reg = try self.register_manager.allocReg(null);
5397	switch (src_ptr) {	5601	switch (src_ptr) {
...	@@ -5425,18 +5629,13 @@ fn genInlineMemcpy(	...	@@ -5425,18 +5629,13 @@ fn genInlineMemcpy(
5425	return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});	5629	return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
5426	},	5630	},
5427	}	5631	}
5428	self.register_manager.freezeRegs(&.{src_addr_reg});	5632	const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5429	defer self.register_manager.unfreezeRegs(&.{src_addr_reg});	5633	defer self.register_manager.unlockReg(src_addr_reg_lock);
5430		5634
5431	const regs = try self.register_manager.allocRegs(2, .{ null, null });	5635	const regs = try self.register_manager.allocRegs(2, .{ null, null });
5432	const count_reg = regs[0].to64();	5636	const count_reg = regs[0].to64();
5433	const tmp_reg = regs[1].to8();	5637	const tmp_reg = regs[1].to8();
5434		5638
5435	self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
5436
5437	try self.register_manager.getReg(.rax, null);
5438	try self.register_manager.getReg(.rcx, null);
5439
5440	try self.genSetReg(Type.usize, count_reg, len);	5639	try self.genSetReg(Type.usize, count_reg, len);
5441		5640
5442	// mov rcx, 0	5641	// mov rcx, 0
...	@@ -5540,7 +5739,9 @@ fn genInlineMemset(	...	@@ -5540,7 +5739,9 @@ fn genInlineMemset(
5540	len: MCValue,	5739	len: MCValue,
5541	opts: InlineMemcpyOpts,	5740	opts: InlineMemcpyOpts,
5542	) InnerError!void {	5741	) InnerError!void {
5543	self.register_manager.freezeRegs(&.{.rax});	5742	try self.register_manager.getReg(.rax, null);
		5743	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
		5744	defer self.register_manager.unlockReg(rax_lock);
5544		5745
5545	const addr_reg = try self.register_manager.allocReg(null);	5746	const addr_reg = try self.register_manager.allocReg(null);
5546	switch (dst_ptr) {	5747	switch (dst_ptr) {
...	@@ -5574,11 +5775,8 @@ fn genInlineMemset(	...	@@ -5574,11 +5775,8 @@ fn genInlineMemset(
5574	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});	5775	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
5575	},	5776	},
5576	}	5777	}
5577	self.register_manager.freezeRegs(&.{addr_reg});	5778	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
5578	defer self.register_manager.unfreezeRegs(&.{addr_reg});	5779	defer self.register_manager.unlockReg(addr_reg_lock);
5579
5580	self.register_manager.unfreezeRegs(&.{.rax});
5581	try self.register_manager.getReg(.rax, null);
5582		5780
5583	try self.genSetReg(Type.usize, .rax, len);	5781	try self.genSetReg(Type.usize, .rax, len);
5584	try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });	5782	try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });
...	@@ -6017,16 +6215,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6017,16 +6215,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
6017	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	6215	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6018		6216
6019	const dst_ptr = try self.resolveInst(pl_op.operand);	6217	const dst_ptr = try self.resolveInst(pl_op.operand);
6020	dst_ptr.freezeIfRegister(&self.register_manager);	6218	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
6021	defer dst_ptr.unfreezeIfRegister(&self.register_manager);	6219	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		6220	else => null,
		6221	};
		6222	defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
6022		6223
6023	const src_val = try self.resolveInst(extra.lhs);	6224	const src_val = try self.resolveInst(extra.lhs);
6024	src_val.freezeIfRegister(&self.register_manager);	6225	const src_val_lock: ?RegisterLock = switch (src_val) {
6025	defer src_val.unfreezeIfRegister(&self.register_manager);	6226	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		6227	else => null,
		6228	};
		6229	defer if (src_val_lock) \|lock\| self.register_manager.unlockReg(lock);
6026		6230
6027	const len = try self.resolveInst(extra.rhs);	6231	const len = try self.resolveInst(extra.rhs);
6028	len.freezeIfRegister(&self.register_manager);	6232	const len_lock: ?RegisterLock = switch (len) {
6029	defer len.unfreezeIfRegister(&self.register_manager);	6233	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		6234	else => null,
		6235	};
		6236	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
6030		6237
6031	try self.genInlineMemset(dst_ptr, src_val, len, .{});	6238	try self.genInlineMemset(dst_ptr, src_val, len, .{});
6032		6239
...	@@ -6038,17 +6245,26 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6038,17 +6245,26 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6038	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	6245	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6039		6246
6040	const dst_ptr = try self.resolveInst(pl_op.operand);	6247	const dst_ptr = try self.resolveInst(pl_op.operand);
6041	dst_ptr.freezeIfRegister(&self.register_manager);	6248	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
6042	defer dst_ptr.unfreezeIfRegister(&self.register_manager);	6249	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		6250	else => null,
		6251	};
		6252	defer if (dst_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
6043		6253
6044	const src_ty = self.air.typeOf(extra.lhs);	6254	const src_ty = self.air.typeOf(extra.lhs);
6045	const src_ptr = try self.resolveInst(extra.lhs);	6255	const src_ptr = try self.resolveInst(extra.lhs);
6046	src_ptr.freezeIfRegister(&self.register_manager);	6256	const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
6047	defer src_ptr.unfreezeIfRegister(&self.register_manager);	6257	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		6258	else => null,
		6259	};
		6260	defer if (src_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
6048		6261
6049	const len = try self.resolveInst(extra.rhs);	6262	const len = try self.resolveInst(extra.rhs);
6050	len.freezeIfRegister(&self.register_manager);	6263	const len_lock: ?RegisterLock = switch (len) {
6051	defer len.unfreezeIfRegister(&self.register_manager);	6264	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
		6265	else => null,
		6266	};
		6267	defer if (len_lock) \|lock\| self.register_manager.unlockReg(lock);
6052		6268
6053	// TODO Is this the only condition for pointer dereference for memcpy?	6269	// TODO Is this the only condition for pointer dereference for memcpy?
6054	const src: MCValue = blk: {	6270	const src: MCValue = blk: {
...	@@ -6070,8 +6286,11 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6070,8 +6286,11 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6070	else => break :blk src_ptr,	6286	else => break :blk src_ptr,
6071	}	6287	}
6072	};	6288	};
6073	src.freezeIfRegister(&self.register_manager);	6289	const src_lock: ?RegisterLock = switch (src) {
6074	defer src.unfreezeIfRegister(&self.register_manager);	6290	.register => \|reg\| self.register_manager.lockReg(reg),
		6291	else => null,
		6292	};
		6293	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
6075		6294
6076	try self.genInlineMemcpy(dst_ptr, src, len, .{});	6295	try self.genInlineMemcpy(dst_ptr, src, len, .{});
6077		6296

src/register_manager.zig+79-46

...	@@ -45,9 +45,8 @@ pub fn RegisterManager(	...	@@ -45,9 +45,8 @@ pub fn RegisterManager(
45	/// Tracks all registers allocated in the course of this	45	/// Tracks all registers allocated in the course of this
46	/// function	46	/// function
47	allocated_registers: FreeRegInt = 0,	47	allocated_registers: FreeRegInt = 0,
48	/// Tracks registers which are temporarily blocked from being	48	/// Tracks registers which are locked from being allocated
49	/// allocated	49	locked_registers: FreeRegInt = 0,
50	frozen_registers: FreeRegInt = 0,
51		50
52	const Self = @This();	51	const Self = @This();
53		52
...	@@ -108,34 +107,70 @@ pub fn RegisterManager(	...	@@ -108,34 +107,70 @@ pub fn RegisterManager(
108	return self.allocated_registers & mask != 0;	107	return self.allocated_registers & mask != 0;
109	}	108	}
110		109
111	/// Returns whether this register is frozen	110	/// Returns whether this register is locked
112	///	111	///
113	/// Returns false when this register is not tracked	112	/// Returns false when this register is not tracked
114	pub fn isRegFrozen(self: Self, reg: Register) bool {	113	pub fn isRegLocked(self: Self, reg: Register) bool {
115	const mask = getRegisterMask(reg) orelse return false;	114	const mask = getRegisterMask(reg) orelse return false;
116	return self.frozen_registers & mask != 0;	115	return self.locked_registers & mask != 0;
117	}	116	}
118		117
119	/// Prevents the registers from being allocated until they are	118	pub const RegisterLock = struct {
120	/// unfrozen again	119	register: Register,
121	pub fn freezeRegs(self: *Self, regs: []const Register) void {	120	};
122	for (regs) \|reg\| {	121
123	const mask = getRegisterMask(reg) orelse continue;	122	/// Prevents the register from being allocated until they are
124	self.frozen_registers \|= mask;	123	/// unlocked again.
		124	/// Returns `RegisterLock` if the register was not already
		125	/// locked, or `null` otherwise.
		126	/// Only the owner of the `RegisterLock` can unlock the
		127	/// register later.
		128	pub fn lockReg(self: *Self, reg: Register) ?RegisterLock {
		129	log.debug("locking {}", .{reg});
		130	if (self.isRegLocked(reg)) {
		131	log.debug(" register already locked", .{});
		132	return null;
125	}	133	}
		134	const mask = getRegisterMask(reg) orelse return null;
		135	self.locked_registers \|= mask;
		136	return RegisterLock{ .register = reg };
126	}	137	}
127		138
128	/// Enables the allocation of the registers	139	/// Like `lockReg` but asserts the register was unused always
129	pub fn unfreezeRegs(self: *Self, regs: []const Register) void {	140	/// returning a valid lock.
130	for (regs) \|reg\| {	141	pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock {
131	const mask = getRegisterMask(reg) orelse continue;	142	log.debug("locking asserting free {}", .{reg});
132	self.frozen_registers &= ~mask;	143	assert(!self.isRegLocked(reg));
		144	const mask = getRegisterMask(reg) orelse unreachable;
		145	self.locked_registers \|= mask;
		146	return RegisterLock{ .register = reg };
		147	}
		148
		149	/// Like `lockRegAssumeUnused` but locks multiple registers.
		150	pub fn lockRegsAssumeUnused(
		151	self: *Self,
		152	comptime count: comptime_int,
		153	regs: [count]Register,
		154	) [count]RegisterLock {
		155	var buf: [count]RegisterLock = undefined;
		156	for (regs) \|reg, i\| {
		157	buf[i] = self.lockRegAssumeUnused(reg);
133	}	158	}
		159	return buf;
		160	}
		161
		162	/// Unlocks the register allowing its re-allocation and re-use.
		163	/// Requires `RegisterLock` to unlock a register.
		164	/// Call `lockReg` to obtain the lock first.
		165	pub fn unlockReg(self: *Self, lock: RegisterLock) void {
		166	log.debug("unlocking {}", .{lock.register});
		167	const mask = getRegisterMask(lock.register) orelse return;
		168	self.locked_registers &= ~mask;
134	}	169	}
135		170
136	/// Returns true when at least one register is frozen	171	/// Returns true when at least one register is locked
137	pub fn frozenRegsExist(self: Self) bool {	172	pub fn lockedRegsExist(self: Self) bool {
138	return self.frozen_registers != 0;	173	return self.locked_registers != 0;
139	}	174	}
140		175
141	/// Allocates a specified number of registers, optionally	176	/// Allocates a specified number of registers, optionally
...	@@ -148,15 +183,15 @@ pub fn RegisterManager(	...	@@ -148,15 +183,15 @@ pub fn RegisterManager(
148	) ?[count]Register {	183	) ?[count]Register {
149	comptime assert(count > 0 and count <= tracked_registers.len);	184	comptime assert(count > 0 and count <= tracked_registers.len);
150		185
151	const free_and_not_frozen_registers = self.free_registers & ~self.frozen_registers;	186	const free_and_not_locked_registers = self.free_registers & ~self.locked_registers;
152	const free_and_not_frozen_registers_count = @popCount(FreeRegInt, free_and_not_frozen_registers);	187	const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers);
153	if (free_and_not_frozen_registers_count < count) return null;	188	if (free_and_not_locked_registers_count < count) return null;
154		189
155	var regs: [count]Register = undefined;	190	var regs: [count]Register = undefined;
156	var i: usize = 0;	191	var i: usize = 0;
157	for (tracked_registers) \|reg\| {	192	for (tracked_registers) \|reg\| {
158	if (i >= count) break;	193	if (i >= count) break;
159	if (self.isRegFrozen(reg)) continue;	194	if (self.isRegLocked(reg)) continue;
160	if (!self.isRegFree(reg)) continue;	195	if (!self.isRegFree(reg)) continue;
161		196
162	regs[i] = reg;	197	regs[i] = reg;
...	@@ -194,8 +229,8 @@ pub fn RegisterManager(	...	@@ -194,8 +229,8 @@ pub fn RegisterManager(
194	insts: [count]?Air.Inst.Index,	229	insts: [count]?Air.Inst.Index,
195	) AllocateRegistersError![count]Register {	230	) AllocateRegistersError![count]Register {
196	comptime assert(count > 0 and count <= tracked_registers.len);	231	comptime assert(count > 0 and count <= tracked_registers.len);
197	const frozen_registers_count = @popCount(FreeRegInt, self.frozen_registers);	232	const locked_registers_count = @popCount(FreeRegInt, self.locked_registers);
198	if (count > tracked_registers.len - frozen_registers_count) return error.OutOfRegisters;	233	if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters;
199		234
200	const result = self.tryAllocRegs(count, insts) orelse blk: {	235	const result = self.tryAllocRegs(count, insts) orelse blk: {
201	// We'll take over the first count registers. Spill	236	// We'll take over the first count registers. Spill
...	@@ -205,7 +240,7 @@ pub fn RegisterManager(	...	@@ -205,7 +240,7 @@ pub fn RegisterManager(
205	var i: usize = 0;	240	var i: usize = 0;
206	for (tracked_registers) \|reg\| {	241	for (tracked_registers) \|reg\| {
207	if (i >= count) break;	242	if (i >= count) break;
208	if (self.isRegFrozen(reg)) continue;	243	if (self.isRegLocked(reg)) continue;
209		244
210	regs[i] = reg;	245	regs[i] = reg;
211	self.markRegAllocated(reg);	246	self.markRegAllocated(reg);
...	@@ -416,15 +451,15 @@ test "allocReg: spilling" {	...	@@ -416,15 +451,15 @@ test "allocReg: spilling" {
416	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));	451	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
417	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);	452	try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items);
418		453
419	// Frozen registers	454	// Locked registers
420	function.register_manager.freeReg(.r3);	455	function.register_manager.freeReg(.r3);
421	{	456	{
422	function.register_manager.freezeRegs(&.{.r2});	457	const lock = function.register_manager.lockReg(.r2);
423	defer function.register_manager.unfreezeRegs(&.{.r2});	458	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
424		459
425	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));	460	try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction));
426	}	461	}
427	try expect(!function.register_manager.frozenRegsExist());	462	try expect(!function.register_manager.lockedRegsExist());
428	}	463	}
429		464
430	test "tryAllocRegs" {	465	test "tryAllocRegs" {
...	@@ -442,17 +477,17 @@ test "tryAllocRegs" {	...	@@ -442,17 +477,17 @@ test "tryAllocRegs" {
442	try expect(function.register_manager.isRegAllocated(.r2));	477	try expect(function.register_manager.isRegAllocated(.r2));
443	try expect(!function.register_manager.isRegAllocated(.r3));	478	try expect(!function.register_manager.isRegAllocated(.r3));
444		479
445	// Frozen registers	480	// Locked registers
446	function.register_manager.freeReg(.r0);	481	function.register_manager.freeReg(.r0);
447	function.register_manager.freeReg(.r2);	482	function.register_manager.freeReg(.r2);
448	function.register_manager.freeReg(.r3);	483	function.register_manager.freeReg(.r3);
449	{	484	{
450	function.register_manager.freezeRegs(&.{.r1});	485	const lock = function.register_manager.lockReg(.r1);
451	defer function.register_manager.unfreezeRegs(&.{.r1});	486	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
452		487
453	try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);	488	try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?);
454	}	489	}
455	try expect(!function.register_manager.frozenRegsExist());	490	try expect(!function.register_manager.lockedRegsExist());
456		491
457	try expect(function.register_manager.isRegAllocated(.r0));	492	try expect(function.register_manager.isRegAllocated(.r0));
458	try expect(function.register_manager.isRegAllocated(.r1));	493	try expect(function.register_manager.isRegAllocated(.r1));
...	@@ -475,19 +510,19 @@ test "allocRegs: normal usage" {	...	@@ -475,19 +510,19 @@ test "allocRegs: normal usage" {
475		510
476	// The result register is known and fixed at this point, we	511	// The result register is known and fixed at this point, we
477	// don't want to accidentally allocate lhs or rhs to the	512	// don't want to accidentally allocate lhs or rhs to the
478	// result register, this is why we freeze it.	513	// result register, this is why we lock it.
479	//	514	//
480	// Using defer unfreeze right after freeze is a good idea in	515	// Using defer unlock right after lock is a good idea in
481	// most cases as you probably are using the frozen registers	516	// most cases as you probably are using the locked registers
482	// in the remainder of this scope and don't need to use it	517	// in the remainder of this scope and don't need to use it
483	// after the end of this scope. However, in some situations,	518	// after the end of this scope. However, in some situations,
484	// it may make sense to manually unfreeze registers before the	519	// it may make sense to manually unlock registers before the
485	// end of the scope when you are certain that they don't	520	// end of the scope when you are certain that they don't
486	// contain any valuable data anymore and can be reused. For an	521	// contain any valuable data anymore and can be reused. For an
487	// example of that, see `selectively reducing register	522	// example of that, see `selectively reducing register
488	// pressure`.	523	// pressure`.
489	function.register_manager.freezeRegs(&.{result_reg});	524	const lock = function.register_manager.lockReg(result_reg);
490	defer function.register_manager.unfreezeRegs(&.{result_reg});	525	defer if (lock) \|reg\| function.register_manager.unlockReg(reg);
491		526
492	const regs = try function.register_manager.allocRegs(2, .{ null, null });	527	const regs = try function.register_manager.allocRegs(2, .{ null, null });
493	try function.genAdd(result_reg, regs[0], regs[1]);	528	try function.genAdd(result_reg, regs[0], regs[1]);
...	@@ -507,16 +542,14 @@ test "allocRegs: selectively reducing register pressure" {	...	@@ -507,16 +542,14 @@ test "allocRegs: selectively reducing register pressure" {
507	{	542	{
508	const result_reg: MockRegister2 = .r1;	543	const result_reg: MockRegister2 = .r1;
509		544
510	function.register_manager.freezeRegs(&.{result_reg});	545	const lock = function.register_manager.lockReg(result_reg);
511	defer function.register_manager.unfreezeRegs(&.{result_reg});
512		546
513	// Here, we don't defer unfreeze because we manually unfreeze	547	// Here, we don't defer unlock because we manually unlock
514	// after genAdd	548	// after genAdd
515	const regs = try function.register_manager.allocRegs(2, .{ null, null });	549	const regs = try function.register_manager.allocRegs(2, .{ null, null });
516	function.register_manager.freezeRegs(&.{result_reg});
517		550
518	try function.genAdd(result_reg, regs[0], regs[1]);	551	try function.genAdd(result_reg, regs[0], regs[1]);
519	function.register_manager.unfreezeRegs(&regs);	552	function.register_manager.unlockReg(lock.?);
520		553
521	const extra_summand_reg = try function.register_manager.allocReg(null);	554	const extra_summand_reg = try function.register_manager.allocReg(null);
522	try function.genAdd(result_reg, result_reg, extra_summand_reg);	555	try function.genAdd(result_reg, result_reg, extra_summand_reg);

test/behavior/align.zig+2

...	@@ -7,6 +7,8 @@ var foo: u8 align(4) = 100;	...	@@ -7,6 +7,8 @@ var foo: u8 align(4) = 100;
7		7
8	test "global variable alignment" {	8	test "global variable alignment" {
9	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO	9	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
		10	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
		11	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10		12
11	comptime try expect(@typeInfo(@TypeOf(&foo)).Pointer.alignment == 4);	13	comptime try expect(@typeInfo(@TypeOf(&foo)).Pointer.alignment == 4);
12	comptime try expect(@TypeOf(&foo) == *align(4) u8);	14	comptime try expect(@TypeOf(&foo) == *align(4) u8);

test/behavior/byval_arg_var.zig+1

...	@@ -6,6 +6,7 @@ var result: []const u8 = "wrong";	...	@@ -6,6 +6,7 @@ var result: []const u8 = "wrong";
6	test "pass string literal byvalue to a generic var param" {	6	test "pass string literal byvalue to a generic var param" {
7	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;	7	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
8	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	8	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
		9	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
9		10
10	start();	11	start();
11	blowUpStack(10);	12	blowUpStack(10);