authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-06 02:06:03+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-12 05:25:07+01:00
logf2a62d25fc722da9dd4fb09ec614e278271442b9
tree839cb80844e4c97e5c3c68ac2103dc7f91e35017
parentc619551f7d0aae794d628fc7d2f34cfc70f1cba3

x86_64: implement `ret_safe`

2 files changed, 348 insertions(+), 191 deletions(-)

src/arch/x86_64/CodeGen.zig+348-190
......@@ -537,7 +537,7 @@ const InstTracking = struct {
537537 else => unreachable,
538538 }
539539 tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
540 try function.genCopy(function.typeOfIndex(inst), self.long, self.short);
540 try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{});
541541 }
542542
543543 fn reuseFrame(self: *InstTracking) void {
......@@ -637,8 +637,8 @@ const InstTracking = struct {
637637 ) !void {
638638 const ty = function.typeOfIndex(inst);
639639 if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame)
640 try function.genCopy(ty, target.long, self.short);
641 try function.genCopy(ty, target.short, self.short);
640 try function.genCopy(ty, target.long, self.short, .{});
641 try function.genCopy(ty, target.short, self.short, .{});
642642 }
643643
644644 fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
......@@ -1717,6 +1717,7 @@ fn gen(self: *Self) InnerError!void {
17171717 0,
17181718 Type.usize,
17191719 self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
1720 .{},
17201721 );
17211722 self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
17221723 tracking_log.debug("spill {} to {}", .{ self.ret_mcv.long, frame_index });
......@@ -1740,6 +1741,7 @@ fn gen(self: *Self) InnerError!void {
17401741 @intCast(reg_i * 8),
17411742 Type.usize,
17421743 .{ .register = reg },
1744 .{},
17431745 );
17441746
17451747 try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count));
......@@ -1752,6 +1754,7 @@ fn gen(self: *Self) InnerError!void {
17521754 @intCast(abi.SysV.c_abi_int_param_regs.len * 8 + reg_i * 16),
17531755 vec_2_f64,
17541756 .{ .register = reg },
1757 .{},
17551758 );
17561759
17571760 try self.performReloc(skip_sse_reloc);
......@@ -2006,8 +2009,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
20062009 .load => try self.airLoad(inst),
20072010 .loop => try self.airLoop(inst),
20082011 .int_from_ptr => try self.airIntFromPtr(inst),
2009 .ret => try self.airRet(inst),
2010 .ret_safe => try self.airRet(inst), // TODO
2012 .ret => try self.airRet(inst, false),
2013 .ret_safe => try self.airRet(inst, true),
20112014 .ret_load => try self.airRetLoad(inst),
20122015 .store => try self.airStore(inst, false),
20132016 .store_safe => try self.airStore(inst, true),
......@@ -2209,8 +2212,15 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
22092212 0,
22102213 Type.usize,
22112214 .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
2215 .{},
2216 );
2217 try self.genSetMem(
2218 .{ .reg = ret_reg },
2219 8,
2220 Type.usize,
2221 .{ .immediate = tag_name.len },
2222 .{},
22122223 );
2213 try self.genSetMem(.{ .reg = ret_reg }, 8, Type.usize, .{ .immediate = tag_name.len });
22142224
22152225 exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
22162226 try self.performReloc(skip_reloc);
......@@ -2651,7 +2661,7 @@ pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {
26512661/// This can have a side effect of spilling instructions to the stack to free up a register.
26522662fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
26532663 const reg = try self.register_manager.allocReg(null, self.regClassForType(ty));
2654 try self.genSetReg(reg, ty, mcv);
2664 try self.genSetReg(reg, ty, mcv, .{});
26552665 return reg;
26562666}
26572667
......@@ -2666,7 +2676,7 @@ fn copyToRegisterWithInstTracking(
26662676 mcv: MCValue,
26672677) !MCValue {
26682678 const reg: Register = try self.register_manager.allocReg(reg_owner, self.regClassForType(ty));
2669 try self.genSetReg(reg, ty, mcv);
2679 try self.genSetReg(reg, ty, mcv, .{});
26702680 return MCValue{ .register = reg };
26712681}
26722682
......@@ -2919,7 +2929,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
29192929 math.divCeil(u32, src_storage_bits, 64) catch unreachable and
29202930 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
29212931 const dst_mcv = try self.allocRegOrMem(inst, true);
2922 try self.genCopy(min_ty, dst_mcv, src_mcv);
2932 try self.genCopy(min_ty, dst_mcv, src_mcv, .{});
29232933 break :dst dst_mcv;
29242934 };
29252935
......@@ -2954,7 +2964,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
29542964 if (high_bits > 0) {
29552965 try self.truncateRegister(src_ty, high_reg);
29562966 const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
2957 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg });
2967 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{});
29582968 }
29592969
29602970 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
......@@ -2973,6 +2983,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
29732983 .unsigned => .{ .immediate = 0 },
29742984 },
29752985 .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 },
2986 .{},
29762987 );
29772988
29782989 break :result dst_mcv;
......@@ -3006,7 +3017,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
30063017 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
30073018 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
30083019
3009 try self.genCopy(dst_ty, dst_mcv, src_mcv);
3020 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
30103021 break :dst dst_mcv;
30113022 } else try self.allocRegOrMem(inst, true);
30123023
......@@ -3082,11 +3093,15 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
30823093 // when truncating a `u16` to `u5`, for example, those top 3 bits in the result
30833094 // have to be removed. this only happens if the dst if not a power-of-two size.
30843095 if (dst_abi_size <= 8) {
3085 if (self.regExtraBits(dst_ty) > 0) try self.truncateRegister(dst_ty, dst_mcv.register.to64());
3096 if (self.regExtraBits(dst_ty) > 0) {
3097 try self.truncateRegister(dst_ty, dst_mcv.register.to64());
3098 }
30863099 } else if (dst_abi_size <= 16) {
30873100 const dst_info = dst_ty.intInfo(mod);
30883101 const high_ty = try mod.intType(dst_info.signedness, dst_info.bits - 64);
3089 if (self.regExtraBits(high_ty) > 0) try self.truncateRegister(high_ty, dst_mcv.register_pair[1].to64());
3102 if (self.regExtraBits(high_ty) > 0) {
3103 try self.truncateRegister(high_ty, dst_mcv.register_pair[1].to64());
3104 }
30903105 }
30913106
30923107 break :result dst_mcv;
......@@ -3116,7 +3131,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
31163131 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
31173132
31183133 const ptr_ty = self.typeOf(bin_op.lhs);
3119 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs });
3134 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});
31203135
31213136 const len_ty = self.typeOf(bin_op.rhs);
31223137 try self.genSetMem(
......@@ -3124,6 +3139,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
31243139 @intCast(ptr_ty.abiSize(mod)),
31253140 len_ty,
31263141 .{ .air_ref = bin_op.rhs },
3142 .{},
31273143 );
31283144
31293145 const result = MCValue{ .load_frame = .{ .index = frame_index } };
......@@ -3161,9 +3177,9 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
31613177
31623178 const hi_ty = try mod.intType(.unsigned, @intCast((dst_ty.bitSize(mod) - 1) % 64 + 1));
31633179 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
3164 try self.genSetReg(tmp_reg, hi_ty, hi_mcv);
3180 try self.genSetReg(tmp_reg, hi_ty, hi_mcv, .{});
31653181 try self.truncateRegister(dst_ty, tmp_reg);
3166 try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
3182 try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg }, .{});
31673183 }
31683184 }
31693185 }
......@@ -3476,7 +3492,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
34763492 if (reg_extra_bits > 0) {
34773493 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .{ .immediate = reg_extra_bits });
34783494 }
3479 try self.genSetReg(limit_reg, ty, dst_mcv);
3495 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
34803496 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
34813497 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
34823498 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
......@@ -3499,7 +3515,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
34993515 } else cc: {
35003516 try self.genSetReg(limit_reg, ty, .{
35013517 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(mod)),
3502 });
3518 }, .{});
35033519
35043520 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
35053521 if (reg_extra_bits > 0) {
......@@ -3559,7 +3575,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
35593575 if (reg_extra_bits > 0) {
35603576 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .{ .immediate = reg_extra_bits });
35613577 }
3562 try self.genSetReg(limit_reg, ty, dst_mcv);
3578 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
35633579 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
35643580 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
35653581 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
......@@ -3580,7 +3596,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
35803596 } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
35813597 break :cc .o;
35823598 } else cc: {
3583 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
3599 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 }, .{});
35843600 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
35853601 break :cc .c;
35863602 };
......@@ -3713,7 +3729,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
37133729
37143730 const reg_bits = self.regBitSize(ty);
37153731 const cc: Condition = if (ty.isSignedInt(mod)) cc: {
3716 try self.genSetReg(limit_reg, ty, lhs_mcv);
3732 try self.genSetReg(limit_reg, ty, lhs_mcv, .{});
37173733 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
37183734 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
37193735 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
......@@ -3723,7 +3739,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
37233739 } else cc: {
37243740 try self.genSetReg(limit_reg, ty, .{
37253741 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits),
3726 });
3742 }, .{});
37273743 break :cc .c;
37283744 };
37293745
......@@ -3779,12 +3795,14 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
37793795 @intCast(tuple_ty.structFieldOffset(1, mod)),
37803796 Type.u1,
37813797 .{ .eflags = cc },
3798 .{},
37823799 );
37833800 try self.genSetMem(
37843801 .{ .frame = frame_index },
37853802 @intCast(tuple_ty.structFieldOffset(0, mod)),
37863803 ty,
37873804 partial_mcv,
3805 .{},
37883806 );
37893807 break :result .{ .load_frame = .{ .index = frame_index } };
37903808 }
......@@ -3852,12 +3870,14 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
38523870 @intCast(tuple_ty.structFieldOffset(1, mod)),
38533871 tuple_ty.structFieldType(1, mod),
38543872 .{ .eflags = cc },
3873 .{},
38553874 );
38563875 try self.genSetMem(
38573876 .{ .frame = frame_index },
38583877 @intCast(tuple_ty.structFieldOffset(0, mod)),
38593878 tuple_ty.structFieldType(0, mod),
38603879 partial_mcv,
3880 .{},
38613881 );
38623882 break :result .{ .load_frame = .{ .index = frame_index } };
38633883 }
......@@ -3912,7 +3932,7 @@ fn genSetFrameTruncatedOverflowCompare(
39123932 src_mcv.address().offset(int_info.bits / 64 * 8).deref()
39133933 else
39143934 src_mcv;
3915 try self.genSetReg(scratch_reg, limb_ty, hi_limb_mcv);
3935 try self.genSetReg(scratch_reg, limb_ty, hi_limb_mcv, .{});
39163936 try self.truncateRegister(hi_ty, scratch_reg);
39173937 try self.genBinOpMir(.{ ._, .cmp }, limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
39183938
......@@ -3928,18 +3948,26 @@ fn genSetFrameTruncatedOverflowCompare(
39283948 }
39293949
39303950 const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, mod));
3931 if (hi_limb_off > 0) try self.genSetMem(.{ .frame = frame_index }, payload_off, rest_ty, src_mcv);
3951 if (hi_limb_off > 0) try self.genSetMem(
3952 .{ .frame = frame_index },
3953 payload_off,
3954 rest_ty,
3955 src_mcv,
3956 .{},
3957 );
39323958 try self.genSetMem(
39333959 .{ .frame = frame_index },
39343960 payload_off + hi_limb_off,
39353961 limb_ty,
39363962 .{ .register = scratch_reg },
3963 .{},
39373964 );
39383965 try self.genSetMem(
39393966 .{ .frame = frame_index },
39403967 @intCast(tuple_ty.structFieldOffset(1, mod)),
39413968 tuple_ty.structFieldType(1, mod),
39423969 if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
3970 .{},
39433971 );
39443972}
39453973
......@@ -3967,6 +3995,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
39673995 dst_mcv.address(),
39683996 .{ .immediate = 0 },
39693997 .{ .immediate = tuple_ty.abiSize(mod) },
3998 .{},
39703999 );
39714000 const lhs_mcv = try self.resolveInst(bin_op.lhs);
39724001 const rhs_mcv = try self.resolveInst(bin_op.rhs);
......@@ -4122,6 +4151,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
41224151 @intCast(tuple_ty.structFieldOffset(0, mod)),
41234152 tuple_ty.structFieldType(0, mod),
41244153 result,
4154 .{},
41254155 );
41264156 try self.asmMemoryImmediate(
41274157 .{ ._, .cmp },
......@@ -4133,6 +4163,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
41334163 @intCast(tuple_ty.structFieldOffset(1, mod)),
41344164 tuple_ty.structFieldType(1, mod),
41354165 .{ .eflags = .ne },
4166 .{},
41364167 );
41374168 try self.freeValue(overflow);
41384169 break :result dst_mcv;
......@@ -4244,12 +4275,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
42444275 @intCast(tuple_ty.structFieldOffset(0, mod)),
42454276 tuple_ty.structFieldType(0, mod),
42464277 .{ .register_pair = .{ .rax, .rdx } },
4278 .{},
42474279 );
42484280 try self.genSetMem(
42494281 .{ .frame = dst_mcv.load_frame.index },
42504282 @intCast(tuple_ty.structFieldOffset(1, mod)),
42514283 tuple_ty.structFieldType(1, mod),
42524284 .{ .register = tmp_regs[1] },
4285 .{},
42534286 );
42544287 break :result dst_mcv;
42554288 },
......@@ -4297,12 +4330,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
42974330 @intCast(tuple_ty.structFieldOffset(0, mod)),
42984331 tuple_ty.structFieldType(0, mod),
42994332 partial_mcv,
4333 .{},
43004334 );
43014335 try self.genSetMem(
43024336 .{ .frame = frame_index },
43034337 @intCast(tuple_ty.structFieldOffset(1, mod)),
43044338 tuple_ty.structFieldType(1, mod),
43054339 .{ .immediate = 0 }, // cc being set is impossible
4340 .{},
43064341 );
43074342 } else try self.genSetFrameTruncatedOverflowCompare(
43084343 tuple_ty,
......@@ -4330,7 +4365,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
43304365 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
43314366 }
43324367
4333 try self.genSetReg(.rax, ty, lhs);
4368 try self.genSetReg(.rax, ty, lhs, .{});
43344369 switch (tag[1]) {
43354370 else => unreachable,
43364371 .mul => {},
......@@ -4459,6 +4494,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
44594494 tmp_reg,
44604495 tmp_ty,
44614496 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },
4497 .{},
44624498 );
44634499 try self.truncateRegister(lhs_ty, tmp_reg);
44644500 try self.genSetMem(
......@@ -4466,6 +4502,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
44664502 off,
44674503 tmp_ty,
44684504 .{ .register = tmp_reg },
4505 .{},
44694506 );
44704507 },
44714508 else => {},
......@@ -4726,7 +4763,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
47264763 try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) {
47274764 else => opt_mcv,
47284765 .register_overflow => |ro| .{ .register = ro.reg },
4729 });
4766 }, .{});
47304767 break :result pl_mcv;
47314768 };
47324769 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -4773,7 +4810,13 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
47734810
47744811 const pl_ty = dst_ty.childType(mod);
47754812 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(mod));
4776 try self.genSetMem(.{ .reg = dst_mcv.getReg().? }, pl_abi_size, Type.bool, .{ .immediate = 1 });
4813 try self.genSetMem(
4814 .{ .reg = dst_mcv.getReg().? },
4815 pl_abi_size,
4816 Type.bool,
4817 .{ .immediate = 1 },
4818 .{},
4819 );
47774820 break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
47784821 };
47794822 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -5039,7 +5082,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
50395082 defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);
50405083
50415084 const opt_mcv = try self.allocRegOrMem(inst, true);
5042 try self.genCopy(pl_ty, opt_mcv, pl_mcv);
5085 try self.genCopy(pl_ty, opt_mcv, pl_mcv, .{});
50435086
50445087 if (!same_repr) {
50455088 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(mod));
......@@ -5089,8 +5132,8 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
50895132 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
50905133 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
50915134 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
5092 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
5093 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
5135 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
5136 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});
50945137 break :result .{ .load_frame = .{ .index = frame_index } };
50955138 };
50965139 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -5111,9 +5154,9 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
51115154 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
51125155 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
51135156 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
5114 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
5157 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
51155158 const operand = try self.resolveInst(ty_op.operand);
5116 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
5159 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});
51175160 break :result .{ .load_frame = .{ .index = frame_index } };
51185161 };
51195162 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -5127,7 +5170,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
51275170
51285171 const dst_mcv = try self.allocRegOrMem(inst, true);
51295172 const dst_ty = self.typeOfIndex(inst);
5130 try self.genCopy(dst_ty, dst_mcv, src_mcv);
5173 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
51315174 break :result dst_mcv;
51325175 };
51335176 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -5147,7 +5190,7 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
51475190 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
51485191
51495192 const dst_mcv = try self.allocRegOrMem(inst, true);
5150 try self.genCopy(Type.usize, dst_mcv, len_mcv);
5193 try self.genCopy(Type.usize, dst_mcv, len_mcv, .{});
51515194 break :result dst_mcv;
51525195 },
51535196 else => return self.fail("TODO implement slice_len for {}", .{src_mcv}),
......@@ -5211,7 +5254,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
52115254 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
52125255 // and set the register directly to the scaled offset as an immediate.
52135256 const reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5214 try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size });
5257 try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size }, .{});
52155258 break :blk reg;
52165259 },
52175260 else => {
......@@ -5251,7 +5294,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
52515294 defer self.register_manager.unlockReg(offset_reg_lock);
52525295
52535296 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5254 try self.genSetReg(addr_reg, Type.usize, slice_mcv);
5297 try self.genSetReg(addr_reg, Type.usize, slice_mcv, .{});
52555298 // TODO we could allocate register here, but need to expect addr register and potentially
52565299 // offset register.
52575300 try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
......@@ -5353,7 +5396,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
53535396 switch (array_mcv) {
53545397 .register => {
53555398 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));
5356 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv);
5399 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
53575400 try self.asmRegisterMemory(
53585401 .{ ._, .lea },
53595402 addr_reg,
......@@ -5373,7 +5416,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
53735416 .load_direct,
53745417 .load_got,
53755418 .load_tlv,
5376 => try self.genSetReg(addr_reg, Type.usize, array_mcv.address()),
5419 => try self.genSetReg(addr_reg, Type.usize, array_mcv.address(), .{}),
53775420 .lea_symbol, .lea_direct, .lea_tlv => unreachable,
53785421 else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{
53795422 @tagName(array_mcv), array_ty.fmt(mod),
......@@ -5393,7 +5436,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
53935436 .{ .register = addr_reg },
53945437 .{ .register = offset_reg },
53955438 );
5396 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
5439 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});
53975440 break :result dst_mcv;
53985441 };
53995442 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -5533,7 +5576,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
55335576 } else ptr;
55345577
55355578 const ptr_tag_ty = try mod.adjustPtrTypeChild(ptr_union_ty, tag_ty);
5536 try self.store(ptr_tag_ty, adjusted_ptr, tag);
5579 try self.store(ptr_tag_ty, adjusted_ptr, tag, .{});
55375580
55385581 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
55395582}
......@@ -5743,7 +5786,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
57435786 registerAlias(dst_reg, cmov_abi_size),
57445787 );
57455788
5746 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
5789 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 }, .{});
57475790 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
57485791 }
57495792 break :result dst_mcv;
......@@ -5798,7 +5841,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
57985841 const tmp_mcv = tmp: {
57995842 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
58005843 break :tmp src_mcv;
5801 try self.genSetReg(dst_reg, wide_ty, src_mcv);
5844 try self.genSetReg(dst_reg, wide_ty, src_mcv, .{});
58025845 break :tmp dst_mcv;
58035846 };
58045847 try self.genBinOpMir(
......@@ -5826,7 +5869,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
58265869 else
58275870 .{ .register = mat_src_mcv.register_pair[1] };
58285871 const masked_mcv = if (src_bits < 128) masked: {
5829 try self.genCopy(Type.u64, dst_mcv, hi_mat_src_mcv);
5872 try self.genCopy(Type.u64, dst_mcv, hi_mat_src_mcv, .{});
58305873 try self.genBinOpMir(
58315874 .{ ._, .@"or" },
58325875 Type.u64,
......@@ -5941,7 +5984,7 @@ fn genPopCount(
59415984 if (src_abi_size > 1) src_ty else Type.u32,
59425985 .{ .register = dst_reg },
59435986 if (src_abi_size > 1) src_mcv else src: {
5944 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv);
5987 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
59455988 try self.truncateRegister(try src_ty.toUnsigned(mod), dst_reg);
59465989 break :src .{ .register = dst_reg };
59475990 },
......@@ -5964,7 +6007,7 @@ fn genPopCount(
59646007 else
59656008 undefined;
59666009
5967 if (!dst_contains_src) try self.genSetReg(dst, src_ty, src_mcv);
6010 if (!dst_contains_src) try self.genSetReg(dst, src_ty, src_mcv, .{});
59686011 // dst = operand
59696012 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
59706013 // tmp = operand
......@@ -6102,7 +6145,7 @@ fn genByteSwap(
61026145 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
61036146 defer self.register_manager.unlockReg(dst_lock);
61046147
6105 try self.genSetReg(dst_reg, src_ty, src_mcv);
6148 try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
61066149 switch (abi_size) {
61076150 else => unreachable,
61086151 2 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
......@@ -6275,7 +6318,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
62756318 if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
62766319 try self.register_manager.getReg(.st0, inst);
62776320
6278 try self.genCopy(ty, dst_mcv, src_mcv);
6321 try self.genCopy(ty, dst_mcv, src_mcv, .{});
62796322 switch (tag) {
62806323 .neg => try self.asmOpOnly(.{ .f_, .chs }),
62816324 .abs => try self.asmOpOnly(.{ .f_, .abs }),
......@@ -6492,7 +6535,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
64926535 const mod = self.bin_file.comp.module.?;
64936536 const mir_tag = self.getRoundTag(ty) orelse {
64946537 const result = try self.genRoundLibcall(ty, src_mcv, mode);
6495 return self.genSetReg(dst_reg, ty, result);
6538 return self.genSetReg(dst_reg, ty, result, .{});
64966539 };
64976540 const abi_size: u32 = @intCast(ty.abiSize(mod));
64986541 const dst_alias = registerAlias(dst_reg, abi_size);
......@@ -6583,7 +6626,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
65836626 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
65846627 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
65856628
6586 try self.genCopy(ty, dst_mcv, src_mcv);
6629 try self.genCopy(ty, dst_mcv, src_mcv, .{});
65876630 break :dst dst_mcv;
65886631 };
65896632 const dst_regs = dst_mcv.register_pair;
......@@ -6940,9 +6983,9 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
69406983 defer self.register_manager.unlockReg(tmp_lock);
69416984
69426985 const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
6943 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
6986 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
69446987 try self.truncateRegister(val_ty, tmp_reg);
6945 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
6988 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
69466989 }
69476990 }
69486991 return;
......@@ -7011,7 +7054,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
70117054 }
70127055
70137056 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
7014 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg });
7057 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});
70157058}
70167059
70177060fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
......@@ -7036,7 +7079,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
70367079 .lea_got,
70377080 .lea_tlv,
70387081 .lea_frame,
7039 => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
7082 => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref(), .{}),
70407083 .memory,
70417084 .indirect,
70427085 .load_symbol,
......@@ -7049,7 +7092,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
70497092 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
70507093 defer self.register_manager.unlockReg(addr_lock);
70517094
7052 try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
7095 try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});
70537096 },
70547097 .air_ref => |ptr_ref| try self.load(dst_mcv, ptr_ty, try self.resolveInst(ptr_ref)),
70557098 }
......@@ -7105,6 +7148,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
71057148 if (elem_size <= 8) elem_ty else Type.usize,
71067149 high_mcv,
71077150 .{ .register = high_reg },
7151 .{},
71087152 );
71097153 }
71107154 break :result dst_mcv;
......@@ -7167,7 +7211,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
71677211 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
71687212 defer self.register_manager.unlockReg(tmp_lock);
71697213
7170 try self.genSetReg(tmp_reg, limb_ty, src_mcv);
7214 try self.genSetReg(tmp_reg, limb_ty, src_mcv, .{});
71717215 switch (limb_i) {
71727216 0 => try self.genShiftBinOpMir(
71737217 .{ ._l, .sh },
......@@ -7198,7 +7242,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
71987242 try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
71997243 0 => src_mcv,
72007244 else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),
7201 });
7245 }, .{});
72027246 try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
72037247 try self.asmMemoryRegister(
72047248 .{ ._, .@"or" },
......@@ -7209,7 +7253,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
72097253 }
72107254}
72117255
7212fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
7256fn store(
7257 self: *Self,
7258 ptr_ty: Type,
7259 ptr_mcv: MCValue,
7260 src_mcv: MCValue,
7261 opts: CopyOptions,
7262) InnerError!void {
72137263 const mod = self.bin_file.comp.module.?;
72147264 const src_ty = ptr_ty.childType(mod);
72157265 if (!src_ty.hasRuntimeBitsIgnoreComptime(mod)) return;
......@@ -7231,7 +7281,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr
72317281 .lea_got,
72327282 .lea_tlv,
72337283 .lea_frame,
7234 => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
7284 => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv, opts),
72357285 .memory,
72367286 .indirect,
72377287 .load_symbol,
......@@ -7244,9 +7294,9 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr
72447294 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
72457295 defer self.register_manager.unlockReg(addr_lock);
72467296
7247 try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
7297 try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv, opts);
72487298 },
7249 .air_ref => |ptr_ref| try self.store(ptr_ty, try self.resolveInst(ptr_ref), src_mcv),
7299 .air_ref => |ptr_ref| try self.store(ptr_ty, try self.resolveInst(ptr_ref), src_mcv, opts),
72507300 }
72517301}
72527302
......@@ -7269,7 +7319,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
72697319 if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) {
72707320 try self.packedStore(ptr_ty, ptr_mcv, src_mcv);
72717321 } else {
7272 try self.store(ptr_ty, ptr_mcv, src_mcv);
7322 try self.store(ptr_ty, ptr_mcv, src_mcv, .{ .safety = safety });
72737323 }
72747324 }
72757325 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -7389,7 +7439,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
73897439 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
73907440 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
73917441
7392 try self.genCopy(container_ty, .{ .register_pair = dst_regs }, src_mcv);
7442 try self.genCopy(container_ty, .{ .register_pair = dst_regs }, src_mcv, .{});
73937443 break :dst dst_regs;
73947444 };
73957445 const dst_mcv = MCValue{ .register_pair = dst_regs };
......@@ -7493,7 +7543,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
74937543 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
74947544 defer self.register_manager.unlockReg(dst_lock);
74957545
7496 try self.genCopy(int_ty, dst_mcv, off_mcv);
7546 try self.genCopy(int_ty, dst_mcv, off_mcv, .{});
74977547 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(int_ty, dst_reg);
74987548 break :result if (field_is_gp)
74997549 dst_mcv
......@@ -7507,7 +7557,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
75077557 off_mcv
75087558 else dst: {
75097559 const dst_mcv = try self.allocRegOrMem(inst, true);
7510 try self.genCopy(field_ty, dst_mcv, off_mcv);
7560 try self.genCopy(field_ty, dst_mcv, off_mcv, .{});
75117561 break :dst dst_mcv;
75127562 };
75137563 if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
......@@ -7517,9 +7567,9 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
75177567
75187568 const hi_mcv =
75197569 dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
7520 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
7570 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
75217571 try self.truncateRegister(field_ty, tmp_reg);
7522 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
7572 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
75237573 }
75247574 break :result dst_mcv;
75257575 }
......@@ -7650,7 +7700,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
76507700 if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) break :dst src_mcv;
76517701
76527702 const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, maybe_inst, true);
7653 try self.genCopy(src_ty, dst_mcv, src_mcv);
7703 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
76547704 break :dst dst_mcv;
76557705 };
76567706 const dst_lock = switch (dst_mcv) {
......@@ -7697,9 +7747,9 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
76977747 defer self.register_manager.unlockReg(tmp_lock);
76987748
76997749 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
7700 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
7750 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
77017751 try self.truncateRegister(src_ty, tmp_reg);
7702 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
7752 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
77037753 }
77047754 }
77057755 },
......@@ -7736,7 +7786,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
77367786 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
77377787 defer self.register_manager.unlockReg(addr_reg_lock);
77387788
7739 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
7789 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});
77407790 try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
77417791 .size = Memory.Size.fromSize(abi_size),
77427792 } } });
......@@ -7767,7 +7817,7 @@ fn genShiftBinOpMir(
77677817 else => {},
77687818 }
77697819 self.register_manager.getRegAssumeFree(.rcx, null);
7770 try self.genSetReg(.cl, Type.u8, shift_mcv);
7820 try self.genSetReg(.cl, Type.u8, shift_mcv, .{});
77717821 break :rhs .{ .register = .rcx };
77727822 };
77737823
......@@ -8161,7 +8211,7 @@ fn genShiftBinOp(
81618211 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
81628212 }
81638213 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
8164 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
8214 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{});
81658215 break :dst dst_mcv;
81668216 };
81678217
......@@ -8290,6 +8340,7 @@ fn genMulDivBinOp(
82908340 dst_mcv.address(),
82918341 .{ .immediate = 0 },
82928342 .{ .immediate = src_abi_size },
8343 .{},
82938344 );
82948345
82958346 const temp_regs = try self.register_manager.allocRegs(
......@@ -8891,7 +8942,7 @@ fn genBinOp(
88918942 if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
88928943 copied_to_dst = false
88938944 else
8894 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
8945 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{});
88958946 rhs_mcv = try self.resolveInst(ordered_air[1]);
88968947 break :dst dst_mcv;
88978948 };
......@@ -8906,7 +8957,7 @@ fn genBinOp(
89068957 const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg|
89078958 if (self.hasFeature(.avx) and unmat_src_mcv.isRegister() and maybe_inst != null and
89088959 self.liveness.operandDies(maybe_inst.?, if (flipped) 0 else 1)) unmat_src_mcv else src: {
8909 try self.genSetReg(mask_reg, rhs_ty, unmat_src_mcv);
8960 try self.genSetReg(mask_reg, rhs_ty, unmat_src_mcv, .{});
89108961 break :src .{ .register = mask_reg };
89118962 }
89128963 else
......@@ -8981,7 +9032,7 @@ fn genBinOp(
89819032 defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock|
89829033 self.register_manager.unlockReg(lock);
89839034
8984 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv);
9035 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{});
89859036 break :dst dst_regs;
89869037 },
89879038 };
......@@ -9036,7 +9087,7 @@ fn genBinOp(
90369087 try self.asmCmovccRegisterRegister(cc, dst_regs[0], src_mcv.register_pair[0]);
90379088 try self.asmCmovccRegisterRegister(cc, dst_regs[1], src_mcv.register_pair[1]);
90389089 }
9039 try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs });
9090 try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs }, .{});
90409091 } else {
90419092 const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) {
90429093 .immediate,
......@@ -9146,7 +9197,7 @@ fn genBinOp(
91469197 },
91479198 ),
91489199 }
9149 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
9200 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg }, .{});
91509201 }
91519202 },
91529203
......@@ -10519,7 +10570,7 @@ fn genBinOpMir(
1051910570 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
1052010571 errdefer self.register_manager.unlockReg(dst_addr_lock);
1052110572
10522 try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address());
10573 try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address(), .{});
1052310574 break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock };
1052410575 },
1052510576 .load_frame => null,
......@@ -10566,7 +10617,7 @@ fn genBinOpMir(
1056610617 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
1056710618 errdefer self.register_manager.unlockReg(src_addr_lock);
1056810619
10569 try self.genSetReg(src_addr_reg, Type.usize, resolved_src_mcv.address());
10620 try self.genSetReg(src_addr_reg, Type.usize, resolved_src_mcv.address(), .{});
1057010621 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
1057110622 },
1057210623 };
......@@ -10867,7 +10918,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
1086710918 defer self.register_manager.unlockReg(tmp_lock);
1086810919
1086910920 try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv);
10870 try self.genCopy(dst_ty, dst_mcv, tmp_mcv);
10921 try self.genCopy(dst_ty, dst_mcv, tmp_mcv, .{});
1087110922 },
1087210923 }
1087310924}
......@@ -10890,7 +10941,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1089010941 .indirect => |reg_off| dst: {
1089110942 self.register_manager.getRegAssumeFree(reg_off.reg, inst);
1089210943 const dst_mcv = try self.allocRegOrMem(inst, false);
10893 try self.genCopy(arg_ty, dst_mcv, src_mcv);
10944 try self.genCopy(arg_ty, dst_mcv, src_mcv, .{});
1089410945 break :dst dst_mcv;
1089510946 },
1089610947 else => return self.fail("TODO implement arg for {}", .{src_mcv}),
......@@ -10998,13 +11049,13 @@ fn airBreakpoint(self: *Self) !void {
1099811049
1099911050fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
1100011051 const dst_mcv = try self.allocRegOrMem(inst, true);
11001 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
11052 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
1100211053 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1100311054}
1100411055
1100511056fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
1100611057 const dst_mcv = try self.allocRegOrMem(inst, true);
11007 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
11058 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
1100811059 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1100911060}
1101011061
......@@ -11143,12 +11194,12 @@ fn genCall(self: *Self, info: union(enum) {
1114311194 },
1114411195 .indirect => |reg_off| {
1114511196 frame_index.* = try self.allocFrameIndex(FrameAlloc.initType(arg_ty, mod));
11146 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);
11197 try self.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, .{});
1114711198 try self.register_manager.getReg(reg_off.reg, null);
1114811199 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
1114911200 },
1115011201 .load_frame => {
11151 try self.genCopy(arg_ty, dst_arg, src_arg);
11202 try self.genCopy(arg_ty, dst_arg, src_arg, .{});
1115211203 try self.freeValue(src_arg);
1115311204 },
1115411205 else => unreachable,
......@@ -11162,7 +11213,7 @@ fn genCall(self: *Self, info: union(enum) {
1116211213 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, mod));
1116311214 try self.genSetReg(reg_off.reg, Type.usize, .{
1116411215 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
11165 });
11216 }, .{});
1116611217 call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
1116711218 try reg_locks.append(self.register_manager.lockReg(reg_off.reg));
1116811219 },
......@@ -11177,20 +11228,21 @@ fn genCall(self: *Self, info: union(enum) {
1117711228 registerAlias(dst_reg, @intCast(arg_ty.abiSize(mod))),
1117811229 arg_ty,
1117911230 src_arg,
11231 .{},
1118011232 ),
1118111233 .C, .SysV, .Win64 => {
1118211234 const promoted_ty = self.promoteInt(arg_ty);
1118311235 const promoted_abi_size: u32 = @intCast(promoted_ty.abiSize(mod));
1118411236 const dst_alias = registerAlias(dst_reg, promoted_abi_size);
11185 try self.genSetReg(dst_alias, promoted_ty, src_arg);
11237 try self.genSetReg(dst_alias, promoted_ty, src_arg, .{});
1118611238 if (promoted_ty.toIntern() != arg_ty.toIntern())
1118711239 try self.truncateRegister(arg_ty, dst_alias);
1118811240 },
1118911241 },
11190 .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg),
11242 .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, .{}),
1119111243 .indirect => |reg_off| try self.genSetReg(reg_off.reg, Type.usize, .{
1119211244 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
11193 }),
11245 }, .{}),
1119411246 else => unreachable,
1119511247 };
1119611248
......@@ -11231,6 +11283,7 @@ fn genCall(self: *Self, info: union(enum) {
1123111283 callee_reg,
1123211284 Type.usize,
1123311285 .{ .load_symbol = .{ .sym = sym.esym_index } },
11286 .{},
1123411287 );
1123511288 try self.asmRegister(.{ ._, .call }, callee_reg);
1123611289 } else try self.asmMemory(.{ ._, .call }, .{
......@@ -11243,12 +11296,17 @@ fn genCall(self: *Self, info: union(enum) {
1124311296 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
1124411297 const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
1124511298 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
11246 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
11299 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index }, .{});
1124711300 try self.asmRegister(.{ ._, .call }, .rax);
1124811301 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
1124911302 const sym_index = try macho_file.getZigObject().?.getOrCreateMetadataForDecl(macho_file, func.owner_decl);
1125011303 const sym = macho_file.getSymbol(sym_index);
11251 try self.genSetReg(.rax, Type.usize, .{ .load_symbol = .{ .sym = sym.nlist_idx } });
11304 try self.genSetReg(
11305 .rax,
11306 Type.usize,
11307 .{ .load_symbol = .{ .sym = sym.nlist_idx } },
11308 .{},
11309 );
1125211310 try self.asmRegister(.{ ._, .call }, .rax);
1125311311 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {
1125411312 const atom_index = try p9.seeDecl(func.owner_decl);
......@@ -11273,7 +11331,7 @@ fn genCall(self: *Self, info: union(enum) {
1127311331 }
1127411332 } else {
1127511333 assert(self.typeOf(callee).zigTypeTag(mod) == .Pointer);
11276 try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee });
11334 try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }, .{});
1127711335 try self.asmRegister(.{ ._, .call }, .rax);
1127811336 },
1127911337 .lib => |lib| try self.genExternSymbolRef(.call, lib.lib, lib.callee),
......@@ -11281,7 +11339,7 @@ fn genCall(self: *Self, info: union(enum) {
1128111339 return call_info.return_value.short;
1128211340}
1128311341
11284fn airRet(self: *Self, inst: Air.Inst.Index) !void {
11342fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1128511343 const mod = self.bin_file.comp.module.?;
1128611344 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1128711345
......@@ -11290,14 +11348,20 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
1129011348 .none => {},
1129111349 .register,
1129211350 .register_pair,
11293 => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }),
11351 => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }),
1129411352 .indirect => |reg_off| {
1129511353 try self.register_manager.getReg(reg_off.reg, null);
1129611354 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
1129711355 defer self.register_manager.unlockReg(lock);
1129811356
11299 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
11300 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, .{ .air_ref = un_op });
11357 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long, .{});
11358 try self.genSetMem(
11359 .{ .reg = reg_off.reg },
11360 reg_off.off,
11361 ret_ty,
11362 .{ .air_ref = un_op },
11363 .{ .safety = safety },
11364 );
1130111365 },
1130211366 else => unreachable,
1130311367 }
......@@ -11318,7 +11382,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
1131811382 switch (self.ret_mcv.short) {
1131911383 .none => {},
1132011384 .register, .register_pair => try self.load(self.ret_mcv.short, ptr_ty, ptr),
11321 .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr),
11385 .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr, .{}),
1132211386 else => unreachable,
1132311387 }
1132411388 self.ret_mcv.liveOut(self, inst);
......@@ -11423,7 +11487,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1142311487 const dst_mcv = if (unmat_dst_mcv.isRegister() or
1142411488 (abi_size <= 8 and unmat_dst_mcv.isMemory())) unmat_dst_mcv else dst: {
1142511489 const dst_mcv = try self.allocTempRegOrMem(ty, true);
11426 try self.genCopy(ty, dst_mcv, unmat_dst_mcv);
11490 try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{});
1142711491 break :dst dst_mcv;
1142811492 };
1142911493 const dst_lock =
......@@ -11494,6 +11558,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1149411558 dst_addr_reg,
1149511559 Type.usize,
1149611560 resolved_dst_mcv.address(),
11561 .{},
1149711562 );
1149811563 break :dst .{
1149911564 .addr_reg = dst_addr_reg,
......@@ -11553,6 +11618,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1155311618 src_addr_reg,
1155411619 Type.usize,
1155511620 resolved_src_mcv.address(),
11621 .{},
1155611622 );
1155711623 break :src .{
1155811624 .addr_reg = src_addr_reg,
......@@ -11594,7 +11660,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1159411660 .off = frame_addr.off + off,
1159511661 } },
1159611662 else => unreachable,
11597 });
11663 }, .{});
1159811664
1159911665 try self.genBinOpMir(
1160011666 .{ ._, .xor },
......@@ -12000,7 +12066,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
1200012066 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1200112067 defer self.register_manager.unlockReg(addr_reg_lock);
1200212068
12003 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
12069 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address(), .{});
1200412070 const some_abi_size: u32 = @intCast(some_info.ty.abiSize(mod));
1200512071 try self.asmMemoryImmediate(
1200612072 .{ ._, .cmp },
......@@ -12440,7 +12506,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
1244012506 try self.getValue(block_tracking.short, br.block_inst);
1244112507 break :dst block_tracking.short;
1244212508 };
12443 try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand));
12509 try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand), .{});
1244412510 break :result dst_mcv;
1244512511 };
1244612512
......@@ -12604,7 +12670,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1260412670 rc.isSet(RegisterManager.indexOfRegIntoTracked(input_mcv.getReg().?).?))
1260512671 break :arg input_mcv;
1260612672 const reg = try self.register_manager.allocReg(null, rc);
12607 try self.genSetReg(reg, ty, input_mcv);
12673 try self.genSetReg(reg, ty, input_mcv, .{});
1260812674 break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(mod))) };
1260912675 } else if (mem.eql(u8, constraint, "i") or mem.eql(u8, constraint, "n"))
1261012676 switch (input_mcv) {
......@@ -12621,16 +12687,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1262112687 .load_symbol, .load_direct, .load_got, .load_tlv => {},
1262212688 else => {
1262312689 const temp_mcv = try self.allocTempRegOrMem(ty, false);
12624 try self.genCopy(ty, temp_mcv, input_mcv);
12690 try self.genCopy(ty, temp_mcv, input_mcv, .{});
1262512691 break :arg temp_mcv;
1262612692 },
1262712693 }
1262812694 const addr_reg = self.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse {
1262912695 const temp_mcv = try self.allocTempRegOrMem(ty, false);
12630 try self.genCopy(ty, temp_mcv, input_mcv);
12696 try self.genCopy(ty, temp_mcv, input_mcv, .{});
1263112697 break :arg temp_mcv;
1263212698 };
12633 try self.genSetReg(addr_reg, Type.usize, input_mcv.address());
12699 try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});
1263412700 break :arg .{ .indirect = .{ .reg = addr_reg } };
1263512701 } else if (mem.eql(u8, constraint, "g") or
1263612702 mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or
......@@ -12643,7 +12709,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1264312709 else => {},
1264412710 }
1264512711 const temp_mcv = try self.allocTempRegOrMem(ty, true);
12646 try self.genCopy(ty, temp_mcv, input_mcv);
12712 try self.genCopy(ty, temp_mcv, input_mcv, .{});
1264712713 break :arg temp_mcv;
1264812714 } else if (mem.eql(u8, constraint, "X"))
1264912715 input_mcv
......@@ -12651,7 +12717,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1265112717 const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
1265212718 return self.fail("invalid register constraint: '{s}'", .{constraint});
1265312719 try self.register_manager.getReg(reg, null);
12654 try self.genSetReg(reg, ty, input_mcv);
12720 try self.genSetReg(reg, ty, input_mcv, .{});
1265512721 break :arg .{ .register = reg };
1265612722 } else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {
1265712723 const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;
......@@ -13105,7 +13171,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1310513171 if (output == .none) continue;
1310613172 if (arg_mcv != .register) continue;
1310713173 if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue;
13108 try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv);
13174 try self.store(self.typeOf(output), .{ .air_ref = output }, arg_mcv, .{});
1310913175 }
1311013176
1311113177 simple: {
......@@ -13477,7 +13543,11 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
1347713543 return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});
1347813544}
1347913545
13480fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
13546const CopyOptions = struct {
13547 safety: bool = false,
13548};
13549
13550fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
1348113551 const mod = self.bin_file.comp.module.?;
1348213552
1348313553 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
......@@ -13499,7 +13569,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
1349913569 .reserved_frame,
1350013570 .air_ref,
1350113571 => unreachable, // unmodifiable destination
13502 .register => |reg| try self.genSetReg(reg, ty, src_mcv),
13572 .register => |reg| try self.genSetReg(reg, ty, src_mcv, opts),
1350313573 .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) {
1350413574 .none,
1350513575 .unreach,
......@@ -13517,7 +13587,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
1351713587 .reg = try self.copyToTmpRegister(ty, src_mcv),
1351813588 .off = -dst_reg_off.off,
1351913589 } },
13520 }),
13590 }, opts),
1352113591 .register_pair => |dst_regs| {
1352213592 const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {
1352313593 .register_pair, .memory, .indirect, .load_frame => null,
......@@ -13527,10 +13597,15 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
1352713597 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
1352813598 errdefer self.register_manager.unlockReg(src_addr_lock);
1352913599
13530 try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address());
13600 try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address(), opts);
1353113601 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
1353213602 },
13533 .air_ref => |src_ref| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref)),
13603 .air_ref => |src_ref| return self.genCopy(
13604 ty,
13605 dst_mcv,
13606 try self.resolveInst(src_ref),
13607 opts,
13608 ),
1353413609 else => return self.fail("TODO implement genCopy for {s} of {}", .{
1353513610 @tagName(src_mcv), ty.fmt(mod),
1353613611 }),
......@@ -13547,15 +13622,21 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
1354713622 .off = part_disp,
1354813623 } },
1354913624 else => unreachable,
13550 });
13625 }, opts);
1355113626 part_disp += @intCast(dst_ty.abiSize(mod));
1355213627 }
1355313628 },
13554 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
13629 .indirect => |reg_off| try self.genSetMem(
13630 .{ .reg = reg_off.reg },
13631 reg_off.off,
13632 ty,
13633 src_mcv,
13634 opts,
13635 ),
1355513636 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => {
1355613637 switch (dst_mcv) {
1355713638 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr|
13558 return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv),
13639 return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts),
1355913640 .load_symbol, .load_direct, .load_got, .load_tlv => {},
1356013641 else => unreachable,
1356113642 }
......@@ -13564,18 +13645,25 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
1356413645 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1356513646 defer self.register_manager.unlockReg(addr_lock);
1356613647
13567 try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv);
13648 try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv, opts);
1356813649 },
1356913650 .load_frame => |frame_addr| try self.genSetMem(
1357013651 .{ .frame = frame_addr.index },
1357113652 frame_addr.off,
1357213653 ty,
1357313654 src_mcv,
13655 opts,
1357413656 ),
1357513657 }
1357613658}
1357713659
13578fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
13660fn genSetReg(
13661 self: *Self,
13662 dst_reg: Register,
13663 ty: Type,
13664 src_mcv: MCValue,
13665 opts: CopyOptions,
13666) InnerError!void {
1357913667 const mod = self.bin_file.comp.module.?;
1358013668 const abi_size: u32 = @intCast(ty.abiSize(mod));
1358113669 if (ty.bitSize(mod) > dst_reg.bitSize())
......@@ -13587,7 +13675,27 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1358713675 .register_overflow,
1358813676 .reserved_frame,
1358913677 => unreachable,
13590 .undef => {},
13678 .undef => if (opts.safety) switch (dst_reg.class()) {
13679 .general_purpose => switch (abi_size) {
13680 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), Immediate.u(0xAA)),
13681 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), Immediate.u(0xAAAA)),
13682 3...4 => try self.asmRegisterImmediate(
13683 .{ ._, .mov },
13684 dst_reg.to32(),
13685 Immediate.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))),
13686 ),
13687 5...8 => try self.asmRegisterImmediate(
13688 .{ ._, .mov },
13689 dst_reg.to64(),
13690 Immediate.u(0xAAAAAAAAAAAAAAAA),
13691 ),
13692 else => unreachable,
13693 },
13694 .segment, .x87, .mmx, .sse => try self.genSetReg(dst_reg, ty, try self.genTypedValue(.{
13695 .ty = ty,
13696 .val = try mod.undefValue(ty),
13697 }), opts),
13698 },
1359113699 .eflags => |cc| try self.asmSetccRegister(cc, dst_reg.to8()),
1359213700 .immediate => |imm| {
1359313701 if (imm == 0) {
......@@ -13672,6 +13780,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1367213780 dst_reg,
1367313781 ty,
1367413782 .{ .register = try self.copyToTmpRegister(ty, src_mcv) },
13783 opts,
1367513784 ),
1367613785 .x87, .mmx => unreachable,
1367713786 .sse => try self.asmRegisterRegister(
......@@ -13701,14 +13810,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1370113810 ),
1370213811 },
1370313812 },
13704 .register_pair => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }),
13813 .register_pair => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts),
1370513814 .register_offset,
1370613815 .indirect,
1370713816 .load_frame,
1370813817 .lea_frame,
1370913818 => try @as(MoveStrategy, switch (src_mcv) {
1371013819 .register_offset => |reg_off| switch (reg_off.off) {
13711 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
13820 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }, opts),
1371213821 else => .{ .move = .{ ._, .lea } },
1371313822 },
1371413823 .indirect => try self.moveStrategy(ty, dst_reg.class(), false),
......@@ -13852,11 +13961,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1385213961 });
1385313962 },
1385413963 .lea_tlv => unreachable, // TODO: remove this
13855 .air_ref => |src_ref| try self.genSetReg(dst_reg, ty, try self.resolveInst(src_ref)),
13964 .air_ref => |src_ref| try self.genSetReg(dst_reg, ty, try self.resolveInst(src_ref), opts),
1385613965 }
1385713966}
1385813967
13859fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCValue) InnerError!void {
13968fn genSetMem(
13969 self: *Self,
13970 base: Memory.Base,
13971 disp: i32,
13972 ty: Type,
13973 src_mcv: MCValue,
13974 opts: CopyOptions,
13975) InnerError!void {
1386013976 const mod = self.bin_file.comp.module.?;
1386113977 const abi_size: u32 = @intCast(ty.abiSize(mod));
1386213978 const dst_ptr_mcv: MCValue = switch (base) {
......@@ -13867,10 +13983,11 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1386713983 };
1386813984 switch (src_mcv) {
1386913985 .none, .unreach, .dead, .reserved_frame => unreachable,
13870 .undef => try self.genInlineMemset(
13986 .undef => if (opts.safety) try self.genInlineMemset(
1387113987 dst_ptr_mcv,
13872 .{ .immediate = 0xaa },
13988 src_mcv,
1387313989 .{ .immediate = abi_size },
13990 opts,
1387413991 ),
1387513992 .immediate => |imm| switch (abi_size) {
1387613993 1, 2, 4 => {
......@@ -13951,7 +14068,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1395114068 } } },
1395214069 src_alias,
1395314070 );
13954 try self.genSetMem(base, disp, ty, frame_mcv);
14071 try self.genSetMem(base, disp, ty, frame_mcv, opts);
1395514072 try self.freeValue(frame_mcv);
1395614073 } else try (try self.moveStrategy(ty, src_alias.class(), switch (base) {
1395714074 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
......@@ -13975,7 +14092,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1397514092 .register_pair => |src_regs| {
1397614093 var part_disp: i32 = disp;
1397714094 for (try self.splitType(ty), src_regs) |src_ty, src_reg| {
13978 try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
14095 try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts);
1397914096 part_disp += @intCast(src_ty.abiSize(mod));
1398014097 }
1398114098 },
......@@ -13986,23 +14103,26 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1398614103 disp + @as(i32, @intCast(ty.structFieldOffset(0, mod))),
1398714104 ty.structFieldType(0, mod),
1398814105 .{ .register = ro.reg },
14106 opts,
1398914107 );
1399014108 try self.genSetMem(
1399114109 base,
1399214110 disp + @as(i32, @intCast(ty.structFieldOffset(1, mod))),
1399314111 ty.structFieldType(1, mod),
1399414112 .{ .eflags = ro.eflags },
14113 opts,
1399514114 );
1399614115 },
1399714116 .Optional => {
1399814117 assert(!ty.optionalReprIsPayload(mod));
1399914118 const child_ty = ty.optionalChild(mod);
14000 try self.genSetMem(base, disp, child_ty, .{ .register = ro.reg });
14119 try self.genSetMem(base, disp, child_ty, .{ .register = ro.reg }, opts);
1400114120 try self.genSetMem(
1400214121 base,
1400314122 disp + @as(i32, @intCast(child_ty.abiSize(mod))),
1400414123 Type.bool,
1400514124 .{ .eflags = ro.eflags },
14125 opts,
1400614126 );
1400714127 },
1400814128 else => return self.fail("TODO implement genSetMem for {s} of {}", .{
......@@ -14029,27 +14149,37 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1402914149 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
1403014150 defer self.register_manager.unlockReg(src_lock);
1403114151
14032 try self.genSetMem(base, disp, ty, .{ .register = src_reg });
14152 try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts);
1403314153 },
14034 else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }),
14154 else => try self.genInlineMemcpy(
14155 dst_ptr_mcv,
14156 src_mcv.address(),
14157 .{ .immediate = abi_size },
14158 ),
1403514159 },
14036 .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref)),
14160 .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts),
1403714161 }
1403814162}
1403914163
1404014164fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
1404114165 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
14042 try self.genSetReg(.rsi, Type.usize, src_ptr);
14043 try self.genSetReg(.rdi, Type.usize, dst_ptr);
14044 try self.genSetReg(.rcx, Type.usize, len);
14166 try self.genSetReg(.rsi, Type.usize, src_ptr, .{});
14167 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
14168 try self.genSetReg(.rcx, Type.usize, len, .{});
1404514169 try self.asmOpOnly(.{ .@"rep _sb", .mov });
1404614170}
1404714171
14048fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
14172fn genInlineMemset(
14173 self: *Self,
14174 dst_ptr: MCValue,
14175 value: MCValue,
14176 len: MCValue,
14177 opts: CopyOptions,
14178) InnerError!void {
1404914179 try self.spillRegisters(&.{ .rdi, .al, .rcx });
14050 try self.genSetReg(.rdi, Type.usize, dst_ptr);
14051 try self.genSetReg(.al, Type.u8, value);
14052 try self.genSetReg(.rcx, Type.usize, len);
14180 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
14181 try self.genSetReg(.al, Type.u8, value, opts);
14182 try self.genSetReg(.rcx, Type.usize, len, .{});
1405314183 try self.asmOpOnly(.{ .@"rep _sb", .sto });
1405414184}
1405514185
......@@ -14113,10 +14243,10 @@ fn genLazySymbolRef(
1411314243 switch (tag) {
1411414244 .lea, .call => try self.genSetReg(reg, Type.usize, .{
1411514245 .load_symbol = .{ .sym = sym.esym_index },
14116 }),
14246 }, .{}),
1411714247 .mov => try self.genSetReg(reg, Type.usize, .{
1411814248 .load_symbol = .{ .sym = sym.esym_index },
14119 }),
14249 }, .{}),
1412014250 else => unreachable,
1412114251 }
1412214252 switch (tag) {
......@@ -14173,8 +14303,8 @@ fn genLazySymbolRef(
1417314303 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1417414304 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
1417514305 switch (tag) {
14176 .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }),
14177 .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }),
14306 .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }, .{}),
14307 .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }, .{}),
1417814308 else => unreachable,
1417914309 }
1418014310 switch (tag) {
......@@ -14187,8 +14317,13 @@ fn genLazySymbolRef(
1418714317 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1418814318 const sym = macho_file.getSymbol(sym_index);
1418914319 switch (tag) {
14190 .lea, .call => try self.genSetReg(reg, Type.usize, .{ .load_symbol = .{ .sym = sym.nlist_idx } }),
14191 .mov => try self.genSetReg(reg, Type.usize, .{ .load_symbol = .{ .sym = sym.nlist_idx } }),
14320 .lea, .call => try self.genSetReg(
14321 reg,
14322 Type.usize,
14323 .{ .load_symbol = .{ .sym = sym.nlist_idx } },
14324 .{},
14325 ),
14326 .mov => try self.genSetReg(reg, Type.usize, .{ .load_symbol = .{ .sym = sym.nlist_idx } }, .{}),
1419214327 else => unreachable,
1419314328 }
1419414329 switch (tag) {
......@@ -14210,7 +14345,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
1421014345
1421114346 const dst_mcv = try self.allocRegOrMem(inst, true);
1421214347 const dst_ty = self.typeOfIndex(inst);
14213 try self.genCopy(dst_ty, dst_mcv, src_mcv);
14348 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
1421414349 break :result dst_mcv;
1421514350 };
1421614351 return self.finishAir(inst, result, .{ un_op, .none, .none });
......@@ -14237,7 +14372,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1423714372 .lt => dst_ty,
1423814373 .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
1423914374 .gt => src_ty,
14240 }, dst_mcv, src_mcv);
14375 }, dst_mcv, src_mcv, .{});
1424114376 break :dst dst_mcv;
1424214377 };
1424314378
......@@ -14268,6 +14403,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1426814403 if (abi_size <= 8) dst_ty else Type.usize,
1426914404 high_mcv,
1427014405 .{ .register = high_reg },
14406 .{},
1427114407 );
1427214408 break :result dst_mcv;
1427314409 };
......@@ -14285,12 +14421,13 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
1428514421 const array_len = array_ty.arrayLen(mod);
1428614422
1428714423 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
14288 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
14424 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});
1428914425 try self.genSetMem(
1429014426 .{ .frame = frame_index },
1429114427 @intCast(ptr_ty.abiSize(mod)),
1429214428 Type.usize,
1429314429 .{ .immediate = array_len },
14430 .{},
1429414431 );
1429514432
1429614433 const result = MCValue{ .load_frame = .{ .index = frame_index } };
......@@ -14470,9 +14607,9 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1447014607 if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1447114608 defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock);
1447214609
14473 try self.genSetReg(.rax, Type.usize, exp_addr_mcv.deref());
14474 try self.genSetReg(.rdx, Type.usize, exp_addr_mcv.offset(8).deref());
14475 } else try self.genSetReg(.rax, val_ty, exp_mcv);
14610 try self.genSetReg(.rax, Type.usize, exp_addr_mcv.deref(), .{});
14611 try self.genSetReg(.rdx, Type.usize, exp_addr_mcv.offset(8).deref(), .{});
14612 } else try self.genSetReg(.rax, val_ty, exp_mcv, .{});
1447614613
1447714614 const new_mcv = try self.resolveInst(extra.new_value);
1447814615 const new_reg = if (val_abi_size > 8) new: {
......@@ -14484,8 +14621,8 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1448414621 if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1448514622 defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock);
1448614623
14487 try self.genSetReg(.rbx, Type.usize, new_addr_mcv.deref());
14488 try self.genSetReg(.rcx, Type.usize, new_addr_mcv.offset(8).deref());
14624 try self.genSetReg(.rbx, Type.usize, new_addr_mcv.deref(), .{});
14625 try self.genSetReg(.rcx, Type.usize, new_addr_mcv.offset(8).deref(), .{});
1448914626 break :new null;
1449014627 } else try self.copyToTmpRegister(val_ty, new_mcv);
1449114628 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
......@@ -14526,9 +14663,9 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1452614663 }
1452714664
1452814665 const dst_mcv = try self.allocRegOrMem(inst, false);
14529 try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax });
14530 try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx });
14531 try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne });
14666 try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax }, .{});
14667 try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
14668 try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
1453214669 break :result dst_mcv;
1453314670 };
1453414671 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
......@@ -14612,7 +14749,7 @@ fn atomicOp(
1461214749 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1461314750 defer self.register_manager.unlockReg(dst_lock);
1461414751
14615 try self.genSetReg(dst_reg, val_ty, val_mcv);
14752 try self.genSetReg(dst_reg, val_ty, val_mcv, .{});
1461614753 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
1461714754 try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv);
1461814755 }
......@@ -14645,7 +14782,7 @@ fn atomicOp(
1464514782 try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
1464614783 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
1464714784 if (!use_sse and rmw_op orelse .Xchg != .Xchg) {
14648 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
14785 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax }, .{});
1464914786 }
1465014787 if (rmw_op) |op| if (use_sse) {
1465114788 const mir_tag = @as(?Mir.Inst.FixedTag, switch (op) {
......@@ -14673,7 +14810,7 @@ fn atomicOp(
1467314810 }) orelse return self.fail("TODO implement atomicOp of {s} for {}", .{
1467414811 @tagName(op), val_ty.fmt(mod),
1467514812 });
14676 try self.genSetReg(sse_reg, val_ty, .{ .register = .rax });
14813 try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{});
1467714814 switch (mir_tag[0]) {
1467814815 .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory(
1467914816 mir_tag,
......@@ -14703,9 +14840,9 @@ fn atomicOp(
1470314840 ),
1470414841 else => unreachable,
1470514842 }
14706 try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg });
14843 try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg }, .{});
1470714844 } else switch (op) {
14708 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
14845 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv, .{}),
1470914846 .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
1471014847 .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
1471114848 .And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv),
......@@ -14995,7 +15132,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1499515132 };
1499615133 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
1499715134
14998 try self.genInlineMemset(ptr, src_val, len);
15135 try self.genInlineMemset(ptr, src_val, len, .{ .safety = safety });
1499915136 break :result;
1500015137 }
1500115138
......@@ -15017,11 +15154,11 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1501715154 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
1501815155 defer self.register_manager.unlockReg(len_lock);
1501915156
15020 try self.genSetReg(len_reg, Type.usize, len);
15157 try self.genSetReg(len_reg, Type.usize, len, .{});
1502115158 try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);
1502215159
1502315160 const skip_reloc = try self.asmJccReloc(.z, undefined);
15024 try self.store(slice_ptr_ty, ptr, src_val);
15161 try self.store(slice_ptr_ty, ptr, src_val, .{ .safety = safety });
1502515162
1502615163 const second_elem_ptr_reg =
1502715164 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
......@@ -15033,7 +15170,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1503315170 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
1503415171 .reg = try self.copyToTmpRegister(Type.usize, ptr),
1503515172 .off = elem_abi_size,
15036 } });
15173 } }, .{});
1503715174
1503815175 try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
1503915176 try self.asmRegisterRegisterImmediate(
......@@ -15052,7 +15189,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1505215189 const len = dst_ptr_ty.childType(mod).arrayLen(mod);
1505315190
1505415191 assert(len != 0); // prevented by Sema
15055 try self.store(elem_ptr_ty, dst_ptr, src_val);
15192 try self.store(elem_ptr_ty, dst_ptr, src_val, .{ .safety = safety });
1505615193
1505715194 const second_elem_ptr_reg =
1505815195 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
......@@ -15064,7 +15201,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1506415201 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
1506515202 .reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
1506615203 .off = elem_abi_size,
15067 } });
15204 } }, .{});
1506815205
1506915206 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
1507015207 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
......@@ -15158,10 +15295,10 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
1515815295 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
1515915296
1516015297 const dst_mcv = try self.allocRegOrMem(inst, false);
15161 try self.genSetReg(param_regs[0], Type.usize, dst_mcv.address());
15298 try self.genSetReg(param_regs[0], Type.usize, dst_mcv.address(), .{});
1516215299
1516315300 const operand = try self.resolveInst(un_op);
15164 try self.genSetReg(param_regs[1], enum_ty, operand);
15301 try self.genSetReg(param_regs[1], enum_ty, operand, .{});
1516515302
1516615303 try self.genLazySymbolRef(
1516715304 .call,
......@@ -15291,11 +15428,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1529115428 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
1529215429 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
1529315430
15294 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 });
15431 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 }, .{});
1529515432 try self.genSetReg(
1529615433 regs[1],
1529715434 vector_ty,
1529815435 .{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
15436 .{},
1529915437 );
1530015438 const src_mcv = try self.resolveInst(ty_op.operand);
1530115439 const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
......@@ -15360,7 +15498,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1536015498 try src_mcv.mem(self, self.memSize(scalar_ty)),
1536115499 ) else {
1536215500 if (mir_tag[0] == .vp_i128) break :avx2;
15363 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
15501 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
1536415502 try self.asmRegisterRegister(
1536515503 mir_tag,
1536615504 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
......@@ -15373,7 +15511,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1537315511 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1537415512 defer self.register_manager.unlockReg(dst_lock);
1537515513
15376 try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand });
15514 try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand }, .{});
1537715515 if (vector_len == 1) break :result .{ .register = dst_reg };
1537815516
1537915517 const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod)));
......@@ -15423,7 +15561,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1542315561 const src_mcv = try self.resolveInst(ty_op.operand);
1542415562 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
1542515563 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
15426 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
15564 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
1542715565 break :result .{ .register = dst_reg };
1542815566 },
1542915567 2...4 => {
......@@ -15506,7 +15644,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1550615644 const src_mcv = try self.resolveInst(ty_op.operand);
1550715645 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
1550815646 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
15509 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
15647 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
1551015648 break :result .{ .register = dst_reg };
1551115649 },
1551215650 2 => {
......@@ -15575,7 +15713,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
1557515713 const src_mcv = try self.resolveInst(ty_op.operand);
1557615714 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
1557715715 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
15578 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
15716 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
1557915717 break :result .{ .register = dst_reg };
1558015718 },
1558115719 2 => if (self.hasFeature(.avx)) {
......@@ -15699,6 +15837,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1569915837 .{ .lea_frame = .{ .index = frame_index } },
1570015838 .{ .immediate = 0 },
1570115839 .{ .immediate = result_ty.abiSize(mod) },
15840 .{},
1570215841 );
1570315842 for (elements, 0..) |elem, elem_i_usize| {
1570415843 const elem_i: u32 = @intCast(elem_i_usize);
......@@ -15783,7 +15922,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1578315922 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
1578415923 else => elem_mcv,
1578515924 };
15786 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
15925 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv, .{});
1578715926 }
1578815927 break :result .{ .load_frame = .{ .index = frame_index } };
1578915928 },
......@@ -15831,13 +15970,20 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1583115970 else => elem_mcv,
1583215971 };
1583315972 const elem_off: i32 = @intCast(elem_size * elem_i);
15834 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
15973 try self.genSetMem(
15974 .{ .frame = frame_index },
15975 elem_off,
15976 elem_ty,
15977 mat_elem_mcv,
15978 .{},
15979 );
1583515980 }
1583615981 if (result_ty.sentinel(mod)) |sentinel| try self.genSetMem(
1583715982 .{ .frame = frame_index },
1583815983 @intCast(elem_size * elements.len),
1583915984 elem_ty,
1584015985 try self.genTypedValue(.{ .ty = elem_ty, .val = sentinel }),
15986 .{},
1584115987 );
1584215988 break :result .{ .load_frame = .{ .index = frame_index } };
1584315989 }
......@@ -15872,7 +16018,7 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
1587216018 self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
1587316019
1587416020 const dst_mcv = try self.allocRegOrMem(inst, true);
15875 try self.genCopy(src_ty, dst_mcv, src_mcv);
16021 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
1587616022 break :result dst_mcv;
1587716023 }
1587816024
......@@ -15889,13 +16035,18 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
1588916035 @intCast(layout.payload_size)
1589016036 else
1589116037 0;
15892 try self.genCopy(tag_ty, dst_mcv.address().offset(tag_off).deref(), .{ .immediate = tag_int });
16038 try self.genCopy(
16039 tag_ty,
16040 dst_mcv.address().offset(tag_off).deref(),
16041 .{ .immediate = tag_int },
16042 .{},
16043 );
1589316044
1589416045 const pl_off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align))
1589516046 0
1589616047 else
1589716048 @intCast(layout.tag_size);
15898 try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv);
16049 try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{});
1589916050
1590016051 break :result dst_mcv;
1590116052 };
......@@ -16093,6 +16244,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
1609316244 field_off,
1609416245 Type.c_uint,
1609516246 .{ .immediate = info.gp_count * 8 },
16247 .{},
1609616248 );
1609716249 field_off += @intCast(Type.c_uint.abiSize(mod));
1609816250 // fp_offset: c_uint,
......@@ -16101,6 +16253,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
1610116253 field_off,
1610216254 Type.c_uint,
1610316255 .{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
16256 .{},
1610416257 );
1610516258 field_off += @intCast(Type.c_uint.abiSize(mod));
1610616259 // overflow_arg_area: *anyopaque,
......@@ -16109,6 +16262,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
1610916262 field_off,
1611016263 ptr_anyopaque_ty,
1611116264 .{ .lea_frame = info.overflow_arg_area },
16265 .{},
1611216266 );
1611316267 field_off += @intCast(ptr_anyopaque_ty.abiSize(mod));
1611416268 // reg_save_area: *anyopaque,
......@@ -16117,6 +16271,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
1611716271 field_off,
1611816272 ptr_anyopaque_ty,
1611916273 .{ .lea_frame = info.reg_save_area },
16274 .{},
1612016275 );
1612116276 field_off += @intCast(ptr_anyopaque_ty.abiSize(mod));
1612216277 break :result .{ .load_frame = .{ .index = dst_fi } };
......@@ -16171,13 +16326,13 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1617116326 .integer => {
1617216327 assert(classes.len == 1);
1617316328
16174 try self.genSetReg(offset_reg, Type.c_uint, gp_offset);
16329 try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});
1617516330 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
1617616331 abi.SysV.c_abi_int_param_regs.len * 8,
1617716332 ));
1617816333 const mem_reloc = try self.asmJccReloc(.ae, undefined);
1617916334
16180 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
16335 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
1618116336 if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
1618216337 .base = .{ .reg = addr_reg },
1618316338 .mod = .{ .rm = .{
......@@ -16192,11 +16347,11 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1619216347 .disp = 8,
1619316348 } },
1619416349 });
16195 try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg });
16350 try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg }, .{});
1619616351 const done_reloc = try self.asmJmpReloc(undefined);
1619716352
1619816353 try self.performReloc(mem_reloc);
16199 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
16354 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
1620016355 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
1620116356 .base = .{ .reg = addr_reg },
1620216357 .mod = .{ .rm = .{
......@@ -16208,23 +16363,24 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1620816363 ptr_anyopaque_ty,
1620916364 overflow_arg_area,
1621016365 .{ .register = offset_reg.to64() },
16366 .{},
1621116367 );
1621216368
1621316369 try self.performReloc(done_reloc);
1621416370 if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
1621516371 .indirect = .{ .reg = addr_reg },
16216 });
16372 }, .{});
1621716373 },
1621816374 .sse => {
1621916375 assert(classes.len == 1);
1622016376
16221 try self.genSetReg(offset_reg, Type.c_uint, fp_offset);
16377 try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});
1622216378 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u(
1622316379 abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
1622416380 ));
1622516381 const mem_reloc = try self.asmJccReloc(.ae, undefined);
1622616382
16227 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area);
16383 try self.genSetReg(addr_reg, ptr_anyopaque_ty, reg_save_area, .{});
1622816384 if (!unused) try self.asmRegisterMemory(.{ ._, .lea }, addr_reg, .{
1622916385 .base = .{ .reg = addr_reg },
1623016386 .mod = .{ .rm = .{
......@@ -16239,11 +16395,11 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1623916395 .disp = 16,
1624016396 } },
1624116397 });
16242 try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg });
16398 try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg }, .{});
1624316399 const done_reloc = try self.asmJmpReloc(undefined);
1624416400
1624516401 try self.performReloc(mem_reloc);
16246 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area);
16402 try self.genSetReg(addr_reg, ptr_anyopaque_ty, overflow_arg_area, .{});
1624716403 try self.asmRegisterMemory(.{ ._, .lea }, offset_reg.to64(), .{
1624816404 .base = .{ .reg = addr_reg },
1624916405 .mod = .{ .rm = .{
......@@ -16255,12 +16411,13 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1625516411 ptr_anyopaque_ty,
1625616412 overflow_arg_area,
1625716413 .{ .register = offset_reg.to64() },
16414 .{},
1625816415 );
1625916416
1626016417 try self.performReloc(done_reloc);
1626116418 if (!unused) try self.genCopy(promote_ty, promote_mcv, .{
1626216419 .indirect = .{ .reg = addr_reg },
16263 });
16420 }, .{});
1626416421 },
1626516422 .memory => {
1626616423 assert(classes.len == 1);
......@@ -16276,7 +16433,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
1627616433
1627716434 if (!promote_ty.isRuntimeFloat()) {
1627816435 const dst_mcv = try self.allocRegOrMem(inst, true);
16279 try self.genCopy(ty, dst_mcv, promote_mcv);
16436 try self.genCopy(ty, dst_mcv, promote_mcv, .{});
1628016437 break :result dst_mcv;
1628116438 }
1628216439
......@@ -16367,6 +16524,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
1636716524 0,
1636816525 Type.usize,
1636916526 .{ .lea_symbol = .{ .sym = tlv_sym } },
16527 .{},
1637016528 );
1637116529 break :init .{ .load_frame = .{ .index = frame_index } };
1637216530 },
test/behavior/undefined.zig-1
......@@ -99,7 +99,6 @@ test "reslice of undefined global var slice" {
9999}
100100
101101test "returned undef is 0xaa bytes when runtime safety is enabled" {
102 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
103102 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
104103 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
105104 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO