authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-12-21 15:18:50-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
logb9c44007762b9b302c263a89b7c2e8a8470b97ec
tree6cea52f080dd0a4df1bd7bc9ee007920b0965ecd
parentc4b93555b0d2c56cbbd85f77bb387ebb43d23369

x86_64: implement fallback for pcmpeqq

8 files changed, 720 insertions(+), 605 deletions(-)

src/Type.zig+14-22
......@@ -962,7 +962,6 @@ pub fn abiAlignmentInner(
962962) SemaError!AbiAlignmentInner {
963963 const pt = strat.pt(zcu, tid);
964964 const target = zcu.getTarget();
965 const use_llvm = zcu.comp.config.use_llvm;
966965 const ip = &zcu.intern_pool;
967966
968967 switch (ty.toIntern()) {
......@@ -970,7 +969,7 @@ pub fn abiAlignmentInner(
970969 else => switch (ip.indexToKey(ty.toIntern())) {
971970 .int_type => |int_type| {
972971 if (int_type.bits == 0) return .{ .scalar = .@"1" };
973 return .{ .scalar = intAbiAlignment(int_type.bits, target, use_llvm) };
972 return .{ .scalar = intAbiAlignment(int_type.bits, target) };
974973 },
975974 .ptr_type, .anyframe_type => {
976975 return .{ .scalar = ptrAbiAlignment(target) };
......@@ -1023,7 +1022,7 @@ pub fn abiAlignmentInner(
10231022 .error_set_type, .inferred_error_set_type => {
10241023 const bits = zcu.errorSetBits();
10251024 if (bits == 0) return .{ .scalar = .@"1" };
1026 return .{ .scalar = intAbiAlignment(bits, target, use_llvm) };
1025 return .{ .scalar = intAbiAlignment(bits, target) };
10271026 },
10281027
10291028 // represents machine code; not a pointer
......@@ -1036,7 +1035,7 @@ pub fn abiAlignmentInner(
10361035
10371036 .usize,
10381037 .isize,
1039 => return .{ .scalar = intAbiAlignment(target.ptrBitWidth(), target, use_llvm) },
1038 => return .{ .scalar = intAbiAlignment(target.ptrBitWidth(), target) },
10401039
10411040 .c_char => return .{ .scalar = cTypeAlign(target, .char) },
10421041 .c_short => return .{ .scalar = cTypeAlign(target, .short) },
......@@ -1067,7 +1066,7 @@ pub fn abiAlignmentInner(
10671066 .anyerror, .adhoc_inferred_error_set => {
10681067 const bits = zcu.errorSetBits();
10691068 if (bits == 0) return .{ .scalar = .@"1" };
1070 return .{ .scalar = intAbiAlignment(bits, target, use_llvm) };
1069 return .{ .scalar = intAbiAlignment(bits, target) };
10711070 },
10721071
10731072 .void,
......@@ -1291,7 +1290,6 @@ pub fn abiSizeInner(
12911290 tid: strat.Tid(),
12921291) SemaError!AbiSizeInner {
12931292 const target = zcu.getTarget();
1294 const use_llvm = zcu.comp.config.use_llvm;
12951293 const ip = &zcu.intern_pool;
12961294
12971295 switch (ty.toIntern()) {
......@@ -1300,7 +1298,7 @@ pub fn abiSizeInner(
13001298 else => switch (ip.indexToKey(ty.toIntern())) {
13011299 .int_type => |int_type| {
13021300 if (int_type.bits == 0) return .{ .scalar = 0 };
1303 return .{ .scalar = intAbiSize(int_type.bits, target, use_llvm) };
1301 return .{ .scalar = intAbiSize(int_type.bits, target) };
13041302 },
13051303 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
13061304 .slice => return .{ .scalar = @divExact(target.ptrBitWidth(), 8) * 2 },
......@@ -1362,7 +1360,7 @@ pub fn abiSizeInner(
13621360 .error_set_type, .inferred_error_set_type => {
13631361 const bits = zcu.errorSetBits();
13641362 if (bits == 0) return .{ .scalar = 0 };
1365 return .{ .scalar = intAbiSize(bits, target, use_llvm) };
1363 return .{ .scalar = intAbiSize(bits, target) };
13661364 },
13671365
13681366 .error_union_type => |error_union_type| {
......@@ -1455,7 +1453,7 @@ pub fn abiSizeInner(
14551453 .anyerror, .adhoc_inferred_error_set => {
14561454 const bits = zcu.errorSetBits();
14571455 if (bits == 0) return .{ .scalar = 0 };
1458 return .{ .scalar = intAbiSize(bits, target, use_llvm) };
1456 return .{ .scalar = intAbiSize(bits, target) };
14591457 },
14601458
14611459 .noreturn => unreachable,
......@@ -1609,11 +1607,11 @@ pub fn ptrAbiAlignment(target: Target) Alignment {
16091607 return Alignment.fromNonzeroByteUnits(@divExact(target.ptrBitWidth(), 8));
16101608}
16111609
1612pub fn intAbiSize(bits: u16, target: Target, use_llvm: bool) u64 {
1613 return intAbiAlignment(bits, target, use_llvm).forward(@as(u16, @intCast((@as(u17, bits) + 7) / 8)));
1610pub fn intAbiSize(bits: u16, target: Target) u64 {
1611 return intAbiAlignment(bits, target).forward(@as(u16, @intCast((@as(u17, bits) + 7) / 8)));
16141612}
16151613
1616pub fn intAbiAlignment(bits: u16, target: Target, use_llvm: bool) Alignment {
1614pub fn intAbiAlignment(bits: u16, target: Target) Alignment {
16171615 return switch (target.cpu.arch) {
16181616 .x86 => switch (bits) {
16191617 0 => .none,
......@@ -1632,19 +1630,16 @@ pub fn intAbiAlignment(bits: u16, target: Target, use_llvm: bool) Alignment {
16321630 9...16 => .@"2",
16331631 17...32 => .@"4",
16341632 33...64 => .@"8",
1635 else => switch (target_util.zigBackend(target, use_llvm)) {
1636 .stage2_x86_64 => .@"8",
1637 else => .@"16",
1638 },
1633 else => .@"16",
16391634 },
16401635 else => return Alignment.fromByteUnits(@min(
16411636 std.math.ceilPowerOfTwoPromote(u16, @as(u16, @intCast((@as(u17, bits) + 7) / 8))),
1642 maxIntAlignment(target, use_llvm),
1637 maxIntAlignment(target),
16431638 )),
16441639 };
16451640}
16461641
1647pub fn maxIntAlignment(target: std.Target, use_llvm: bool) u16 {
1642pub fn maxIntAlignment(target: std.Target) u16 {
16481643 return switch (target.cpu.arch) {
16491644 .avr => 1,
16501645 .msp430 => 2,
......@@ -1685,10 +1680,7 @@ pub fn maxIntAlignment(target: std.Target, use_llvm: bool) u16 {
16851680 else => 8,
16861681 },
16871682
1688 .x86_64 => switch (target_util.zigBackend(target, use_llvm)) {
1689 .stage2_x86_64 => 8,
1690 else => 16,
1691 },
1683 .x86_64 => 16,
16921684
16931685 // Even LLVMABIAlignmentOfType(i128) agrees on these targets.
16941686 .x86,
src/arch/x86_64/CodeGen.zig+628-548
......@@ -2097,11 +2097,11 @@ fn gen(self: *CodeGen) InnerError!void {
20972097 // The address where to store the return value for the caller is in a
20982098 // register which the callee is free to clobber. Therefore, we purposely
20992099 // spill it to stack immediately.
2100 const frame_index = try self.allocFrameIndex(.initSpill(Type.usize, zcu));
2100 const frame_index = try self.allocFrameIndex(.initSpill(.usize, zcu));
21012101 try self.genSetMem(
21022102 .{ .frame = frame_index },
21032103 0,
2104 Type.usize,
2104 .usize,
21052105 self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
21062106 .{},
21072107 );
......@@ -2122,13 +2122,7 @@ fn gen(self: *CodeGen) InnerError!void {
21222122 info.reg_save_area = .{ .index = reg_save_area_fi };
21232123
21242124 for (abi.SysV.c_abi_int_param_regs[info.gp_count..], info.gp_count..) |reg, reg_i|
2125 try self.genSetMem(
2126 .{ .frame = reg_save_area_fi },
2127 @intCast(reg_i * 8),
2128 Type.usize,
2129 .{ .register = reg },
2130 .{},
2131 );
2125 try self.genSetMem(.{ .frame = reg_save_area_fi }, @intCast(reg_i * 8), .usize, .{ .register = reg }, .{});
21322126
21332127 try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count));
21342128 const skip_sse_reloc = try self.asmJccReloc(.na, undefined);
......@@ -2556,6 +2550,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
25562550 .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } },
25572551 },
25582552 },
2553 .{
2554 .required_features = &.{.sse},
2555 .mir_tag = .{ ._ps, mir_tag },
2556 .patterns = &.{
2557 .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } },
2558 .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } },
2559 .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } },
2560 .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } },
2561 },
2562 },
25592563 .{
25602564 .required_features = &.{.mmx},
25612565 .mir_tag = .{ .p_, mir_tag },
......@@ -2615,6 +2619,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
26152619 .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .xmm_limb } },
26162620 },
26172621 },
2622 .{
2623 .required_features = &.{.sse},
2624 .loop = .bitwise,
2625 .mir_tag = .{ ._ps, mir_tag },
2626 .patterns = &.{
2627 .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } },
2628 .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } },
2629 .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .xmm_limb } },
2630 },
2631 },
26182632 .{
26192633 .required_features = &.{.mmx},
26202634 .loop = .bitwise,
......@@ -3070,11 +3084,58 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
30703084 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } },
30713085 },
30723086 },
3073 }, .{ .invert_result = switch (cmp_op) {
3074 .eq => false,
3075 .neq => true,
3076 else => unreachable,
3077 } }),
3087 .{
3088 .scalar = .{ .any_int = .byte },
3089 .clobbers = .{ .eflags = true },
3090 .loop = .elementwise,
3091 .mir_tag = .{ ._, .cmp },
3092 .patterns = &.{
3093 .{ .ops = &.{ .cc_mask_limb, .mem_limb, .gpr_limb } },
3094 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .mem_limb } },
3095 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .gpr_limb } },
3096 },
3097 },
3098 .{
3099 .scalar = .{ .any_int = .word },
3100 .clobbers = .{ .eflags = true },
3101 .loop = .elementwise,
3102 .mir_tag = .{ ._, .cmp },
3103 .patterns = &.{
3104 .{ .ops = &.{ .cc_mask_limb, .mem_limb, .gpr_limb } },
3105 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .mem_limb } },
3106 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .gpr_limb } },
3107 },
3108 },
3109 .{
3110 .scalar = .{ .any_int = .dword },
3111 .clobbers = .{ .eflags = true },
3112 .loop = .elementwise,
3113 .mir_tag = .{ ._, .cmp },
3114 .patterns = &.{
3115 .{ .ops = &.{ .cc_mask_limb, .mem_limb, .gpr_limb } },
3116 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .mem_limb } },
3117 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .gpr_limb } },
3118 },
3119 },
3120 .{
3121 .scalar = .{ .any_int = .qword },
3122 .clobbers = .{ .eflags = true },
3123 .loop = .elementwise,
3124 .mir_tag = .{ ._, .cmp },
3125 .patterns = &.{
3126 .{ .ops = &.{ .cc_mask_limb, .mem_limb, .gpr_limb } },
3127 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .mem_limb } },
3128 .{ .ops = &.{ .cc_mask_limb, .gpr_limb, .gpr_limb } },
3129 },
3130 },
3131 }, .{
3132 .cc = .e,
3133 .invert_result = switch (cmp_op) {
3134 .eq => false,
3135 .neq => true,
3136 else => unreachable,
3137 },
3138 }),
30783139 .gte => unreachable,
30793140 .gt => unreachable,
30803141 }
......@@ -3242,7 +3303,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
32423303 const opt_child_ty = opt_ty.optionalChild(zcu);
32433304 const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu));
32443305 try ops[0].toOffset(opt_child_abi_size, cg);
3245 var has_value = try cg.tempFromValue(Type.bool, .{ .immediate = 1 });
3306 var has_value = try cg.tempFromValue(.bool, .{ .immediate = 1 });
32463307 try ops[0].store(&has_value, cg);
32473308 try has_value.die(cg);
32483309 try ops[0].toOffset(-opt_child_abi_size, cg);
......@@ -3426,7 +3487,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
34263487 .array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else {
34273488 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
34283489 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
3429 var len = try cg.tempFromValue(Type.usize, .{
3490 var len = try cg.tempFromValue(.usize, .{
34303491 .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu),
34313492 });
34323493 try ops[0].toPair(&len, cg);
......@@ -3510,17 +3571,11 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
35103571 try self.genSetMem(
35113572 .{ .reg = ret_reg },
35123573 0,
3513 Type.usize,
3574 .usize,
35143575 .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
35153576 .{},
35163577 );
3517 try self.genSetMem(
3518 .{ .reg = ret_reg },
3519 8,
3520 Type.usize,
3521 .{ .immediate = tag_name_len },
3522 .{},
3523 );
3578 try self.genSetMem(.{ .reg = ret_reg }, 8, .usize, .{ .immediate = tag_name_len }, .{});
35243579
35253580 exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
35263581 self.performReloc(skip_reloc);
......@@ -4495,8 +4550,8 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
44954550 high_mcv.getReg().?
44964551 else
44974552 try self.copyToTmpRegister(switch (src_int_info.signedness) {
4498 .signed => Type.isize,
4499 .unsigned => Type.usize,
4553 .signed => .isize,
4554 .unsigned => .usize,
45004555 }, high_mcv);
45014556 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
45024557 defer self.register_manager.unlockReg(high_lock);
......@@ -4504,7 +4559,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
45044559 const high_bits = src_int_info.bits % 64;
45054560 if (high_bits > 0) {
45064561 try self.truncateRegister(src_ty, high_reg);
4507 const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
4562 const high_ty: Type = if (dst_int_info.bits >= 64) .usize else dst_ty;
45084563 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{});
45094564 }
45104565
......@@ -4513,13 +4568,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
45134568 switch (extend) {
45144569 .signed => extend: {
45154570 const extend_mcv = MCValue{ .register = high_reg };
4516 try self.genShiftBinOpMir(
4517 .{ ._r, .sa },
4518 Type.isize,
4519 extend_mcv,
4520 Type.u8,
4521 .{ .immediate = 63 },
4522 );
4571 try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, extend_mcv, .u8, .{ .immediate = 63 });
45234572 break :extend extend_mcv;
45244573 },
45254574 .unsigned => .{ .immediate = 0 },
......@@ -4621,7 +4670,7 @@ fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
46214670 const splat_mcv = try self.genTypedValue(.fromInterned(splat_val));
46224671 const splat_addr_mcv: MCValue = switch (splat_mcv) {
46234672 .memory, .indirect, .load_frame => splat_mcv.address(),
4624 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
4673 else => .{ .register = try self.copyToTmpRegister(.usize, splat_mcv.address()) },
46254674 };
46264675
46274676 const dst_reg = dst_mcv.getReg().?;
......@@ -4837,7 +4886,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
48374886 state: State,
48384887 reloc: Mir.Inst.Index,
48394888 } = if (signed and tag == .div_floor) state: {
4840 const frame_index = try self.allocFrameIndex(.initType(Type.usize, zcu));
4889 const frame_index = try self.allocFrameIndex(.initType(.usize, zcu));
48414890 try self.asmMemoryImmediate(
48424891 .{ ._, .mov },
48434892 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
......@@ -4852,7 +4901,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
48524901 const mat_lhs_mcv = switch (lhs_mcv) {
48534902 .load_symbol => mat_lhs_mcv: {
48544903 // TODO clean this up!
4855 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
4904 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
48564905 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
48574906 },
48584907 else => lhs_mcv,
......@@ -4876,7 +4925,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
48764925 const mat_rhs_mcv = switch (rhs_mcv) {
48774926 .load_symbol => mat_rhs_mcv: {
48784927 // TODO clean this up!
4879 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
4928 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
48804929 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
48814930 },
48824931 else => rhs_mcv,
......@@ -4975,7 +5024,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
49755024 const mat_rhs_mcv = switch (rhs_mcv) {
49765025 .load_symbol => mat_rhs_mcv: {
49775026 // TODO clean this up!
4978 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
5027 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
49795028 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
49805029 },
49815030 else => rhs_mcv,
......@@ -5062,22 +5111,10 @@ fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
50625111 const reg_extra_bits = self.regExtraBits(ty);
50635112 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
50645113 if (reg_extra_bits > 0) {
5065 try self.genShiftBinOpMir(
5066 .{ ._l, .sa },
5067 ty,
5068 dst_mcv,
5069 Type.u8,
5070 .{ .immediate = reg_extra_bits },
5071 );
5114 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
50725115 }
50735116 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
5074 try self.genShiftBinOpMir(
5075 .{ ._r, .sa },
5076 ty,
5077 limit_mcv,
5078 Type.u8,
5079 .{ .immediate = reg_bits - 1 },
5080 );
5117 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
50815118 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
50825119 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
50835120 });
......@@ -5087,13 +5124,7 @@ fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
50875124 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
50885125 defer self.register_manager.unlockReg(shifted_rhs_lock);
50895126
5090 try self.genShiftBinOpMir(
5091 .{ ._l, .sa },
5092 ty,
5093 shifted_rhs_mcv,
5094 Type.u8,
5095 .{ .immediate = reg_extra_bits },
5096 );
5127 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
50975128 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv);
50985129 } else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
50995130 break :cc .o;
......@@ -5117,13 +5148,8 @@ fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
51175148 registerAlias(limit_reg, cmov_abi_size),
51185149 );
51195150
5120 if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) try self.genShiftBinOpMir(
5121 .{ ._r, .sa },
5122 ty,
5123 dst_mcv,
5124 Type.u8,
5125 .{ .immediate = reg_extra_bits },
5126 );
5151 if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
5152 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
51275153
51285154 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
51295155}
......@@ -5163,22 +5189,10 @@ fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
51635189 const reg_extra_bits = self.regExtraBits(ty);
51645190 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
51655191 if (reg_extra_bits > 0) {
5166 try self.genShiftBinOpMir(
5167 .{ ._l, .sa },
5168 ty,
5169 dst_mcv,
5170 Type.u8,
5171 .{ .immediate = reg_extra_bits },
5172 );
5192 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
51735193 }
51745194 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
5175 try self.genShiftBinOpMir(
5176 .{ ._r, .sa },
5177 ty,
5178 limit_mcv,
5179 Type.u8,
5180 .{ .immediate = reg_bits - 1 },
5181 );
5195 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
51825196 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
51835197 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
51845198 });
......@@ -5188,13 +5202,7 @@ fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
51885202 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
51895203 defer self.register_manager.unlockReg(shifted_rhs_lock);
51905204
5191 try self.genShiftBinOpMir(
5192 .{ ._l, .sa },
5193 ty,
5194 shifted_rhs_mcv,
5195 Type.u8,
5196 .{ .immediate = reg_extra_bits },
5197 );
5205 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
51985206 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv);
51995207 } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
52005208 break :cc .o;
......@@ -5211,13 +5219,8 @@ fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
52115219 registerAlias(limit_reg, cmov_abi_size),
52125220 );
52135221
5214 if (reg_extra_bits > 0 and ty.isSignedInt(zcu)) try self.genShiftBinOpMir(
5215 .{ ._r, .sa },
5216 ty,
5217 dst_mcv,
5218 Type.u8,
5219 .{ .immediate = reg_extra_bits },
5220 );
5222 if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
5223 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
52215224
52225225 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
52235226}
......@@ -5230,14 +5233,14 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
52305233
52315234 const result = result: {
52325235 if (ty.toIntern() == .i128_type) {
5233 const ptr_c_int = try pt.singleMutPtrType(Type.c_int);
5234 const overflow = try self.allocTempRegOrMem(Type.c_int, false);
5236 const ptr_c_int = try pt.singleMutPtrType(.c_int);
5237 const overflow = try self.allocTempRegOrMem(.c_int, false);
52355238
52365239 const dst_mcv = try self.genCall(.{ .lib = .{
52375240 .return_type = .i128_type,
52385241 .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
52395242 .callee = "__muloti4",
5240 } }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{
5243 } }, &.{ .i128, .i128, ptr_c_int }, &.{
52415244 .{ .air_ref = bin_op.lhs },
52425245 .{ .air_ref = bin_op.rhs },
52435246 overflow.address(),
......@@ -5253,7 +5256,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
52535256 const mat_lhs_mcv = switch (lhs_mcv) {
52545257 .load_symbol => mat_lhs_mcv: {
52555258 // TODO clean this up!
5256 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
5259 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
52575260 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
52585261 },
52595262 else => lhs_mcv,
......@@ -5277,7 +5280,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
52775280 const mat_rhs_mcv = switch (rhs_mcv) {
52785281 .load_symbol => mat_rhs_mcv: {
52795282 // TODO clean this up!
5280 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
5283 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
52815284 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
52825285 },
52835286 else => rhs_mcv,
......@@ -5339,13 +5342,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
53395342 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
53405343 try self.genSetReg(limit_reg, ty, lhs_mcv, .{});
53415344 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
5342 try self.genShiftBinOpMir(
5343 .{ ._r, .sa },
5344 ty,
5345 limit_mcv,
5346 Type.u8,
5347 .{ .immediate = reg_bits - 1 },
5348 );
5345 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
53495346 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
53505347 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
53515348 });
......@@ -5410,7 +5407,7 @@ fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
54105407 try self.genSetMem(
54115408 .{ .frame = frame_index },
54125409 @intCast(tuple_ty.structFieldOffset(1, zcu)),
5413 Type.u1,
5410 .u1,
54145411 .{ .eflags = cc },
54155412 .{},
54165413 );
......@@ -5558,12 +5555,7 @@ fn genSetFrameTruncatedOverflowCompare(
55585555 const eq_reg = temp_regs[2];
55595556 if (overflow_cc) |_| {
55605557 try self.asmSetccRegister(.ne, eq_reg.to8());
5561 try self.genBinOpMir(
5562 .{ ._, .@"or" },
5563 Type.u8,
5564 .{ .register = overflow_reg },
5565 .{ .register = eq_reg },
5566 );
5558 try self.genBinOpMir(.{ ._, .@"or" }, .u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
55675559 }
55685560
55695561 const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
......@@ -5743,13 +5735,13 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
57435735 if (src_bits > 64 and src_bits <= 128 and
57445736 dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) {
57455737 .signed => {
5746 const ptr_c_int = try pt.singleMutPtrType(Type.c_int);
5747 const overflow = try self.allocTempRegOrMem(Type.c_int, false);
5738 const ptr_c_int = try pt.singleMutPtrType(.c_int);
5739 const overflow = try self.allocTempRegOrMem(.c_int, false);
57485740 const result = try self.genCall(.{ .lib = .{
57495741 .return_type = .i128_type,
57505742 .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
57515743 .callee = "__muloti4",
5752 } }, &.{ Type.i128, Type.i128, ptr_c_int }, &.{
5744 } }, &.{ .i128, .i128, ptr_c_int }, &.{
57535745 .{ .air_ref = bin_op.lhs },
57545746 .{ .air_ref = bin_op.rhs },
57555747 overflow.address(),
......@@ -5765,7 +5757,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
57655757 );
57665758 try self.asmMemoryImmediate(
57675759 .{ ._, .cmp },
5768 try overflow.mem(self, .{ .size = self.memSize(Type.c_int) }),
5760 try overflow.mem(self, .{ .size = self.memSize(.c_int) }),
57695761 .s(0),
57705762 );
57715763 try self.genSetMem(
......@@ -5794,7 +5786,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
57945786 const mat_lhs_mcv = switch (lhs_mcv) {
57955787 .load_symbol => mat_lhs_mcv: {
57965788 // TODO clean this up!
5797 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
5789 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
57985790 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
57995791 },
58005792 else => lhs_mcv,
......@@ -5807,7 +5799,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
58075799 const mat_rhs_mcv = switch (rhs_mcv) {
58085800 .load_symbol => mat_rhs_mcv: {
58095801 // TODO clean this up!
5810 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
5802 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
58115803 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
58125804 },
58135805 else => rhs_mcv,
......@@ -6100,7 +6092,7 @@ fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
61006092 defer self.register_manager.unlockReg(tmp_lock);
61016093
61026094 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu));
6103 const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
6095 const tmp_ty: Type = if (lhs_bits > 64) .usize else lhs_ty;
61046096 const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
61056097 try self.genSetReg(
61066098 tmp_reg,
......@@ -6283,13 +6275,12 @@ fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
62836275 const mask_mcv = try self.genTypedValue(.fromInterned(try pt.intern(.{ .aggregate = .{
62846276 .ty = mask_ty.toIntern(),
62856277 .storage = .{ .elems = &([1]InternPool.Index{
6286 (try rhs_ty.childType(zcu).maxIntScalar(pt, Type.u8)).toIntern(),
6278 (try rhs_ty.childType(zcu).maxIntScalar(pt, .u8)).toIntern(),
62876279 } ++ [1]InternPool.Index{
6288 (try pt.intValue(Type.u8, 0)).toIntern(),
6280 (try pt.intValue(.u8, 0)).toIntern(),
62896281 } ** 15) },
62906282 } })));
6291 const mask_addr_reg =
6292 try self.copyToTmpRegister(Type.usize, mask_mcv.address());
6283 const mask_addr_reg = try self.copyToTmpRegister(.usize, mask_mcv.address());
62936284 const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
62946285 defer self.register_manager.unlockReg(mask_addr_lock);
62956286
......@@ -6423,7 +6414,7 @@ fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
64236414 try self.genSetMem(
64246415 .{ .reg = dst_mcv.getReg().? },
64256416 pl_abi_size,
6426 Type.bool,
6417 .bool,
64276418 .{ .immediate = 1 },
64286419 .{},
64296420 );
......@@ -6462,9 +6453,9 @@ fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
64626453 .{ ._r, .sh },
64636454 err_union_ty,
64646455 result,
6465 Type.u8,
6456 .u8,
64666457 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
6467 ) else try self.truncateRegister(Type.anyerror, result.register);
6458 ) else try self.truncateRegister(.anyerror, result.register);
64686459 break :result result;
64696460 },
64706461 .load_frame => |frame_addr| break :result .{ .load_frame = .{
......@@ -6623,7 +6614,7 @@ fn genUnwrapErrUnionPayloadMir(
66236614 .{ ._r, .sh },
66246615 err_union_ty,
66256616 result_mcv,
6626 Type.u8,
6617 .u8,
66276618 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
66286619 ) else try self.truncateRegister(payload_ty, result_mcv.register);
66296620 break :result if (payload_in_gp)
......@@ -6929,7 +6920,7 @@ fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValu
69296920 defer self.register_manager.unlockReg(offset_reg_lock);
69306921
69316922 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6932 try self.genSetReg(addr_reg, Type.usize, slice_mcv, .{});
6923 try self.genSetReg(addr_reg, .usize, slice_mcv, .{});
69336924 // TODO we could allocate register here, but need to expect addr register and potentially
69346925 // offset register.
69356926 try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
......@@ -7064,7 +7055,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
70647055 .{ ._, .bt },
70657056 .{
70667057 .base = .{
7067 .reg = try self.copyToTmpRegister(Type.usize, array_mat_mcv.address()),
7058 .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
70687059 },
70697060 .mod = .{ .rm = .{ .size = .qword } },
70707061 },
......@@ -7074,7 +7065,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
70747065 .{ ._, .bt },
70757066 .{
70767067 .base = .{
7077 .reg = try self.copyToTmpRegister(Type.usize, array_mat_mcv.address()),
7068 .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
70787069 },
70797070 .mod = .{ .rm = .{ .size = .qword } },
70807071 },
......@@ -7122,7 +7113,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
71227113 .load_direct,
71237114 .load_got,
71247115 .load_tlv,
7125 => try self.genSetReg(addr_reg, Type.usize, array_mcv.address(), .{}),
7116 => try self.genSetReg(addr_reg, .usize, array_mcv.address(), .{}),
71267117 .lea_symbol, .lea_direct, .lea_tlv => unreachable,
71277118 else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{
71287119 @tagName(array_mcv), array_ty.fmt(pt),
......@@ -7136,12 +7127,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
71367127 // TODO we could allocate register here, but need to expect addr register and potentially
71377128 // offset register.
71387129 const dst_mcv = try self.allocRegOrMem(inst, false);
7139 try self.genBinOpMir(
7140 .{ ._, .add },
7141 Type.usize,
7142 .{ .register = addr_reg },
7143 .{ .register = offset_reg },
7144 );
7130 try self.genBinOpMir(.{ ._, .add }, .usize, .{ .register = addr_reg }, .{ .register = offset_reg });
71457131 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});
71467132 break :result dst_mcv;
71477133 };
......@@ -7329,13 +7315,7 @@ fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
73297315 .register => {
73307316 const shift: u6 = @intCast(layout.tagOffset() * 8);
73317317 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
7332 try self.genShiftBinOpMir(
7333 .{ ._r, .sh },
7334 Type.usize,
7335 result,
7336 Type.u8,
7337 .{ .immediate = shift },
7338 );
7318 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, result, .u8, .{ .immediate = shift });
73397319 break :blk MCValue{
73407320 .register = registerAlias(result.register, @intCast(layout.tag_size)),
73417321 };
......@@ -7444,7 +7424,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
74447424 if (src_bits <= 8) {
74457425 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
74467426 try self.truncateRegister(src_ty, wide_reg);
7447 try self.genBinOpMir(.{ ._, .lzcnt }, Type.u32, dst_mcv, .{ .register = wide_reg });
7427 try self.genBinOpMir(.{ ._, .lzcnt }, .u32, dst_mcv, .{ .register = wide_reg });
74487428 try self.genBinOpMir(
74497429 .{ ._, .sub },
74507430 dst_ty,
......@@ -7464,25 +7444,15 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
74647444 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
74657445 defer self.register_manager.unlockReg(tmp_lock);
74667446
7467 try self.genBinOpMir(
7468 .{ ._, .lzcnt },
7469 Type.u64,
7470 dst_mcv,
7471 if (mat_src_mcv.isBase())
7472 mat_src_mcv
7473 else
7474 .{ .register = mat_src_mcv.register_pair[0] },
7475 );
7447 try self.genBinOpMir(.{ ._, .lzcnt }, .u64, dst_mcv, if (mat_src_mcv.isBase())
7448 mat_src_mcv
7449 else
7450 .{ .register = mat_src_mcv.register_pair[0] });
74767451 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
7477 try self.genBinOpMir(
7478 .{ ._, .lzcnt },
7479 Type.u64,
7480 tmp_mcv,
7481 if (mat_src_mcv.isBase())
7482 mat_src_mcv.address().offset(8).deref()
7483 else
7484 .{ .register = mat_src_mcv.register_pair[1] },
7485 );
7452 try self.genBinOpMir(.{ ._, .lzcnt }, .u64, tmp_mcv, if (mat_src_mcv.isBase())
7453 mat_src_mcv.address().offset(8).deref()
7454 else
7455 .{ .register = mat_src_mcv.register_pair[1] });
74867456 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
74877457
74887458 if (src_bits < 128) try self.genBinOpMir(
......@@ -7510,7 +7480,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
75107480 defer self.register_manager.unlockReg(wide_lock);
75117481
75127482 try self.truncateRegister(src_ty, wide_reg);
7513 try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });
7483 try self.genBinOpMir(.{ ._, .bsr }, .u16, dst_mcv, .{ .register = wide_reg });
75147484 } else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
75157485
75167486 try self.asmCmovccRegisterRegister(
......@@ -7534,7 +7504,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
75347504 try self.truncateRegister(src_ty, wide_reg);
75357505 try self.genBinOpMir(
75367506 .{ ._, .bsr },
7537 if (src_bits <= 8) Type.u16 else src_ty,
7507 if (src_bits <= 8) .u16 else src_ty,
75387508 dst_mcv,
75397509 .{ .register = wide_reg },
75407510 );
......@@ -7645,7 +7615,7 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
76457615 break :result dst_mcv;
76467616 }
76477617
7648 const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;
7618 const wide_ty: Type = if (src_bits <= 8) .u16 else src_ty;
76497619 if (has_bmi) {
76507620 if (src_bits <= 64) {
76517621 const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
......@@ -7682,18 +7652,18 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
76827652 else
76837653 .{ .register = mat_src_mcv.register_pair[1] };
76847654 const masked_mcv = if (src_bits < 128) masked: {
7685 try self.genCopy(Type.u64, dst_mcv, hi_mat_src_mcv, .{});
7655 try self.genCopy(.u64, dst_mcv, hi_mat_src_mcv, .{});
76867656 try self.genBinOpMir(
76877657 .{ ._, .@"or" },
7688 Type.u64,
7658 .u64,
76897659 dst_mcv,
76907660 .{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) },
76917661 );
76927662 break :masked dst_mcv;
76937663 } else hi_mat_src_mcv;
7694 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
7664 try self.genBinOpMir(.{ ._, .tzcnt }, .u64, dst_mcv, masked_mcv);
76957665 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
7696 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, lo_mat_src_mcv);
7666 try self.genBinOpMir(.{ ._, .tzcnt }, .u64, tmp_mcv, lo_mat_src_mcv);
76977667 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
76987668 }
76997669 break :result dst_mcv;
......@@ -7766,7 +7736,7 @@ fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void {
77667736 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
77677737 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
77687738
7769 try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isBase())
7739 try self.genPopCount(tmp_regs[0], .usize, if (mat_src_mcv.isBase())
77707740 mat_src_mcv
77717741 else
77727742 .{ .register = mat_src_mcv.register_pair[0] }, false);
......@@ -7794,7 +7764,7 @@ fn genPopCount(
77947764 const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu));
77957765 if (self.hasFeature(.popcnt)) return self.genBinOpMir(
77967766 .{ ._, .popcnt },
7797 if (src_abi_size > 1) src_ty else Type.u32,
7767 if (src_abi_size > 1) src_ty else .u32,
77987768 .{ .register = dst_reg },
77997769 if (src_abi_size > 1) src_mcv else src: {
78007770 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
......@@ -8057,7 +8027,7 @@ fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void {
80578027 } },
80588028 src_ty,
80598029 dst_mcv,
8060 if (src_bits > 256) Type.u16 else Type.u8,
8030 if (src_bits > 256) .u16 else .u8,
80618031 .{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits },
80628032 );
80638033 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
......@@ -8181,7 +8151,7 @@ fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
81818151 if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) {
81828152 .signed => .{ ._r, .sa },
81838153 .unsigned => .{ ._r, .sh },
8184 }, src_ty, dst_mcv, Type.u8, .{ .immediate = extra_bits });
8154 }, src_ty, dst_mcv, .u8, .{ .immediate = extra_bits });
81858155
81868156 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
81878157}
......@@ -8252,7 +8222,7 @@ fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Ty
82528222 try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) })
82538223 else
82548224 .{
8255 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
8225 .base = .{ .reg = try self.copyToTmpRegister(.usize, sign_mcv.address()) },
82568226 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
82578227 };
82588228
......@@ -8947,9 +8917,9 @@ fn packedLoad(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue)
89478917 defer self.register_manager.unlockReg(tmp_lock);
89488918
89498919 const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
8950 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
8920 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
89518921 try self.truncateRegister(val_ty, tmp_reg);
8952 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
8922 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
89538923 }
89548924 }
89558925 return;
......@@ -9104,13 +9074,13 @@ fn airLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
91049074 const high_reg = if (high_mcv.isRegister())
91059075 high_mcv.getReg().?
91069076 else
9107 try self.copyToTmpRegister(Type.usize, high_mcv);
9077 try self.copyToTmpRegister(.usize, high_mcv);
91089078 const high_lock = self.register_manager.lockReg(high_reg);
91099079 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
91109080
91119081 try self.truncateRegister(elem_ty, high_reg);
91129082 if (!high_mcv.isRegister()) try self.genCopy(
9113 if (elem_size <= 8) elem_ty else Type.usize,
9083 if (elem_size <= 8) elem_ty else .usize,
91149084 high_mcv,
91159085 .{ .register = high_reg },
91169086 .{},
......@@ -9183,14 +9153,14 @@ fn packedStore(self: *CodeGen, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue)
91839153 .{ ._l, .sh },
91849154 limb_ty,
91859155 tmp_mcv,
9186 Type.u8,
9156 .u8,
91879157 .{ .immediate = src_bit_off },
91889158 ),
91899159 1 => try self.genShiftBinOpMir(
91909160 .{ ._r, .sh },
91919161 limb_ty,
91929162 tmp_mcv,
9193 Type.u8,
9163 .u8,
91949164 .{ .immediate = limb_abi_bits - src_bit_off },
91959165 ),
91969166 else => unreachable,
......@@ -9355,7 +9325,10 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
93559325 const src_mcv = try self.resolveInst(operand);
93569326 const field_off: u32 = switch (container_ty.containerLayout(zcu)) {
93579327 .auto, .@"extern" => @intCast(container_ty.structFieldOffset(extra.field_index, zcu) * 8),
9358 .@"packed" => if (zcu.typeToStruct(container_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0,
9328 .@"packed" => if (zcu.typeToStruct(container_ty)) |struct_obj|
9329 pt.structPackedFieldBitOffset(struct_obj, extra.field_index)
9330 else
9331 0,
93599332 };
93609333
93619334 switch (src_mcv) {
......@@ -9370,20 +9343,14 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
93709343 else if (field_off == 0)
93719344 (try self.copyToRegisterWithInstTracking(inst, field_ty, src_mcv)).register
93729345 else
9373 try self.copyToTmpRegister(Type.usize, .{ .register = src_reg });
9346 try self.copyToTmpRegister(.usize, .{ .register = src_reg });
93749347 const dst_mcv: MCValue = .{ .register = dst_reg };
93759348 const dst_lock = self.register_manager.lockReg(dst_reg);
93769349 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
93779350
93789351 if (field_off > 0) {
93799352 try self.spillEflagsIfOccupied();
9380 try self.genShiftBinOpMir(
9381 .{ ._r, .sh },
9382 Type.usize,
9383 dst_mcv,
9384 Type.u8,
9385 .{ .immediate = field_off },
9386 );
9353 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, dst_mcv, .u8, .{ .immediate = field_off });
93879354 }
93889355 if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and
93899356 container_ty.abiSize(zcu) * 8 > field_ty.bitSize(zcu))
......@@ -9421,13 +9388,7 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
94219388
94229389 if (field_off > 0) {
94239390 try self.spillEflagsIfOccupied();
9424 try self.genShiftBinOpMir(
9425 .{ ._r, .sh },
9426 Type.u128,
9427 dst_mcv,
9428 Type.u8,
9429 .{ .immediate = field_off },
9430 );
9391 try self.genShiftBinOpMir(.{ ._r, .sh }, .u128, dst_mcv, .u8, .{ .immediate = field_off });
94319392 }
94329393
94339394 if (field_bit_size <= 64) {
......@@ -9451,20 +9412,14 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
94519412 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
94529413 };
94539414
9454 const dst_reg = try self.copyToTmpRegister(Type.usize, .{ .register = src_reg });
9415 const dst_reg = try self.copyToTmpRegister(.usize, .{ .register = src_reg });
94559416 const dst_mcv = MCValue{ .register = dst_reg };
94569417 const dst_lock = self.register_manager.lockReg(dst_reg);
94579418 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
94589419
94599420 if (field_off % 64 > 0) {
94609421 try self.spillEflagsIfOccupied();
9461 try self.genShiftBinOpMir(
9462 .{ ._r, .sh },
9463 Type.usize,
9464 dst_mcv,
9465 Type.u8,
9466 .{ .immediate = field_off % 64 },
9467 );
9422 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, dst_mcv, .u8, .{ .immediate = field_off % 64 });
94689423 }
94699424 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
94709425
......@@ -9479,11 +9434,7 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
94799434 0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
94809435 self.eflags_inst = null; // actually stop tracking the overflow part
94819436 break :result .{ .register = ro.reg };
9482 } else break :result try self.copyToRegisterWithInstTracking(
9483 inst,
9484 Type.usize,
9485 .{ .register = ro.reg },
9486 ),
9437 } else break :result try self.copyToRegisterWithInstTracking(inst, .usize, .{ .register = ro.reg }),
94879438 // Get overflow bit.
94889439 1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
94899440 self.eflags_inst = inst; // actually keep tracking the overflow part
......@@ -9541,9 +9492,9 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
95419492
95429493 const hi_mcv =
95439494 dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
9544 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
9495 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
95459496 try self.truncateRegister(field_ty, tmp_reg);
9546 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
9497 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
95479498 }
95489499 break :result dst_mcv;
95499500 }
......@@ -9684,8 +9635,8 @@ fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_a
96849635 switch (tag) {
96859636 .not => {
96869637 const limb_abi_size: u16 = @min(abi_size, 8);
9687 const int_info = if (src_ty.ip_index == .bool_type)
9688 std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }
9638 const int_info: InternPool.Key.IntType = if (src_ty.ip_index == .bool_type)
9639 .{ .signedness = .unsigned, .bits = 1 }
96899640 else
96909641 src_ty.intInfo(zcu);
96919642 var byte_off: i32 = 0;
......@@ -9718,9 +9669,9 @@ fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_a
97189669 defer self.register_manager.unlockReg(tmp_lock);
97199670
97209671 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
9721 try self.genSetReg(tmp_reg, Type.usize, hi_mcv, .{});
9672 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
97229673 try self.truncateRegister(src_ty, tmp_reg);
9723 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg }, .{});
9674 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
97249675 }
97259676 }
97269677 },
......@@ -9759,7 +9710,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv:
97599710 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
97609711 defer self.register_manager.unlockReg(addr_reg_lock);
97619712
9762 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address(), .{});
9713 try self.genSetReg(addr_reg, .usize, dst_mcv.address(), .{});
97639714 try self.asmMemory(mir_tag, .{ .base = .{ .reg = addr_reg }, .mod = .{ .rm = .{
97649715 .size = .fromSize(abi_size),
97659716 } } });
......@@ -10495,7 +10446,7 @@ fn genMulDivBinOp(
1049510446 const mat_lhs_mcv = switch (lhs_mcv) {
1049610447 .load_symbol => mat_lhs_mcv: {
1049710448 // TODO clean this up!
10498 const addr_reg = try self.copyToTmpRegister(Type.usize, lhs_mcv.address());
10449 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
1049910450 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
1050010451 },
1050110452 else => lhs_mcv,
......@@ -10508,7 +10459,7 @@ fn genMulDivBinOp(
1050810459 const mat_rhs_mcv = switch (rhs_mcv) {
1050910460 .load_symbol => mat_rhs_mcv: {
1051010461 // TODO clean this up!
10511 const addr_reg = try self.copyToTmpRegister(Type.usize, rhs_mcv.address());
10462 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
1051210463 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
1051310464 },
1051410465 else => rhs_mcv,
......@@ -10696,7 +10647,7 @@ fn genMulDivBinOp(
1069610647 manyptr_u32_ty,
1069710648 manyptr_const_u32_ty,
1069810649 manyptr_const_u32_ty,
10699 Type.usize,
10650 .usize,
1070010651 }, &.{
1070110652 dst_mcv.address(),
1070210653 lhs_mcv.address(),
......@@ -12493,7 +12444,7 @@ fn genBinOp(
1249312444 try not_mcv.mem(self, .{ .size = .fromSize(abi_size) })
1249412445 else
1249512446 .{ .base = .{
12496 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
12447 .reg = try self.copyToTmpRegister(.usize, not_mcv.address()),
1249712448 }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } };
1249812449 switch (mir_tag[0]) {
1249912450 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
......@@ -12757,7 +12708,7 @@ fn genBinOpMir(
1275712708 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
1275812709 errdefer self.register_manager.unlockReg(dst_addr_lock);
1275912710
12760 try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address(), .{});
12711 try self.genSetReg(dst_addr_reg, .usize, dst_mcv.address(), .{});
1276112712 break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock };
1276212713 },
1276312714 .load_frame => null,
......@@ -12808,7 +12759,7 @@ fn genBinOpMir(
1280812759 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
1280912760 errdefer self.register_manager.unlockReg(src_addr_lock);
1281012761
12811 try self.genSetReg(src_addr_reg, Type.usize, resolved_src_mcv.address(), .{});
12762 try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
1281212763 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
1281312764 },
1281412765 };
......@@ -12816,9 +12767,9 @@ fn genBinOpMir(
1281612767
1281712768 const ty_signedness =
1281812769 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned;
12819 const limb_ty = if (abi_size <= 8) ty else switch (ty_signedness) {
12820 .signed => Type.usize,
12821 .unsigned => Type.isize,
12770 const limb_ty: Type = if (abi_size <= 8) ty else switch (ty_signedness) {
12771 .signed => .usize,
12772 .unsigned => .isize,
1282212773 };
1282312774 var limb_i: usize = 0;
1282412775 var off: i32 = 0;
......@@ -13310,13 +13261,13 @@ fn genLocalDebugInfo(
1331013261
1331113262fn airRetAddr(self: *CodeGen, inst: Air.Inst.Index) !void {
1331213263 const dst_mcv = try self.allocRegOrMem(inst, true);
13313 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
13264 try self.genCopy(.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
1331413265 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1331513266}
1331613267
1331713268fn airFrameAddress(self: *CodeGen, inst: Air.Inst.Index) !void {
1331813269 const dst_mcv = try self.allocRegOrMem(inst, true);
13319 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
13270 try self.genCopy(.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
1332013271 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
1332113272}
1332213273
......@@ -13461,13 +13412,10 @@ fn genCall(self: *CodeGen, info: union(enum) {
1346113412 defer self.register_manager.unlockReg(index_lock);
1346213413
1346313414 const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{
13464 .base = .{ .reg = try self.copyToTmpRegister(
13465 Type.usize,
13466 switch (src_arg) {
13467 else => src_arg,
13468 .air_ref => |src_ref| try self.resolveInst(src_ref),
13469 }.address(),
13470 ) },
13415 .base = .{ .reg = try self.copyToTmpRegister(.usize, switch (src_arg) {
13416 else => src_arg,
13417 .air_ref => |src_ref| try self.resolveInst(src_ref),
13418 }.address()) },
1347113419 .mod = .{ .rm = .{ .size = .dword } },
1347213420 };
1347313421 const src_lock = switch (src_mem.base) {
......@@ -13519,7 +13467,7 @@ fn genCall(self: *CodeGen, info: union(enum) {
1351913467 .indirect => |reg_off| {
1352013468 const ret_ty: Type = .fromInterned(fn_info.return_type);
1352113469 const frame_index = try self.allocFrameIndex(.initSpill(ret_ty, zcu));
13522 try self.genSetReg(reg_off.reg, Type.usize, .{
13470 try self.genSetReg(reg_off.reg, .usize, .{
1352313471 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
1352413472 }, .{});
1352513473 call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
......@@ -13548,13 +13496,13 @@ fn genCall(self: *CodeGen, info: union(enum) {
1354813496 },
1354913497 },
1355013498 .register_pair => try self.genCopy(arg_ty, dst_arg, src_arg, .{}),
13551 .indirect => |reg_off| try self.genSetReg(reg_off.reg, Type.usize, .{
13499 .indirect => |reg_off| try self.genSetReg(reg_off.reg, .usize, .{
1355213500 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
1355313501 }, .{}),
1355413502 .elementwise_regs_then_frame => |regs_frame_addr| {
1355513503 const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{
1355613504 .base = .{ .reg = try self.copyToTmpRegister(
13557 Type.usize,
13505 .usize,
1355813506 switch (src_arg) {
1355913507 else => src_arg,
1356013508 .air_ref => |src_ref| try self.resolveInst(src_ref),
......@@ -13604,7 +13552,7 @@ fn genCall(self: *CodeGen, info: union(enum) {
1360413552 } else if (self.bin_file.cast(.coff)) |coff_file| {
1360513553 const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav);
1360613554 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
13607 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index }, .{});
13555 try self.genSetReg(.rax, .usize, .{ .lea_got = sym_index }, .{});
1360813556 try self.asmRegister(.{ ._, .call }, .rax);
1360913557 } else if (self.bin_file.cast(.macho)) |macho_file| {
1361013558 const zo = macho_file.getZigObject().?;
......@@ -13644,7 +13592,7 @@ fn genCall(self: *CodeGen, info: union(enum) {
1364413592 }
1364513593 } else {
1364613594 assert(self.typeOf(callee).zigTypeTag(zcu) == .pointer);
13647 try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }, .{});
13595 try self.genSetReg(.rax, .usize, .{ .air_ref = callee }, .{});
1364813596 try self.asmRegister(.{ ._, .call }, .rax);
1364913597 },
1365013598 .lib => |lib| if (self.bin_file.cast(.elf)) |elf_file| {
......@@ -13676,7 +13624,7 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1367613624 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
1367713625 defer self.register_manager.unlockReg(lock);
1367813626
13679 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long, .{});
13627 try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{});
1368013628 try self.genSetMem(
1368113629 .{ .reg = reg_off.reg },
1368213630 reg_off.off,
......@@ -13775,7 +13723,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v
1377513723 floatCompilerRtAbiName(float_bits),
1377613724 }) catch unreachable,
1377713725 } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
13778 try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
13726 try self.genBinOpMir(.{ ._, .@"test" }, .i32, ret, ret);
1377913727 break :result switch (op) {
1378013728 .eq => .e,
1378113729 .neq => .ne,
......@@ -13938,12 +13886,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v
1393813886 self.register_manager.lockRegAssumeUnused(dst_addr_reg);
1393913887 errdefer self.register_manager.unlockReg(dst_addr_lock);
1394013888
13941 try self.genSetReg(
13942 dst_addr_reg,
13943 Type.usize,
13944 resolved_dst_mcv.address(),
13945 .{},
13946 );
13889 try self.genSetReg(dst_addr_reg, .usize, resolved_dst_mcv.address(), .{});
1394713890 break :dst .{
1394813891 .addr_reg = dst_addr_reg,
1394913892 .addr_lock = dst_addr_lock,
......@@ -14000,12 +13943,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v
1400013943 self.register_manager.lockRegAssumeUnused(src_addr_reg);
1400113944 errdefer self.register_manager.unlockReg(src_addr_lock);
1400213945
14003 try self.genSetReg(
14004 src_addr_reg,
14005 Type.usize,
14006 resolved_src_mcv.address(),
14007 .{},
14008 );
13946 try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
1400913947 break :src .{
1401013948 .addr_reg = src_addr_reg,
1401113949 .addr_lock = src_addr_lock,
......@@ -14026,7 +13964,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v
1402613964 const off = limb_i * 8;
1402713965 const tmp_reg = regs[@min(limb_i, 1)].to64();
1402813966
14029 try self.genSetReg(tmp_reg, Type.usize, if (dst_info) |info| .{
13967 try self.genSetReg(tmp_reg, .usize, if (dst_info) |info| .{
1403013968 .indirect = .{ .reg = info.addr_reg, .off = off },
1403113969 } else switch (resolved_dst_mcv) {
1403213970 inline .register_pair,
......@@ -14049,7 +13987,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v
1404913987
1405013988 try self.genBinOpMir(
1405113989 .{ ._, .xor },
14052 Type.usize,
13990 .usize,
1405313991 .{ .register = tmp_reg },
1405413992 if (src_info) |info| .{
1405513993 .indirect = .{ .reg = info.addr_reg, .off = off },
......@@ -14244,7 +14182,7 @@ fn genTry(
1424414182 else
1424514183 try self.isErr(null, operand_ty, operand_mcv);
1424614184
14247 const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv);
14185 const reloc = try self.genCondBrMir(.anyerror, is_err_mcv);
1424814186
1424914187 if (self.liveness.operandDies(inst, 0)) {
1425014188 if (operand.toIndex()) |operand_inst| try self.processDeath(operand_inst);
......@@ -14406,7 +14344,7 @@ fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue)
1440614344 const some_info: struct { off: u31, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
1440714345 .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
1440814346 else
14409 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
14347 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = .bool };
1441014348
1441114349 self.eflags_inst = inst;
1441214350 switch (opt_mcv) {
......@@ -14481,7 +14419,7 @@ fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue)
1448114419 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1448214420 defer self.register_manager.unlockReg(addr_reg_lock);
1448314421
14484 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address(), .{});
14422 try self.genSetReg(addr_reg, .usize, opt_mcv.address(), .{});
1448514423 const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu));
1448614424 try self.asmMemoryImmediate(
1448714425 .{ ._, .cmp },
......@@ -14536,7 +14474,7 @@ fn isNullPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValu
1453614474 const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
1453714475 .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
1453814476 else
14539 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
14477 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = .bool };
1454014478
1454114479 const ptr_reg = switch (ptr_mcv) {
1454214480 .register => |reg| reg,
......@@ -14582,22 +14520,17 @@ fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCVal
1458214520 .{ ._r, .sh },
1458314521 eu_ty,
1458414522 .{ .register = tmp_reg },
14585 Type.u8,
14523 .u8,
1458614524 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
1458714525 );
1458814526 } else {
14589 try self.truncateRegister(Type.anyerror, tmp_reg);
14527 try self.truncateRegister(.anyerror, tmp_reg);
1459014528 }
14591 try self.genBinOpMir(
14592 .{ ._, .cmp },
14593 Type.anyerror,
14594 .{ .register = tmp_reg },
14595 .{ .immediate = 0 },
14596 );
14529 try self.genBinOpMir(.{ ._, .cmp }, .anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
1459714530 },
1459814531 .load_frame => |frame_addr| try self.genBinOpMir(
1459914532 .{ ._, .cmp },
14600 Type.anyerror,
14533 .anyerror,
1460114534 .{ .load_frame = .{
1460214535 .index = frame_addr.index,
1460314536 .off = frame_addr.off + err_off,
......@@ -14633,7 +14566,7 @@ fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv:
1463314566 .{
1463414567 .base = .{ .reg = ptr_reg },
1463514568 .mod = .{ .rm = .{
14636 .size = self.memSize(Type.anyerror),
14569 .size = self.memSize(.anyerror),
1463714570 .disp = err_off,
1463814571 } },
1463914572 },
......@@ -15200,7 +15133,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1520015133 .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
1520115134 else => {},
1520215135 }
15203 break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(Type.usize, ptr_mcv) } };
15136 break :arg .{ .indirect = .{ .reg = try self.copyToTmpRegister(.usize, ptr_mcv) } };
1520415137 };
1520515138 };
1520615139 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
......@@ -15262,7 +15195,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1526215195 try self.genCopy(ty, temp_mcv, input_mcv, .{});
1526315196 break :arg temp_mcv;
1526415197 };
15265 try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{});
15198 try self.genSetReg(addr_reg, .usize, input_mcv.address(), .{});
1526615199 break :arg .{ .indirect = .{ .reg = addr_reg } };
1526715200 } else if (std.mem.eql(u8, constraint, "g") or
1526815201 std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or
......@@ -15539,11 +15472,11 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
1553915472 else
1554015473 return self.fail("invalid modifier: '{s}'", .{modifier}),
1554115474 .lea_got => |sym_index| if (std.mem.eql(u8, modifier, "P"))
15542 .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) }
15475 .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_got = sym_index }) }
1554315476 else
1554415477 return self.fail("invalid modifier: '{s}'", .{modifier}),
1554515478 .lea_symbol => |sym_off| if (std.mem.eql(u8, modifier, "P"))
15546 .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) }
15479 .{ .reg = try self.copyToTmpRegister(.usize, .{ .lea_symbol = sym_off }) }
1554715480 else
1554815481 return self.fail("invalid modifier: '{s}'", .{modifier}),
1554915482 else => return self.fail("invalid constraint: '{s}'", .{op_str}),
......@@ -16099,7 +16032,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
1609916032 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
1610016033 errdefer self.register_manager.unlockReg(src_addr_lock);
1610116034
16102 try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address(), opts);
16035 try self.genSetReg(src_addr_reg, .usize, src_mcv.address(), opts);
1610316036 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
1610416037 },
1610516038 .air_ref => |src_ref| return self.genCopy(
......@@ -16146,7 +16079,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
1614616079 else => unreachable,
1614716080 }
1614816081
16149 const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());
16082 const addr_reg = try self.copyToTmpRegister(.usize, dst_mcv.address());
1615016083 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1615116084 defer self.register_manager.unlockReg(addr_lock);
1615216085
......@@ -16445,7 +16378,7 @@ fn genSetReg(
1644516378 else => unreachable,
1644616379 }
1644716380
16448 const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());
16381 const addr_reg = try self.copyToTmpRegister(.usize, src_mcv.address());
1644916382 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1645016383 defer self.register_manager.unlockReg(addr_lock);
1645116384
......@@ -16656,7 +16589,7 @@ fn genSetMem(
1665616589 try self.genSetMem(
1665716590 base,
1665816591 disp + @as(i32, @intCast(child_ty.abiSize(zcu))),
16659 Type.bool,
16592 .bool,
1666016593 .{ .eflags = ro.eflags },
1666116594 opts,
1666216595 );
......@@ -16732,9 +16665,9 @@ fn genSetMem(
1673216665
1673316666fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
1673416667 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
16735 try self.genSetReg(.rsi, Type.usize, src_ptr, .{});
16736 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
16737 try self.genSetReg(.rcx, Type.usize, len, .{});
16668 try self.genSetReg(.rsi, .usize, src_ptr, .{});
16669 try self.genSetReg(.rdi, .usize, dst_ptr, .{});
16670 try self.genSetReg(.rcx, .usize, len, .{});
1673816671 try self.asmOpOnly(.{ .@"rep _sb", .mov });
1673916672}
1674016673
......@@ -16746,9 +16679,9 @@ fn genInlineMemset(
1674616679 opts: CopyOptions,
1674716680) InnerError!void {
1674816681 try self.spillRegisters(&.{ .rdi, .al, .rcx });
16749 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
16750 try self.genSetReg(.al, Type.u8, value, opts);
16751 try self.genSetReg(.rcx, Type.usize, len, .{});
16682 try self.genSetReg(.rdi, .usize, dst_ptr, .{});
16683 try self.genSetReg(.al, .u8, value, opts);
16684 try self.genSetReg(.rcx, .usize, len, .{});
1675216685 try self.asmOpOnly(.{ .@"rep _sb", .sto });
1675316686}
1675416687
......@@ -16791,10 +16724,10 @@ fn genLazySymbolRef(
1679116724 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1679216725 if (self.mod.pic) {
1679316726 switch (tag) {
16794 .lea, .call => try self.genSetReg(reg, Type.usize, .{
16727 .lea, .call => try self.genSetReg(reg, .usize, .{
1679516728 .lea_symbol = .{ .sym_index = sym_index },
1679616729 }, .{}),
16797 .mov => try self.genSetReg(reg, Type.usize, .{
16730 .mov => try self.genSetReg(reg, .usize, .{
1679816731 .load_symbol = .{ .sym_index = sym_index },
1679916732 }, .{}),
1680016733 else => unreachable,
......@@ -16844,8 +16777,8 @@ fn genLazySymbolRef(
1684416777 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1684516778 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
1684616779 switch (tag) {
16847 .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }, .{}),
16848 .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }, .{}),
16780 .lea, .call => try self.genSetReg(reg, .usize, .{ .lea_got = sym_index }, .{}),
16781 .mov => try self.genSetReg(reg, .usize, .{ .load_got = sym_index }, .{}),
1684916782 else => unreachable,
1685016783 }
1685116784 switch (tag) {
......@@ -16859,10 +16792,10 @@ fn genLazySymbolRef(
1685916792 return self.fail("{s} creating lazy symbol", .{@errorName(err)});
1686016793 const sym = zo.symbols.items[sym_index];
1686116794 switch (tag) {
16862 .lea, .call => try self.genSetReg(reg, Type.usize, .{
16795 .lea, .call => try self.genSetReg(reg, .usize, .{
1686316796 .lea_symbol = .{ .sym_index = sym.nlist_idx },
1686416797 }, .{}),
16865 .mov => try self.genSetReg(reg, Type.usize, .{
16798 .mov => try self.genSetReg(reg, .usize, .{
1686616799 .load_symbol = .{ .sym_index = sym.nlist_idx },
1686716800 }, .{}),
1686816801 else => unreachable,
......@@ -16932,7 +16865,7 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
1693216865 const bit_size = dst_ty.bitSize(zcu);
1693316866 if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
1693416867
16935 const dst_limbs_len = std.math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
16868 const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
1693616869 const high_mcv: MCValue = switch (dst_mcv) {
1693716870 .register => |dst_reg| .{ .register = dst_reg },
1693816871 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
......@@ -16941,17 +16874,43 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
1694116874 const high_reg = if (high_mcv.isRegister())
1694216875 high_mcv.getReg().?
1694316876 else
16944 try self.copyToTmpRegister(Type.usize, high_mcv);
16877 try self.copyToTmpRegister(.usize, high_mcv);
1694516878 const high_lock = self.register_manager.lockReg(high_reg);
1694616879 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
16947
1694816880 try self.truncateRegister(dst_ty, high_reg);
1694916881 if (!high_mcv.isRegister()) try self.genCopy(
16950 if (abi_size <= 8) dst_ty else Type.usize,
16882 if (abi_size <= 8) dst_ty else .usize,
1695116883 high_mcv,
1695216884 .{ .register = high_reg },
1695316885 .{},
1695416886 );
16887 var offset = dst_limbs_len * 8;
16888 if (offset < abi_size) {
16889 const dst_signedness: std.builtin.Signedness = if (dst_ty.isAbiInt(zcu))
16890 dst_ty.intInfo(zcu).signedness
16891 else
16892 .unsigned;
16893 const ext_mcv: MCValue = ext_mcv: switch (dst_signedness) {
16894 .signed => {
16895 try self.asmRegisterImmediate(.{ ._r, .sa }, high_reg, .u(63));
16896 break :ext_mcv .{ .register = high_reg };
16897 },
16898 .unsigned => .{ .immediate = 0 },
16899 };
16900 while (offset < abi_size) : (offset += 8) {
16901 const limb_mcv: MCValue = switch (dst_mcv) {
16902 .register => |dst_reg| .{ .register = dst_reg },
16903 .register_pair => |dst_regs| .{ .register = dst_regs[@divExact(offset, 8)] },
16904 else => dst_mcv.address().offset(offset).deref(),
16905 };
16906 const limb_lock = if (limb_mcv.isRegister())
16907 self.register_manager.lockReg(limb_mcv.getReg().?)
16908 else
16909 null;
16910 defer if (limb_lock) |lock| self.register_manager.unlockReg(lock);
16911 try self.genCopy(.usize, limb_mcv, ext_mcv, .{});
16912 }
16913 }
1695516914 break :result dst_mcv;
1695616915 };
1695716916 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -16973,7 +16932,7 @@ fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
1697316932 try self.genSetMem(
1697416933 .{ .frame = frame_index },
1697516934 @intCast(ptr_ty.abiSize(zcu)),
16976 Type.usize,
16935 .usize,
1697716936 .{ .immediate = array_len },
1697816937 .{},
1697916938 );
......@@ -17151,28 +17110,28 @@ fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
1715117110 if (val_abi_size > 8) {
1715217111 const exp_addr_mcv: MCValue = switch (exp_mcv) {
1715317112 .memory, .indirect, .load_frame => exp_mcv.address(),
17154 else => .{ .register = try self.copyToTmpRegister(Type.usize, exp_mcv.address()) },
17113 else => .{ .register = try self.copyToTmpRegister(.usize, exp_mcv.address()) },
1715517114 };
1715617115 const exp_addr_lock =
1715717116 if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1715817117 defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock);
1715917118
17160 try self.genSetReg(.rax, Type.usize, exp_addr_mcv.deref(), .{});
17161 try self.genSetReg(.rdx, Type.usize, exp_addr_mcv.offset(8).deref(), .{});
17119 try self.genSetReg(.rax, .usize, exp_addr_mcv.deref(), .{});
17120 try self.genSetReg(.rdx, .usize, exp_addr_mcv.offset(8).deref(), .{});
1716217121 } else try self.genSetReg(.rax, val_ty, exp_mcv, .{});
1716317122
1716417123 const new_mcv = try self.resolveInst(extra.new_value);
1716517124 const new_reg = if (val_abi_size > 8) new: {
1716617125 const new_addr_mcv: MCValue = switch (new_mcv) {
1716717126 .memory, .indirect, .load_frame => new_mcv.address(),
17168 else => .{ .register = try self.copyToTmpRegister(Type.usize, new_mcv.address()) },
17127 else => .{ .register = try self.copyToTmpRegister(.usize, new_mcv.address()) },
1716917128 };
1717017129 const new_addr_lock =
1717117130 if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
1717217131 defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock);
1717317132
17174 try self.genSetReg(.rbx, Type.usize, new_addr_mcv.deref(), .{});
17175 try self.genSetReg(.rcx, Type.usize, new_addr_mcv.offset(8).deref(), .{});
17133 try self.genSetReg(.rbx, .usize, new_addr_mcv.deref(), .{});
17134 try self.genSetReg(.rcx, .usize, new_addr_mcv.offset(8).deref(), .{});
1717617135 break :new null;
1717717136 } else try self.copyToTmpRegister(val_ty, new_mcv);
1717817137 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
......@@ -17213,9 +17172,9 @@ fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
1721317172 }
1721417173
1721517174 const dst_mcv = try self.allocRegOrMem(inst, false);
17216 try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax }, .{});
17217 try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
17218 try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
17175 try self.genCopy(.usize, dst_mcv, .{ .register = .rax }, .{});
17176 try self.genCopy(.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx }, .{});
17177 try self.genCopy(.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne }, .{});
1721917178 break :result dst_mcv;
1722017179 };
1722117180 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
......@@ -17488,7 +17447,7 @@ fn atomicOp(
1748817447 const val_mem_mcv: MCValue = switch (val_mcv) {
1748917448 .memory, .indirect, .load_frame => val_mcv,
1749017449 else => .{ .indirect = .{
17491 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
17450 .reg = try self.copyToTmpRegister(.usize, val_mcv.address()),
1749217451 } },
1749317452 };
1749417453 const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword });
......@@ -17545,7 +17504,7 @@ fn atomicOp(
1754517504 },
1754617505 };
1754717506
17548 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .register = .rcx });
17507 const tmp_reg = try self.copyToTmpRegister(.usize, .{ .register = .rcx });
1754917508 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
1755017509 defer self.register_manager.unlockReg(tmp_lock);
1755117510
......@@ -17719,7 +17678,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1771917678 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
1772017679 defer self.register_manager.unlockReg(len_lock);
1772117680
17722 try self.genSetReg(len_reg, Type.usize, len, .{});
17681 try self.genSetReg(len_reg, .usize, len, .{});
1772317682 try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg);
1772417683
1772517684 const skip_reloc = try self.asmJccReloc(.z, undefined);
......@@ -17732,12 +17691,12 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1773217691 self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
1773317692 defer self.register_manager.unlockReg(second_elem_ptr_lock);
1773417693
17735 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
17736 .reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
17694 try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{
17695 .reg = try self.copyToTmpRegister(.usize, dst_ptr),
1773717696 .off = elem_abi_size,
1773817697 } }, .{});
1773917698
17740 try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
17699 try self.genBinOpMir(.{ ._, .sub }, .usize, len_mcv, .{ .immediate = 1 });
1774117700 try self.asmRegisterRegisterImmediate(
1774217701 .{ .i_, .mul },
1774317702 len_reg,
......@@ -17763,8 +17722,8 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
1776317722 self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
1776417723 defer self.register_manager.unlockReg(second_elem_ptr_lock);
1776517724
17766 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
17767 .reg = try self.copyToTmpRegister(Type.usize, dst),
17725 try self.genSetReg(second_elem_ptr_reg, .usize, .{ .register_offset = .{
17726 .reg = try self.copyToTmpRegister(.usize, dst),
1776817727 .off = elem_abi_size,
1776917728 } }, .{});
1777017729
......@@ -17886,7 +17845,7 @@ fn airTagName(self: *CodeGen, inst: Air.Inst.Index) !void {
1788617845 const param_regs = abi.getCAbiIntParamRegs(resolved_cc);
1788717846
1788817847 const dst_mcv = try self.allocRegOrMem(inst, false);
17889 try self.genSetReg(param_regs[0], Type.usize, dst_mcv.address(), .{});
17848 try self.genSetReg(param_regs[0], .usize, dst_mcv.address(), .{});
1789017849
1789117850 const operand = try self.resolveInst(un_op);
1789217851 try self.genSetReg(param_regs[1], enum_ty, operand, .{});
......@@ -18415,7 +18374,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
1841518374 mask_alias,
1841618375 if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{
1841718376 .base = .{ .reg = (try self.copyToTmpRegister(
18418 Type.usize,
18377 .usize,
1841918378 pred_mcv.address(),
1842018379 )).to64() },
1842118380 .mod = .{ .rm = .{ .size = .byte } },
......@@ -18478,7 +18437,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
1847818437 .storage = .{ .elems = mask_elems[0..vec_len] },
1847918438 } })));
1848018439 const mask_mem: Memory = .{
18481 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
18440 .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) },
1848218441 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
1848318442 };
1848418443 if (has_avx) try self.asmRegisterRegisterMemory(
......@@ -18503,7 +18462,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
1850318462 .storage = .{ .elems = mask_elems[0..vec_len] },
1850418463 } })));
1850518464 const mask_mem: Memory = .{
18506 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
18465 .base = .{ .reg = try self.copyToTmpRegister(.usize, mask_mcv.address()) },
1850718466 .mod = .{ .rm = .{ .size = self.memSize(ty) } },
1850818467 };
1850918468 if (has_avx) {
......@@ -19380,7 +19339,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
1938019339 .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
1938119340 } })));
1938219341 const lhs_mask_mem: Memory = .{
19383 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
19342 .base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) },
1938419343 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
1938519344 };
1938619345 if (has_avx) try self.asmRegisterRegisterMemory(
......@@ -19414,7 +19373,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
1941419373 .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
1941519374 } })));
1941619375 const rhs_mask_mem: Memory = .{
19417 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
19376 .base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) },
1941819377 .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
1941919378 };
1942019379 if (has_avx) try self.asmRegisterRegisterMemory(
......@@ -19634,7 +19593,7 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
1963419593 .{ ._l, .sh },
1963519594 elem_ty,
1963619595 .{ .register = temp_alias },
19637 Type.u8,
19596 .u8,
1963819597 .{ .immediate = elem_bit_off },
1963919598 );
1964019599 try self.genBinOpMir(
......@@ -19657,7 +19616,7 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
1965719616 .{ ._r, .sh },
1965819617 elem_ty,
1965919618 .{ .register = temp_reg },
19660 Type.u8,
19619 .u8,
1966119620 .{ .immediate = elem_abi_bits - elem_bit_off },
1966219621 );
1966319622 try self.genBinOpMir(
......@@ -19984,7 +19943,7 @@ fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
1998419943 const pt = self.pt;
1998519944 const zcu = pt.zcu;
1998619945 const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;
19987 const ptr_anyopaque_ty = try pt.singleMutPtrType(Type.anyopaque);
19946 const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque);
1998819947
1998919948 const result: MCValue = switch (abi.resolveCallingConvention(
1999019949 self.fn_type.fnCallingConvention(zcu),
......@@ -19998,7 +19957,7 @@ fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
1999819957 try self.genSetMem(
1999919958 .{ .frame = dst_fi },
2000019959 field_off,
20001 Type.c_uint,
19960 .c_uint,
2000219961 .{ .immediate = info.gp_count * 8 },
2000319962 .{},
2000419963 );
......@@ -20007,7 +19966,7 @@ fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
2000719966 try self.genSetMem(
2000819967 .{ .frame = dst_fi },
2000919968 field_off,
20010 Type.c_uint,
19969 .c_uint,
2001119970 .{ .immediate = abi.SysV.c_abi_int_param_regs.len * 8 + info.fp_count * 16 },
2001219971 .{},
2001319972 );
......@@ -20044,7 +20003,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
2004420003 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2004520004 const ty = self.typeOfIndex(inst);
2004620005 const promote_ty = self.promoteVarArg(ty);
20047 const ptr_anyopaque_ty = try pt.singleMutPtrType(Type.anyopaque);
20006 const ptr_anyopaque_ty = try pt.singleMutPtrType(.anyopaque);
2004820007 const unused = self.liveness.isUnused(inst);
2004920008
2005020009 const result: MCValue = switch (abi.resolveCallingConvention(
......@@ -20083,7 +20042,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
2008320042 .integer => {
2008420043 assert(classes.len == 1);
2008520044
20086 try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{});
20045 try self.genSetReg(offset_reg, .c_uint, gp_offset, .{});
2008720046 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
2008820047 abi.SysV.c_abi_int_param_regs.len * 8,
2008920048 ));
......@@ -20104,7 +20063,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
2010420063 .disp = 8,
2010520064 } },
2010620065 });
20107 try self.genCopy(Type.c_uint, gp_offset, .{ .register = offset_reg }, .{});
20066 try self.genCopy(.c_uint, gp_offset, .{ .register = offset_reg }, .{});
2010820067 const done_reloc = try self.asmJmpReloc(undefined);
2010920068
2011020069 self.performReloc(mem_reloc);
......@@ -20131,7 +20090,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
2013120090 .sse => {
2013220091 assert(classes.len == 1);
2013320092
20134 try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{});
20093 try self.genSetReg(offset_reg, .c_uint, fp_offset, .{});
2013520094 try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u(
2013620095 abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16,
2013720096 ));
......@@ -20152,7 +20111,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
2015220111 .disp = 16,
2015320112 } },
2015420113 });
20155 try self.genCopy(Type.c_uint, fp_offset, .{ .register = offset_reg }, .{});
20114 try self.genCopy(.c_uint, fp_offset, .{ .register = offset_reg }, .{});
2015620115 const done_reloc = try self.asmJmpReloc(undefined);
2015720116
2015820117 self.performReloc(mem_reloc);
......@@ -20277,7 +20236,7 @@ fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue {
2027720236 try self.genSetMem(
2027820237 .{ .frame = frame_index },
2027920238 0,
20280 Type.usize,
20239 .usize,
2028120240 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
2028220241 .{},
2028320242 );
......@@ -20318,7 +20277,7 @@ fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !
2031820277 // This immediate is unsigned.
2031920278 const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
2032020279 if (imm >= std.math.maxInt(U)) {
20321 return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) };
20280 return MCValue{ .register = try self.copyToTmpRegister(.usize, mcv) };
2032220281 }
2032320282 },
2032420283 else => {},
......@@ -20764,7 +20723,7 @@ fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Ty
2076420723 if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| {
2076520724 part.* = switch (class) {
2076620725 .integer => if (part_i < parts_len - 1)
20767 Type.u64
20726 .u64
2076820727 else part: {
2076920728 const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
2077020729 const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8));
......@@ -20773,9 +20732,9 @@ fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Ty
2077320732 else => |array_len| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }),
2077420733 };
2077520734 },
20776 .float => Type.f32,
20735 .float => .f32,
2077720736 .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }),
20778 .sse => Type.f64,
20737 .sse => .f64,
2077920738 else => break,
2078020739 };
2078120740 } else {
......@@ -20791,7 +20750,7 @@ fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Ty
2079120750fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
2079220751 const pt = self.pt;
2079320752 const zcu = pt.zcu;
20794 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{
20753 const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
2079520754 .signedness = .unsigned,
2079620755 .bits = @intCast(ty.bitSize(zcu)),
2079720756 };
......@@ -20799,38 +20758,16 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
2079920758 try self.spillEflagsIfOccupied();
2080020759 switch (int_info.signedness) {
2080120760 .signed => {
20802 try self.genShiftBinOpMir(
20803 .{ ._l, .sa },
20804 Type.isize,
20805 .{ .register = reg },
20806 Type.u8,
20807 .{ .immediate = shift },
20808 );
20809 try self.genShiftBinOpMir(
20810 .{ ._r, .sa },
20811 Type.isize,
20812 .{ .register = reg },
20813 Type.u8,
20814 .{ .immediate = shift },
20815 );
20761 try self.genShiftBinOpMir(.{ ._l, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift });
20762 try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, .{ .register = reg }, .u8, .{ .immediate = shift });
2081620763 },
2081720764 .unsigned => {
2081820765 const mask = ~@as(u64, 0) >> shift;
2081920766 if (int_info.bits <= 32) {
20820 try self.genBinOpMir(
20821 .{ ._, .@"and" },
20822 Type.u32,
20823 .{ .register = reg },
20824 .{ .immediate = mask },
20825 );
20767 try self.genBinOpMir(.{ ._, .@"and" }, .u32, .{ .register = reg }, .{ .immediate = mask });
2082620768 } else {
20827 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
20828 try self.genBinOpMir(
20829 .{ ._, .@"and" },
20830 Type.usize,
20831 .{ .register = reg },
20832 .{ .register = tmp_reg },
20833 );
20769 const tmp_reg = try self.copyToTmpRegister(.usize, .{ .immediate = mask });
20770 try self.genBinOpMir(.{ ._, .@"and" }, .usize, .{ .register = reg }, .{ .register = tmp_reg });
2083420771 }
2083520772 },
2083620773 }
......@@ -20911,7 +20848,7 @@ fn floatCompilerRtAbiName(float_bits: u32) u8 {
2091120848fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
2091220849 if (ty.toIntern() == .f16_type and
2091320850 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
20914 self.target.isDarwin()) return Type.u16;
20851 self.target.isDarwin()) return .u16;
2091520852 return ty;
2091620853}
2091720854
......@@ -20943,9 +20880,9 @@ fn promoteInt(self: *CodeGen, ty: Type) Type {
2094320880 else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
2094420881 };
2094520882 for ([_]Type{
20946 Type.c_int, Type.c_uint,
20947 Type.c_long, Type.c_ulong,
20948 Type.c_longlong, Type.c_ulonglong,
20883 .c_int, .c_uint,
20884 .c_long, .c_ulong,
20885 .c_longlong, .c_ulonglong,
2094920886 }) |promote_ty| {
2095020887 const promote_info = promote_ty.intInfo(zcu);
2095120888 if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
......@@ -20958,10 +20895,10 @@ fn promoteInt(self: *CodeGen, ty: Type) Type {
2095820895fn promoteVarArg(self: *CodeGen, ty: Type) Type {
2095920896 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
2096020897 switch (ty.floatBits(self.target.*)) {
20961 32, 64 => return Type.f64,
20898 32, 64 => return .f64,
2096220899 else => |float_bits| {
2096320900 assert(float_bits == self.target.cTypeBitSize(.longdouble));
20964 return Type.c_longdouble;
20901 return .c_longdouble;
2096520902 },
2096620903 }
2096720904}
......@@ -21036,7 +20973,7 @@ const Temp = struct {
2103620973
2103720974 fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
2103820975 const new_temp_index = cg.next_temp_index;
21039 cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20976 cg.temp_type[@intFromEnum(new_temp_index)] = .usize;
2104020977 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
2104120978 switch (temp.tracking(cg).short) {
2104220979 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
......@@ -21141,7 +21078,7 @@ const Temp = struct {
2114121078
2114221079 fn getLimb(temp: Temp, limb_index: u28, cg: *CodeGen) !Temp {
2114321080 const new_temp_index = cg.next_temp_index;
21144 cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
21081 cg.temp_type[@intFromEnum(new_temp_index)] = .usize;
2114521082 switch (temp.tracking(cg).short) {
2114621083 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
2114721084 .immediate => |imm| {
......@@ -21220,7 +21157,7 @@ const Temp = struct {
2122021157 else => {},
2122121158 .register, .lea_symbol, .lea_frame => {
2122221159 assert(limb_index == 0);
21223 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
21160 cg.temp_type[@intFromEnum(temp_index)] = .usize;
2122421161 return;
2122521162 },
2122621163 .register_pair => |regs| {
......@@ -21232,7 +21169,7 @@ const Temp = struct {
2123221169 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
2123321170 cg.register_manager.freeReg(reg);
2123421171 temp_tracking.* = .init(.{ .register = regs[limb_index] });
21235 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
21172 cg.temp_type[@intFromEnum(temp_index)] = .usize;
2123621173 return;
2123721174 },
2123821175 .load_symbol => |sym_off| {
......@@ -21241,7 +21178,7 @@ const Temp = struct {
2124121178 .sym_index = sym_off.sym_index,
2124221179 .off = sym_off.off + @as(u31, limb_index) * 8,
2124321180 } });
21244 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
21181 cg.temp_type[@intFromEnum(temp_index)] = .usize;
2124521182 return;
2124621183 },
2124721184 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
......@@ -21250,7 +21187,7 @@ const Temp = struct {
2125021187 .index = frame_addr.index,
2125121188 .off = frame_addr.off + @as(u31, limb_index) * 8,
2125221189 } });
21253 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
21190 cg.temp_type[@intFromEnum(temp_index)] = .usize;
2125421191 return;
2125521192 },
2125621193 }
......@@ -21339,7 +21276,7 @@ const Temp = struct {
2133921276 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
2134021277 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
2134121278 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
21342 cg.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
21279 cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8;
2134321280 result_temp_index.tracking(cg).* = .init(result);
2134421281 first_temp.* = result_temp;
2134521282 }
......@@ -21405,7 +21342,7 @@ const Temp = struct {
2140521342 cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
2140621343 const new_reg =
2140721344 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
21408 try cg.genSetReg(new_reg, Type.usize, temp_tracking.short.address(), .{});
21345 try cg.genSetReg(new_reg, .usize, temp_tracking.short.address(), .{});
2140921346 new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
2141021347 try temp.die(cg);
2141121348 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
......@@ -21430,8 +21367,8 @@ const Temp = struct {
2143021367 }
2143121368 },
2143221369 .load_frame => |val_frame_addr| {
21433 var val_ptr = try cg.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });
21434 var len = try cg.tempFromValue(Type.usize, .{ .immediate = val_abi_size });
21370 var val_ptr = try cg.tempFromValue(.usize, .{ .lea_frame = val_frame_addr });
21371 var len = try cg.tempFromValue(.usize, .{ .immediate = val_abi_size });
2143521372 try val_ptr.memcpy(ptr, &len, cg);
2143621373 try val_ptr.die(cg);
2143721374 try len.die(cg);
......@@ -21668,11 +21605,12 @@ fn tempFromOperand(
2166821605 inst: Air.Inst.Index,
2166921606 op_index: Liveness.OperandInt,
2167021607 op_ref: Air.Inst.Ref,
21608 ignore_death: bool,
2167121609) !Temp {
2167221610 const zcu = cg.pt.zcu;
2167321611 const ip = &zcu.intern_pool;
2167421612
21675 if (!cg.liveness.operandDies(inst, op_index)) {
21613 if (ignore_death or !cg.liveness.operandDies(inst, op_index)) {
2167621614 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
2167721615 const val = op_ref.toInterned().?;
2167821616 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
......@@ -21693,7 +21631,7 @@ fn tempFromOperand(
2169321631 try cg.genSetMem(
2169421632 .{ .frame = frame_index },
2169521633 0,
21696 Type.usize,
21634 .usize,
2169721635 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
2169821636 .{},
2169921637 );
......@@ -21721,7 +21659,9 @@ fn tempFromOperand(
2172121659inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
2172221660 var temps: [op_refs.len]Temp = undefined;
2172321661 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
21724 temp.* = try cg.tempFromOperand(inst, op_index, op_ref);
21662 temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| {
21663 if (op_ref == op_refs[prev_op_index]) break true;
21664 } else false);
2172521665 }
2172621666 return temps;
2172721667}
......@@ -21776,6 +21716,8 @@ const Pattern = struct {
2177621716 implicit: u8,
2177721717 /// repeat another operand
2177821718 explicit: u8,
21719 /// a condition code
21720 cc,
2177921721 /// any general purpose register
2178021722 gpr,
2178121723 /// any 64-bit mmx register
......@@ -21808,6 +21750,8 @@ const Pattern = struct {
2180821750 ymm_limb,
2180921751 /// a limb stored in memory
2181021752 mem_limb,
21753 /// a limb stored in a condition code
21754 cc_mask_limb,
2181121755 /// a limb stored in a 64-bit mmx register mask
2181221756 mm_mask_limb,
2181321757 /// a limb stored in a 128-bit sse register masuk
......@@ -21829,7 +21773,7 @@ const Pattern = struct {
2182921773
2183021774 fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool {
2183121775 switch (op) {
21832 .implicit, .explicit => unreachable,
21776 .implicit, .explicit, .cc, .cc_mask_limb => unreachable,
2183321777 else => {},
2183421778 // temp is undefined
2183521779 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => return true,
......@@ -21837,7 +21781,7 @@ const Pattern = struct {
2183721781 const temp_ty = temp.typeOf(cg);
2183821782 const abi_size = temp_ty.abiSize(cg.pt.zcu);
2183921783 return switch (op) {
21840 .implicit, .explicit, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable,
21784 .implicit, .explicit, .cc, .cc_mask_limb, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable,
2184121785 .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) {
2184221786 .register => |reg| reg.class() == .general_purpose,
2184321787 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
......@@ -21861,9 +21805,7 @@ const Pattern = struct {
2186121805 },
2186221806 .mem, .mem_limb => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
2186321807 .gpr_limb => abi_size > 8 and switch (temp.tracking(cg).short) {
21864 inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| {
21865 if (reg.class() != .general_purpose) break false;
21866 } else true,
21808 .register, .register_pair, .register_triple, .register_quadruple => true,
2186721809 else => |mcv| mcv.isMemory(),
2186821810 },
2186921811 .mm_limb, .mm_mask_limb => abi_size > 8 and switch (temp.tracking(cg).short) {
......@@ -21897,6 +21839,7 @@ const Pattern = struct {
2189721839 };
2189821840};
2189921841const SelectOptions = struct {
21842 cc: ?Condition = null,
2190021843 invert_result: bool = false,
2190121844};
2190221845fn select(
......@@ -21914,6 +21857,7 @@ fn select(
2191421857 limb_reloc: Mir.Inst.Index,
2191521858 limb_offset: Offset,
2191621859 limb_size: ?u8,
21860 shuffle_temp: ?Temp,
2191721861 mask_limb_temp: ?Temp,
2191821862 mask_limb_offset: Offset,
2191921863 mask_limb_offset_lock: ?RegisterLock,
......@@ -21935,6 +21879,7 @@ fn select(
2193521879 .limb_reloc = undefined,
2193621880 .limb_offset = .unused,
2193721881 .limb_size = null,
21882 .shuffle_temp = null,
2193821883 .mask_limb_temp = null,
2193921884 .mask_limb_offset = .unused,
2194021885 .mask_limb_offset_lock = null,
......@@ -21992,12 +21937,23 @@ fn select(
2199221937 if (!ref_src_op.matches(is_mut, src_temp, cg)) continue :patterns;
2199321938 }
2199421939
21940 for (pattern.ops) |op| switch (op) {
21941 else => {},
21942 .cc_mask_limb,
21943 .mm_mask_limb,
21944 .xmm_mask_limb,
21945 .ymm_mask_limb,
21946 => if (loop.mask_limb_offset_lock == null and !cg.hasFeature(.bmi2)) {
21947 try cg.register_manager.getKnownReg(.rcx, null);
21948 loop.mask_limb_offset_lock = cg.register_manager.lockKnownRegAssumeUnused(.rcx);
21949 },
21950 };
2199521951 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |*src_temp, src_op| {
21996 if (changed: switch (switch (src_op) {
21952 if (switch (switch (src_op) {
2199721953 .implicit, .explicit => |linked_index| pattern.ops[linked_index],
2199821954 else => src_op,
2199921955 }) {
22000 .implicit, .explicit => unreachable,
21956 .implicit, .explicit, .cc, .cc_mask_limb => unreachable,
2200121957 .gpr => try src_temp.toRegClass(.general_purpose, cg),
2200221958 .mm, .mm_mask, .mm_sign_mask => try src_temp.toRegClass(.mmx, cg),
2200321959 .xmm,
......@@ -22015,23 +21971,22 @@ fn select(
2201521971 .ymm_limb,
2201621972 .mem_limb,
2201721973 => switch (src_temp.tracking(cg).short) {
22018 .register_pair, .register_triple, .register_quadruple => false,
21974 .register, .register_pair, .register_triple, .register_quadruple => false,
2201921975 else => try src_temp.toBase(cg),
2202021976 },
22021 .mm_mask_limb,
22022 .xmm_mask_limb,
22023 .ymm_mask_limb,
22024 => if (!cg.hasFeature(.bmi2) and !cg.register_manager.isKnownRegFree(.rcx)) {
22025 try cg.register_manager.getKnownReg(.rcx, null);
22026 loop.mask_limb_offset_lock = cg.register_manager.lockKnownRegAssumeUnused(.rcx);
22027 break :changed true;
22028 } else false,
21977 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => false,
2202921978 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => false,
2203021979 }) break;
2203121980 } else break;
2203221981
21982 const invert_result = opts.invert_result != pattern_set.invert_result;
2203321983 var dst_is_linked: std.StaticBitSet(4) = .initEmpty();
22034 var mir_ops_len = dst_temps.len;
21984 var mir_ops_len: usize = 0;
21985 for (pattern.ops[0..dst_temps.len]) |dst_op| switch (dst_op) {
21986 else => mir_ops_len += 1,
21987 .cc, .cc_mask_limb => {},
21988 };
21989 const dst_mir_ops_len = mir_ops_len;
2203521990 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
2203621991 defer mir_ops_len += @intFromBool(src_op != .implicit);
2203721992 const linked_src_op, const extra_temp = op: switch (src_op) {
......@@ -22046,13 +22001,17 @@ fn select(
2204622001 };
2204722002 const limb_size: u8, const rc = switch (linked_src_op) {
2204822003 else => continue,
22049 .gpr_limb => .{ 8, abi.RegisterClass.gp },
22004 .gpr_limb => .{ @intCast(@divExact(@as(Memory.Size, switch (pattern_set.scalar) {
22005 .any => .qword,
22006 .bool => unreachable,
22007 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
22008 }).bitSize(), 8)), abi.RegisterClass.gp },
2205022009 .mm_limb, .mm_mask_limb => .{ 8, @panic("TODO") },
2205122010 .xmm_limb, .xmm_mask_limb => .{ 16, abi.RegisterClass.sse },
2205222011 .ymm_limb, .ymm_mask_limb => .{ 32, abi.RegisterClass.sse },
2205322012 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => {
2205422013 assert(extra_temp.* == null);
22055 extra_temp.* = try cg.tempAllocReg(Type.usize, switch (linked_src_op) {
22014 extra_temp.* = try cg.tempAllocReg(.noreturn, switch (linked_src_op) {
2205622015 else => unreachable,
2205722016 .umax_gpr => abi.RegisterClass.gp,
2205822017 .umax_mm => @panic("TODO"),
......@@ -22064,10 +22023,18 @@ fn select(
2206422023 assert(loop.limb_size == null or loop.limb_size == limb_size);
2206522024 loop.limb_size = limb_size;
2206622025 loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu);
22067 switch (src_temp.tracking(cg).short) {
22068 .register_pair, .register_triple, .register_quadruple => switch (loop.limb_offset) {
22069 .unused, .temp => loop.limb_offset = .{ .known = 0 },
22070 .known => {},
22026 const src_mcv = src_temp.tracking(cg).short;
22027 switch (src_mcv) {
22028 .register, .register_pair, .register_triple, .register_quadruple => {
22029 switch (loop.limb_offset) {
22030 .unused, .temp => loop.limb_offset = .{ .known = 0 },
22031 .known => {},
22032 }
22033 if (!rc.isSet(RegisterManager.indexOfRegIntoTracked(src_mcv.getRegs()[0]).?)) {
22034 if (loop.shuffle_temp == null) loop.shuffle_temp = try cg.tempAllocReg(.noreturn, abi.RegisterClass.sse);
22035 assert(extra_temp.* == null);
22036 extra_temp.* = try cg.tempAllocReg(.usize, rc);
22037 }
2207122038 },
2207222039 else => {
2207322040 switch (loop.limb_offset) {
......@@ -22075,7 +22042,7 @@ fn select(
2207522042 .known, .temp => {},
2207622043 }
2207722044 assert(extra_temp.* == null);
22078 extra_temp.* = try cg.tempAllocReg(Type.usize, rc);
22045 extra_temp.* = try cg.tempAllocReg(.usize, rc);
2207922046 },
2208022047 }
2208122048 }
......@@ -22090,13 +22057,17 @@ fn select(
2209022057 dst_temp.* = dst_temp: switch (dst_op) {
2209122058 .implicit => unreachable,
2209222059 .explicit => |linked_index| dst_temps[linked_index],
22060 .cc => try cg.tempFromValue(.bool, .{ .eflags = switch (invert_result) {
22061 false => opts.cc.?,
22062 true => opts.cc.?.negate(),
22063 } }),
2209322064 .gpr => try cg.tempAllocReg(dst_ty, abi.RegisterClass.gp),
2209422065 .mm, .mm_mask, .mm_sign_mask => @panic("TODO"),
2209522066 .xmm, .xmm_mask, .xmm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse),
2209622067 .ymm, .ymm_mask, .ymm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse),
2209722068 .mem => @panic("TODO"),
2209822069 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => {
22099 if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(Type.usize, switch (dst_op) {
22070 if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(.noreturn, switch (dst_op) {
2210022071 else => unreachable,
2210122072 .gpr_limb => abi.RegisterClass.gp,
2210222073 .mm_limb => @panic("TODO"),
......@@ -22105,11 +22076,11 @@ fn select(
2210522076 break :dst_temp try cg.tempAlloc(dst_ty);
2210622077 },
2210722078 .mem_limb => try cg.tempAlloc(dst_ty),
22108 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => unreachable, // already checked
22079 .cc_mask_limb, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => unreachable, // already checked
2210922080 .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination
2211022081 };
2211122082 },
22112 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
22083 .cc_mask_limb, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
2211322084 const scalar_size = @divExact(switch (pattern_set.scalar) {
2211422085 .any, .bool => unreachable,
2211522086 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
......@@ -22131,14 +22102,17 @@ fn select(
2213122102 assert(loop.mask_store_bit_size == null or loop.mask_store_bit_size == mask_store_bit_size);
2213222103 loop.mask_store_bit_size = mask_store_bit_size;
2213322104 loop.mask_limb_offset = loop.limb_offset;
22134 if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(Type.usize, switch (dst_op) {
22135 else => unreachable,
22136 .mm_mask_limb => @panic("TODO"),
22137 .xmm_mask_limb, .ymm_mask_limb => abi.RegisterClass.sse,
22138 });
22139 if (loop.mask_limb_temp == null) loop.mask_limb_temp = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
22105 if (loop.mask_limb_temp == null) {
22106 loop.mask_limb_temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
22107 if (dst_op == .cc_mask_limb and mask_store_bit_size > 8) {
22108 // setcc only clears 8 bits
22109 const mask_limb_alias = loop.mask_limb_temp.?.tracking(cg).short.register.to32();
22110 try cg.spillEflagsIfOccupied();
22111 try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_alias, mask_limb_alias);
22112 }
22113 }
2214022114 if (mask_limb_bit_size < mask_store_bit_size and loop.mask_store_reg == null) {
22141 loop.mask_store_temp = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
22115 loop.mask_store_temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
2214222116 loop.mask_store_reg = loop.mask_store_temp.?.tracking(cg).short.register;
2214322117 }
2214422118 dst_temp.* = if (mask_store_bit_size < mask_bit_size)
......@@ -22152,28 +22126,26 @@ fn select(
2215222126 switch (loop.mask_limb_offset) {
2215322127 .unused, .known => {},
2215422128 .temp => |*mask_limb_offset| {
22155 if (cg.hasFeature(.bmi2)) {
22156 assert(loop.mask_limb_offset_lock == null);
22157 mask_limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
22158 } else {
22159 if (loop.mask_limb_offset_lock) |lock| cg.register_manager.unlockReg(lock);
22160 loop.mask_limb_offset_lock = null;
22161 mask_limb_offset.* = try cg.tempFromValue(Type.usize, .{ .register = .rcx });
22162 }
22129 mask_limb_offset.* = if (cg.hasFeature(.bmi2))
22130 try cg.tempAllocReg(.usize, abi.RegisterClass.gp)
22131 else if (loop.mask_limb_offset_lock != null)
22132 try cg.tempFromValue(.usize, .{ .register = .rcx })
22133 else
22134 unreachable;
2216322135 if (loop.mask_store_reg) |mask_store_reg| {
22164 const mask_store_alias = registerAlias(
22165 mask_store_reg,
22166 @min(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable, 4),
22167 );
22136 const mask_store_alias =
22137 if (loop.mask_store_bit_size.? > 8) mask_store_reg.to32() else mask_store_reg.to8();
2216822138 try cg.spillEflagsIfOccupied();
2216922139 try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias);
2217022140 }
2217122141 },
2217222142 }
22143 if (loop.mask_limb_offset_lock) |lock| cg.register_manager.unlockReg(lock);
22144 loop.mask_limb_offset_lock = null;
2217322145 switch (loop.element_offset) {
2217422146 .unused, .known => {},
2217522147 .temp => |*element_offset| {
22176 element_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
22148 element_offset.* = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
2217722149 const element_offset_reg = element_offset.tracking(cg).short.register;
2217822150 try cg.spillEflagsIfOccupied();
2217922151 try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32());
......@@ -22182,7 +22154,7 @@ fn select(
2218222154 }
2218322155 switch (loop.limb_offset) {
2218422156 .unused, .known => {},
22185 .temp => |*limb_offset| limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp),
22157 .temp => |*limb_offset| limb_offset.* = try cg.tempAllocReg(.usize, abi.RegisterClass.gp),
2218622158 }
2218722159 while (true) {
2218822160 switch (loop.mask_limb_offset) {
......@@ -22204,7 +22176,7 @@ fn select(
2220422176 }
2220522177 while (true) {
2220622178 var mir_ops: [4]Operand = @splat(.none);
22207 mir_ops_len = dst_temps.len;
22179 mir_ops_len = dst_mir_ops_len;
2220822180 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
2220922181 defer mir_ops_len += @intFromBool(src_op != .implicit);
2221022182 const mir_op, const linked_src_op, const extra_temp = switch (src_op) {
......@@ -22217,35 +22189,146 @@ fn select(
2221722189 // src_temp is undefined
2221822190 .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => extra_temp.?,
2221922191 }.tracking(cg).short;
22220 copy_limb: switch (src_mcv) {
22221 .register_pair, .register_triple, .register_quadruple => {},
22222 else => try cg.asmRegisterMemory(
22223 switch (linked_src_op) {
22224 else => break :copy_limb,
22225 .gpr_limb => .{ ._, .mov },
22226 .mm_limb, .mm_mask_limb => .{ ._q, .mov },
22227 .xmm_limb,
22228 .ymm_limb,
22229 .xmm_mask_limb,
22230 .ymm_mask_limb,
22231 => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
22192 switch (linked_src_op) {
22193 else => {},
22194 .gpr_limb,
22195 .mm_limb,
22196 .xmm_limb,
22197 .ymm_limb,
22198 .mm_mask_limb,
22199 .xmm_mask_limb,
22200 .ymm_mask_limb,
22201 => if (extra_temp) |limb_temp| switch (src_mcv) {
22202 .register, .register_pair, .register_triple, .register_quadruple => {
22203 const limb_reg = registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?);
22204 const src_regs = src_mcv.getRegs();
22205 const src_reg_size: u32 = @intCast(switch (src_mcv) {
22206 .register => src_temp.typeOf(cg).abiSize(cg.pt.zcu),
22207 else => @divExact(src_regs[0].bitSize(), 8),
22208 });
22209 const src_reg = src_regs[loop.limb_offset.known / src_reg_size];
22210 assert(src_mcv == .register or src_reg.bitSize() == 8 * src_reg_size);
22211 switch (src_reg.class()) {
22212 else => unreachable,
22213 .general_purpose => try cg.asmRegisterRegister(
22214 .{ ._, .mov },
22215 limb_reg,
22216 registerAlias(src_reg, src_reg_size),
22217 ),
22218 .sse => {
22219 assert(src_reg_size == 16);
22220 const limb_alias_size = @max(loop.limb_size.?, 4);
22221 const limb_alias = registerAlias(limb_reg, limb_alias_size);
22222 const src_reg_offset = loop.limb_offset.known % src_reg_size;
22223 switch (limb_reg_offset: {
22224 extr: {
22225 const limb_size = if (cg.hasFeature(.sse4_1)) loop.limb_size.? else 2;
22226 if (loop.limb_size.? > limb_size) break :extr;
22227 const limb_offset = src_reg_offset / limb_size;
22228 if (limb_offset == 0) break :extr;
22229 try cg.asmRegisterRegisterImmediate(.{ switch (limb_size) {
22230 else => unreachable,
22231 1 => if (cg.hasFeature(.avx)) .vp_b else .p_b,
22232 2 => if (cg.hasFeature(.avx)) .vp_w else .p_w,
22233 4 => if (cg.hasFeature(.avx)) .vp_d else .p_d,
22234 8 => if (cg.hasFeature(.avx)) .vp_q else .p_q,
22235 }, .extr }, limb_alias, src_reg.to128(), .u(limb_offset));
22236 break :limb_reg_offset src_reg_offset % limb_size;
22237 }
22238 try cg.asmRegisterRegister(
22239 .{ switch (limb_alias_size) {
22240 else => unreachable,
22241 4 => ._d,
22242 8 => ._q,
22243 }, .mov },
22244 limb_alias,
22245 if (src_reg_offset < limb_alias_size) src_reg.to128() else shuffle_reg: {
22246 const shuffle_reg = loop.shuffle_temp.?.tracking(cg).short.register.to128();
22247 const mir_fixes: Mir.Inst.Fixes = if (cg.hasFeature(.sse2))
22248 if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).isRuntimeFloat()) switch (limb_alias_size) {
22249 else => unreachable,
22250 4 => if (cg.hasFeature(.avx)) .v_ps else ._ps,
22251 8 => if (cg.hasFeature(.avx)) .v_pd else ._pd,
22252 } else if (cg.hasFeature(.avx)) .vp_d else .p_d
22253 else
22254 ._ps;
22255 try cg.asmRegisterRegisterImmediate(
22256 .{ mir_fixes, .shuf },
22257 shuffle_reg,
22258 src_reg: switch (mir_fixes) {
22259 else => unreachable,
22260 ._ps, ._pd => {
22261 try cg.asmRegisterRegister(.{ mir_fixes, .mova }, shuffle_reg, src_reg.to128());
22262 break :src_reg shuffle_reg;
22263 },
22264 .p_d => src_reg.to128(),
22265 },
22266 .u(switch (mir_fixes) {
22267 else => unreachable,
22268 .v_ps, ._ps, .vp_d, .p_d => switch (limb_alias_size) {
22269 else => unreachable,
22270 4 => switch (src_reg_offset) {
22271 else => unreachable,
22272 4...7 => 0b01_01_01_01,
22273 8...11 => 0b10_10_10_10,
22274 12...15 => 0b11_11_11_11,
22275 },
22276 8 => switch (src_reg_offset) {
22277 else => unreachable,
22278 8...15 => 0b11_10_11_10,
22279 },
22280 },
22281 .v_pd, ._pd => switch (limb_alias_size) {
22282 else => unreachable,
22283 8 => switch (src_reg_offset) {
22284 else => unreachable,
22285 8...15 => 0b1_1,
22286 },
22287 },
22288 }),
22289 );
22290 break :shuffle_reg shuffle_reg;
22291 },
22292 );
22293 break :limb_reg_offset src_reg_offset % limb_alias_size;
22294 }) {
22295 0 => {},
22296 else => |limb_reg_offset| {
22297 try cg.spillEflagsIfOccupied();
22298 try cg.asmRegisterImmediate(.{ ._r, .sh }, limb_alias, .u(limb_reg_offset * 8));
22299 },
22300 }
22301 },
22302 }
2223222303 },
22233 registerAlias(extra_temp.?.tracking(cg).short.register, loop.limb_size.?),
22234 try src_mcv.mem(cg, switch (loop.limb_offset) {
22235 .unused => unreachable,
22236 .known => |limb_offset| .{
22237 .size = .fromSize(loop.limb_size.?),
22238 .disp = limb_offset,
22239 },
22240 .temp => |limb_offset| .{
22241 .size = .fromSize(loop.limb_size.?),
22242 .index = limb_offset.tracking(cg).short.register.to64(),
22304 else => try cg.asmRegisterMemory(
22305 switch (linked_src_op) {
22306 else => unreachable,
22307 .gpr_limb => .{ ._, .mov },
22308 .mm_limb, .mm_mask_limb => .{ ._q, .mov },
22309 .xmm_limb,
22310 .ymm_limb,
22311 .xmm_mask_limb,
22312 .ymm_mask_limb,
22313 => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
2224322314 },
22244 }),
22245 ),
22315 registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?),
22316 try src_mcv.mem(cg, switch (loop.limb_offset) {
22317 .unused => unreachable,
22318 .known => |limb_offset| .{
22319 .size = .fromSize(loop.limb_size.?),
22320 .disp = limb_offset,
22321 },
22322 .temp => |limb_offset| .{
22323 .size = .fromSize(loop.limb_size.?),
22324 .index = limb_offset.tracking(cg).short.register.to64(),
22325 },
22326 }),
22327 ),
22328 },
2224622329 }
2224722330 mir_op.* = switch (linked_src_op) {
22248 .implicit, .explicit => unreachable,
22331 .implicit, .explicit, .cc, .cc_mask_limb => unreachable,
2224922332 .gpr => .{ .reg = registerAlias(
2225022333 src_mcv.register,
2225122334 @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)),
......@@ -22262,25 +22345,12 @@ fn select(
2226222345 .mm_mask_limb,
2226322346 .xmm_mask_limb,
2226422347 .ymm_mask_limb,
22265 => switch (src_mcv) {
22266 inline .register_pair,
22267 .register_triple,
22268 .register_quadruple,
22269 => |src_regs| switch (loop.limb_offset) {
22270 .unused => unreachable,
22271 .known => |limb_offset| .{ .reg = registerAlias(
22272 src_regs[@divExact(limb_offset, loop.limb_size.?)],
22273 loop.limb_size.?,
22274 ) },
22275 .temp => unreachable,
22276 },
22277 else => .{ .reg = registerAlias(
22278 extra_temp.?.tracking(cg).short.register,
22279 loop.limb_size.?,
22280 ) },
22281 },
22348 => .{ .reg = registerAlias(if (extra_temp) |limb_temp|
22349 limb_temp.tracking(cg).short.register
22350 else
22351 src_mcv.getRegs()[@divExact(loop.limb_offset.known, loop.limb_size.?)], loop.limb_size.?) },
2228222352 .mem_limb => .{ .mem = switch (src_mcv) {
22283 .register_pair, .register_triple, .register_quadruple => unreachable,
22353 .register, .register_pair, .register_triple, .register_quadruple => unreachable,
2228422354 else => switch (loop.limb_offset) {
2228522355 .unused => unreachable,
2228622356 .known => |limb_offset| try src_mcv.mem(cg, .{
......@@ -22316,15 +22386,15 @@ fn select(
2231622386 }
2231722387 }
2231822388 for (
22319 mir_ops[0..dst_temps.len],
22320 pattern.ops[0..dst_temps.len],
22321 dst_temps,
22322 dst_tys,
22323 extra_temps[0..dst_temps.len],
22389 mir_ops[0..dst_mir_ops_len],
22390 pattern.ops[0..dst_mir_ops_len],
22391 dst_temps[0..dst_mir_ops_len],
22392 dst_tys[0..dst_mir_ops_len],
22393 extra_temps[0..dst_mir_ops_len],
2232422394 ) |*mir_op, dst_op, dst_temp, dst_ty, extra_temp| {
2232522395 if (mir_op.* != .none) continue;
2232622396 mir_op.* = switch (dst_op) {
22327 .implicit => unreachable,
22397 .implicit, .cc, .cc_mask_limb => unreachable,
2232822398 .explicit => |linked_index| mir_ops[linked_index],
2232922399 .gpr => .{ .reg = registerAlias(
2233022400 dst_temp.tracking(cg).short.register,
......@@ -22334,7 +22404,14 @@ fn select(
2233422404 .xmm, .xmm_mask, .xmm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to128() },
2233522405 .ymm, .ymm_mask, .ymm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to256() },
2233622406 .mem => @panic("TODO"),
22337 .gpr_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to64() },
22407 .gpr_limb => .{ .reg = registerAlias(
22408 extra_temp.?.tracking(cg).short.register,
22409 @intCast(@divExact(@as(Memory.Size, switch (pattern_set.scalar) {
22410 .any => .qword,
22411 .bool => unreachable,
22412 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
22413 }).bitSize(), 8)),
22414 ) },
2233822415 .mm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register },
2233922416 .xmm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() },
2234022417 .ymm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() },
......@@ -22376,7 +22453,6 @@ fn select(
2237622453 },
2237722454 else => |e| return e,
2237822455 };
22379 const invert_result = opts.invert_result != pattern_set.invert_result;
2238022456 for (
2238122457 extra_temps[0..dst_temps.len],
2238222458 pattern.ops[0..dst_temps.len],
......@@ -22410,21 +22486,15 @@ fn select(
2241022486 .any, .bool => unreachable,
2241122487 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
2241222488 }, cg),
22413 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => if (extra_temp) |limb_temp|
22414 switch (dst_temp.tracking(cg).short) {
22415 inline .register_pair,
22416 .register_triple,
22417 .register_quadruple,
22418 => |dst_regs| switch (loop.limb_offset) {
22419 .unused => unreachable,
22420 .known => |limb_offset| try cg.asmRegisterRegister(
22421 .{ ._, .mov },
22422 dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(),
22423 limb_temp.tracking(cg).short.register.to64(),
22424 ),
22425 .temp => unreachable,
22426 },
22427 else => |dst_mcv| try cg.asmMemoryRegister(
22489 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => if (extra_temp) |limb_temp| {
22490 const dst_mcv = dst_temp.tracking(cg).short;
22491 switch (dst_mcv) {
22492 .register_pair, .register_triple, .register_quadruple => try cg.asmRegisterRegister(
22493 .{ ._, .mov },
22494 dst_mcv.getRegs()[@divExact(loop.limb_offset.known, loop.limb_size.?)].to64(),
22495 limb_temp.tracking(cg).short.register.to64(),
22496 ),
22497 else => try cg.asmMemoryRegister(
2242822498 switch (dst_op) {
2242922499 else => unreachable,
2243022500 .gpr_limb => .{ ._, .mov },
......@@ -22444,57 +22514,66 @@ fn select(
2244422514 }),
2244522515 registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?),
2244622516 ),
22447 },
22448 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
22517 }
22518 },
22519 .cc_mask_limb, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
2244922520 const scalar_size = switch (pattern_set.scalar) {
2245022521 .any, .bool => unreachable,
2245122522 .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size,
2245222523 };
22453 switch (scalar_size) {
22454 else => {},
22455 .word => if (cg.hasFeature(.avx)) try cg.asmRegisterRegisterRegister(
22456 .{ .vp_b, .ackssw },
22457 mir_op.reg,
22458 mir_op.reg,
22459 mir_op.reg,
22460 ) else try cg.asmRegisterRegister(
22461 .{ .p_b, .ackssw },
22462 mir_op.reg,
22463 mir_op.reg,
22464 ),
22465 }
2246622524 const mask_store_size: u4 =
2246722525 @intCast(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable);
22468 const mask_limb_reg = registerAlias(
22469 loop.mask_limb_temp.?.tracking(cg).short.register,
22470 mask_store_size,
22471 );
22472 try cg.asmRegisterRegister(switch (scalar_size) {
22526 const known_shl_count = if (loop.mask_store_reg) |_| switch (loop.mask_limb_offset) {
22527 .unused => unreachable,
22528 .known => |mask_limb_offset| mask_limb_offset & (loop.mask_store_bit_size.? - 1),
22529 .temp => null,
22530 } else null;
22531 const mask_limb_reg = registerAlias(if (known_shl_count != 0)
22532 loop.mask_limb_temp.?.tracking(cg).short.register
22533 else
22534 loop.mask_store_reg.?, mask_store_size);
22535 switch (dst_op) {
2247322536 else => unreachable,
22474 .byte, .word => .{ if (cg.hasFeature(.avx)) .vp_b else .p_b, .movmsk },
22475 .dword => .{ if (cg.hasFeature(.avx)) .v_ps else ._ps, .movmsk },
22476 .qword => .{ if (cg.hasFeature(.avx)) .v_pd else ._pd, .movmsk },
22477 }, mask_limb_reg.to32(), mir_op.reg);
22478 if (invert_result) if (loop.mask_store_reg) |_| {
22479 try cg.spillEflagsIfOccupied();
22480 try cg.asmRegisterImmediate(
22481 .{ ._, .xor },
22482 registerAlias(mask_limb_reg, @min(mask_store_size, 4)),
22483 .u((@as(u32, 1) << @intCast(loop.mask_limb_bit_size.?)) - 1),
22484 );
22485 } else try cg.asmRegister(.{ ._, .not }, mask_limb_reg);
22537 .cc_mask_limb => try cg.asmSetccRegister(switch (invert_result) {
22538 false => opts.cc.?,
22539 true => opts.cc.?.negate(),
22540 }, mask_limb_reg.to8()),
22541 .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => {
22542 if (scalar_size == .word) if (cg.hasFeature(.avx)) try cg.asmRegisterRegisterRegister(
22543 .{ .vp_b, .ackssw },
22544 mir_op.reg,
22545 mir_op.reg,
22546 mir_op.reg,
22547 ) else try cg.asmRegisterRegister(
22548 .{ .p_b, .ackssw },
22549 mir_op.reg,
22550 mir_op.reg,
22551 );
22552 try cg.asmRegisterRegister(switch (scalar_size) {
22553 else => unreachable,
22554 .byte, .word => .{ if (cg.hasFeature(.avx)) .vp_b else .p_b, .movmsk },
22555 .dword => .{ if (cg.hasFeature(.avx)) .v_ps else ._ps, .movmsk },
22556 .qword => .{ if (cg.hasFeature(.avx)) .v_pd else ._pd, .movmsk },
22557 }, mask_limb_reg.to32(), mir_op.reg);
22558 if (invert_result) if (loop.mask_store_reg) |_| {
22559 try cg.spillEflagsIfOccupied();
22560 try cg.asmRegisterImmediate(
22561 .{ ._, .xor },
22562 registerAlias(mask_limb_reg, @min(mask_store_size, 4)),
22563 .u((@as(u32, 1) << @intCast(loop.mask_limb_bit_size.?)) - 1),
22564 );
22565 } else try cg.asmRegister(.{ ._, .not }, mask_limb_reg);
22566 },
22567 }
2248622568 if (loop.mask_store_reg) |mask_store_reg| {
2248722569 const mask_store_alias = registerAlias(mask_store_reg, mask_store_size);
2248822570 switch (loop.mask_limb_offset) {
2248922571 .unused => unreachable,
22490 .known => |mask_limb_offset| switch (mask_limb_offset & (loop.mask_store_bit_size.? - 1)) {
22491 0 => try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_alias, mask_limb_reg),
22492 else => |shl_count| {
22493 try cg.spillEflagsIfOccupied();
22494 try cg.asmRegisterImmediate(.{ ._l, .sh }, mask_limb_reg, .u(shl_count));
22495 try cg.spillEflagsIfOccupied();
22496 try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg);
22497 },
22572 .known => if (known_shl_count.? != 0) {
22573 try cg.spillEflagsIfOccupied();
22574 try cg.asmRegisterImmediate(.{ ._l, .sh }, mask_limb_reg, .u(known_shl_count.?));
22575 try cg.spillEflagsIfOccupied();
22576 try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg);
2249822577 },
2249922578 .temp => |mask_limb_offset| {
2250022579 if (cg.hasFeature(.bmi2)) {
......@@ -22524,23 +22603,23 @@ fn select(
2252422603 .unused => unreachable,
2252522604 .known => |*mask_limb_offset| {
2252622605 mask_limb_offset.* += loop.mask_limb_bit_size.?;
22527 if (mask_limb_offset.* & (loop.mask_store_bit_size.? - 1) == 0) {
22528 switch (dst_mcv) {
22529 .register => {},
22530 else => try cg.asmMemoryRegister(
22606 if (mask_limb_offset.* & (loop.mask_store_bit_size.? - 1) == 0) switch (dst_mcv) {
22607 .register => {},
22608 else => {
22609 try cg.asmMemoryRegister(
2253122610 .{ ._, .mov },
2253222611 try dst_mcv.mem(cg, .{
2253322612 .size = .fromSize(mask_store_size),
2253422613 .disp = @divExact(mask_limb_offset.*, 8) - mask_store_size,
2253522614 }),
2253622615 registerAlias(loop.mask_store_reg orelse mask_limb_reg, mask_store_size),
22537 ),
22538 }
22539 if (loop.mask_store_reg) |mask_store_reg| {
22540 const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4));
22541 try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias);
22542 }
22543 }
22616 );
22617 if (loop.mask_store_reg) |mask_store_reg| {
22618 const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4));
22619 try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias);
22620 }
22621 },
22622 };
2254422623 },
2254522624 .temp => |mask_limb_offset| {
2254622625 const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32();
......@@ -22641,6 +22720,7 @@ fn select(
2264122720 },
2264222721 }
2264322722 }
22723 if (loop.shuffle_temp) |shuffle_temp| try shuffle_temp.die(cg);
2264422724 if (loop.mask_limb_temp) |mask_limb_temp| try mask_limb_temp.die(cg);
2264522725 if (loop.mask_store_temp) |mask_store_temp| try mask_store_temp.die(cg);
2264622726 switch (loop.mask_limb_offset) {
src/arch/x86_64/encoder.zig+2-2
......@@ -677,11 +677,11 @@ pub const Instruction = struct {
677677 else => unreachable,
678678 },
679679 .frame => if (@TypeOf(encoder).options.allow_frame_locs) {
680 try encoder.modRm_indirectDisp32(operand_enc, undefined);
680 try encoder.modRm_indirectDisp32(operand_enc, 0);
681681 try encoder.disp32(undefined);
682682 } else return error.CannotEncode,
683683 .reloc => if (@TypeOf(encoder).options.allow_symbols) {
684 try encoder.modRm_indirectDisp32(operand_enc, undefined);
684 try encoder.modRm_indirectDisp32(operand_enc, 0);
685685 try encoder.disp32(undefined);
686686 } else return error.CannotEncode,
687687 },
src/codegen/c/Type.zig+5-5
......@@ -1312,10 +1312,10 @@ pub const Pool = struct {
13121312 },
13131313 else => {
13141314 const target = &mod.resolved_target.result;
1315 const abi_align = Type.intAbiAlignment(int_info.bits, target.*, false);
1315 const abi_align = Type.intAbiAlignment(int_info.bits, target.*);
13161316 const abi_align_bytes = abi_align.toByteUnits().?;
13171317 const array_ctype = try pool.getArray(allocator, .{
1318 .len = @divExact(Type.intAbiSize(int_info.bits, target.*, false), abi_align_bytes),
1318 .len = @divExact(Type.intAbiSize(int_info.bits, target.*), abi_align_bytes),
13191319 .elem_ctype = try pool.fromIntInfo(allocator, .{
13201320 .signedness = .unsigned,
13211321 .bits = @intCast(abi_align_bytes * 8),
......@@ -1429,7 +1429,7 @@ pub const Pool = struct {
14291429 .name = .{ .index = .len },
14301430 .ctype = CType.usize,
14311431 .alignas = AlignAs.fromAbiAlignment(
1432 Type.intAbiAlignment(target.ptrBitWidth(), target.*, false),
1432 Type.intAbiAlignment(target.ptrBitWidth(), target.*),
14331433 ),
14341434 },
14351435 };
......@@ -1524,7 +1524,7 @@ pub const Pool = struct {
15241524 .name = .{ .index = .len },
15251525 .ctype = CType.usize,
15261526 .alignas = AlignAs.fromAbiAlignment(
1527 Type.intAbiAlignment(target.ptrBitWidth(), target.*, false),
1527 Type.intAbiAlignment(target.ptrBitWidth(), target.*),
15281528 ),
15291529 },
15301530 };
......@@ -1644,7 +1644,7 @@ pub const Pool = struct {
16441644 .name = .{ .index = .@"error" },
16451645 .ctype = error_set_ctype,
16461646 .alignas = AlignAs.fromAbiAlignment(
1647 Type.intAbiAlignment(error_set_bits, target.*, false),
1647 Type.intAbiAlignment(error_set_bits, target.*),
16481648 ),
16491649 },
16501650 .{
src/codegen/llvm.zig+1-1
......@@ -581,7 +581,7 @@ const DataLayoutBuilder = struct {
581581 switch (kind) {
582582 .integer => {
583583 if (self.target.ptrBitWidth() <= 16 and size >= 128) return;
584 abi = @min(abi, Type.maxIntAlignment(self.target, true) * 8);
584 abi = @min(abi, Type.maxIntAlignment(self.target) * 8);
585585 switch (self.target.cpu.arch) {
586586 .aarch64,
587587 .aarch64_be,
src/link/C.zig+1-1
......@@ -396,7 +396,7 @@ fn abiDefines(self: *C, target: std.Target) !std.ArrayList(u8) {
396396 else => {},
397397 }
398398 try writer.print("#define ZIG_TARGET_MAX_INT_ALIGNMENT {d}\n", .{
399 Type.maxIntAlignment(target, false),
399 Type.maxIntAlignment(target),
400400 });
401401 return defines;
402402}
test/behavior/align.zig+9-23
......@@ -144,31 +144,17 @@ test "alignment and size of structs with 128-bit fields" {
144144 },
145145 },
146146
147 .x86_64 => switch (builtin.zig_backend) {
148 .stage2_x86_64 => .{
149 .a_align = 8,
150 .a_size = 16,
151
152 .b_align = 16,
153 .b_size = 32,
154
155 .u128_align = 8,
156 .u128_size = 16,
157 .u129_align = 8,
158 .u129_size = 24,
159 },
160 else => .{
161 .a_align = 16,
162 .a_size = 16,
147 .x86_64 => .{
148 .a_align = 16,
149 .a_size = 16,
163150
164 .b_align = 16,
165 .b_size = 32,
151 .b_align = 16,
152 .b_size = 32,
166153
167 .u128_align = 16,
168 .u128_size = 16,
169 .u129_align = 16,
170 .u129_size = 32,
171 },
154 .u128_align = 16,
155 .u128_size = 16,
156 .u129_align = 16,
157 .u129_size = 32,
172158 },
173159
174160 .x86,
test/behavior/x86_64/math.zig+60-3
......@@ -31,7 +31,37 @@ fn testBinary(comptime op: anytype) !void {
3131 try testType(u32, 0x80d7a2c6, 0xbff6a402);
3232 try testType(u64, 0x71138bc6b4a38898, 0x1bc4043de9438c7b);
3333 try testType(u128, 0xe05fc132ef2cd8affee00a907f0a851f, 0x29f912a72cfc6a7c6973426a9636da9a);
34 try testType(
35 u256,
36 0xb7935f5c2f3b1ae7a422c0a7c446884294b7d5370bada307d2fe5a4c4284a999,
37 0x310e6e196ba4f143b8d285ca6addf7f3bb3344224aff221b27607a31e148be08,
38 );
39 try testType(
40 u512,
41 0xe5b1fedca3c77db765e517aabd05ffc524a3a8aff1784bbf67c45b894447ede32b65b9940e78173c591e56e078932d465f235aece7ad47b7f229df7ba8f12295,
42 0x8b4bb7c2969e3b121cc1082c442f8b4330f0a50058438fed56447175bb10178607ecfe425cb54dacc25ef26810f3e04681de1844f1aa8d029aca75d658634806,
43 );
3444
45 try testType(@Vector(1, u8), .{
46 0x1f,
47 }, .{
48 0x06,
49 });
50 try testType(@Vector(2, u8), .{
51 0x80, 0x63,
52 }, .{
53 0xe4, 0x28,
54 });
55 try testType(@Vector(4, u8), .{
56 0x83, 0x9e, 0x1e, 0xc1,
57 }, .{
58 0xf0, 0x5c, 0x46, 0x85,
59 });
60 try testType(@Vector(8, u8), .{
61 0x1e, 0x4d, 0x9d, 0x2a, 0x4c, 0x74, 0x0a, 0x83,
62 }, .{
63 0x28, 0x60, 0xa9, 0xb5, 0xd9, 0xa6, 0xf1, 0xb6,
64 });
3565 try testType(@Vector(16, u8), .{
3666 0xea, 0x80, 0xbb, 0xe8, 0x74, 0x81, 0xc8, 0x66, 0x7b, 0x41, 0x90, 0xcb, 0x30, 0x70, 0x4b, 0x0f,
3767 }, .{
......@@ -75,6 +105,21 @@ fn testBinary(comptime op: anytype) !void {
75105 0x56, 0x4f, 0xf1, 0xaa, 0x0a, 0x0f, 0xdb, 0x1b, 0xc8, 0x45, 0x9b, 0x12, 0xb4, 0x1a, 0xe4, 0xa3,
76106 });
77107
108 try testType(@Vector(1, u16), .{
109 0x9d6f,
110 }, .{
111 0x44b1,
112 });
113 try testType(@Vector(2, u16), .{
114 0xa0fa, 0xc365,
115 }, .{
116 0xe736, 0xc394,
117 });
118 try testType(@Vector(4, u16), .{
119 0x9608, 0xa558, 0x161b, 0x206f,
120 }, .{
121 0x3088, 0xf25c, 0x7837, 0x9b3f,
122 });
78123 try testType(@Vector(8, u16), .{
79124 0xcf61, 0xb121, 0x3cf1, 0x3e9f, 0x43a7, 0x8d69, 0x96f5, 0xc11e,
80125 }, .{
......@@ -118,6 +163,16 @@ fn testBinary(comptime op: anytype) !void {
118163 0x2c02, 0xff5b, 0x19ca, 0xbbf5, 0x870e, 0xc9ca, 0x47bb, 0xcfcc,
119164 });
120165
166 try testType(@Vector(1, u32), .{
167 0x1d0d9cc4,
168 }, .{
169 0xce2d0ab6,
170 });
171 try testType(@Vector(2, u32), .{
172 0x5ab78c03, 0xd21bb513,
173 }, .{
174 0x8a6664eb, 0x79eac37d,
175 });
121176 try testType(@Vector(4, u32), .{
122177 0x234d576e, 0x4151cc9c, 0x39f558e4, 0xba935a32,
123178 }, .{
......@@ -161,9 +216,11 @@ fn testBinary(comptime op: anytype) !void {
161216 0xf080e943, 0xc8718d14, 0x3f920382, 0x18d101b5,
162217 });
163218
164 // TODO: implement fallback for pcmpeqq
165 if (!comptime @import("std").Target.x86.featureSetHas(@import("builtin").cpu.features, .sse4_1)) return;
166
219 try testType(@Vector(1, u64), .{
220 0x333f593bf9d08546,
221 }, .{
222 0x6918bd767e730778,
223 });
167224 try testType(@Vector(2, u64), .{
168225 0x4cd89a317b03d430, 0x28998f61842f63a9,
169226 }, .{