| ... | @@ -1692,7 +1692,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1692,7 +1692,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { |
| 1692 | .unsigned => .int_unsigned, | 1692 | .unsigned => .int_unsigned, |
| 1693 | } }, .data = switch (tag) { | 1693 | } }, .data = switch (tag) { |
| 1694 | else => unreachable, | 1694 | else => unreachable, |
| 1695 | .mul, .mulwrap => std.math.max3( | 1695 | .mul, .mulwrap => math.max3( |
| 1696 | self.activeIntBits(bin_op.lhs), | 1696 | self.activeIntBits(bin_op.lhs), |
| 1697 | self.activeIntBits(bin_op.rhs), | 1697 | self.activeIntBits(bin_op.rhs), |
| 1698 | dst_info.bits / 2, | 1698 | dst_info.bits / 2, |
| ... | @@ -1745,7 +1745,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1745,7 +1745,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { |
| 1745 | break :cc .o; | 1745 | break :cc .o; |
| 1746 | } else cc: { | 1746 | } else cc: { |
| 1747 | try self.genSetReg(ty, limit_reg, .{ | 1747 | try self.genSetReg(ty, limit_reg, .{ |
| 1748 | .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits), | 1748 | .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits), |
| 1749 | }); | 1749 | }); |
| 1750 | break :cc .c; | 1750 | break :cc .c; |
| 1751 | }; | 1751 | }; |
| ... | @@ -1852,7 +1852,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1852,7 +1852,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { |
| 1852 | break :cc .o; | 1852 | break :cc .o; |
| 1853 | } else cc: { | 1853 | } else cc: { |
| 1854 | try self.genSetReg(ty, limit_reg, .{ | 1854 | try self.genSetReg(ty, limit_reg, .{ |
| 1855 | .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits), | 1855 | .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits), |
| 1856 | }); | 1856 | }); |
| 1857 | break :cc .c; | 1857 | break :cc .c; |
| 1858 | }; | 1858 | }; |
| ... | @@ -2069,7 +2069,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2069,7 +2069,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2069 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { | 2069 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { |
| 2070 | .signed => .int_signed, | 2070 | .signed => .int_signed, |
| 2071 | .unsigned => .int_unsigned, | 2071 | .unsigned => .int_unsigned, |
| 2072 | } }, .data = std.math.max3( | 2072 | } }, .data = math.max3( |
| 2073 | self.activeIntBits(bin_op.lhs), | 2073 | self.activeIntBits(bin_op.lhs), |
| 2074 | self.activeIntBits(bin_op.rhs), | 2074 | self.activeIntBits(bin_op.rhs), |
| 2075 | dst_info.bits / 2, | 2075 | dst_info.bits / 2, |
| ... | @@ -3569,6 +3569,62 @@ fn reuseOperand( | ... | @@ -3569,6 +3569,62 @@ fn reuseOperand( |
| 3569 | return true; | 3569 | return true; |
| 3570 | } | 3570 | } |
| 3571 | | 3571 | |
| | 3572 | fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void { |
| | 3573 | const ptr_info = ptr_ty.ptrInfo().data; |
| | 3574 | |
| | 3575 | const val_ty = ptr_info.pointee_type; |
| | 3576 | const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*)); |
| | 3577 | const limb_abi_size = @min(val_abi_size, 8); |
| | 3578 | const limb_abi_bits = limb_abi_size * 8; |
| | 3579 | const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size); |
| | 3580 | const val_bit_off = ptr_info.bit_offset % limb_abi_bits; |
| | 3581 | const val_extra_bits = self.regExtraBits(val_ty); |
| | 3582 | |
| | 3583 | if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{ |
| | 3584 | val_ty.fmt(self.bin_file.options.module.?), |
| | 3585 | }); |
| | 3586 | |
| | 3587 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); |
| | 3588 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); |
| | 3589 | defer self.register_manager.unlockReg(ptr_lock); |
| | 3590 | |
| | 3591 | const dst_reg = switch (dst_mcv) { |
| | 3592 | .register => |reg| reg, |
| | 3593 | else => try self.register_manager.allocReg(null, gp), |
| | 3594 | }; |
| | 3595 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| | 3596 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| | 3597 | |
| | 3598 | const load_abi_size = |
| | 3599 | if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2; |
| | 3600 | if (load_abi_size <= 8) { |
| | 3601 | const load_reg = registerAlias(dst_reg, load_abi_size); |
| | 3602 | try self.asmRegisterMemory(.mov, load_reg, Memory.sib( |
| | 3603 | Memory.PtrSize.fromSize(load_abi_size), |
| | 3604 | .{ .base = ptr_reg, .disp = val_byte_off }, |
| | 3605 | )); |
| | 3606 | try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off)); |
| | 3607 | } else { |
| | 3608 | const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size); |
| | 3609 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| | 3610 | defer self.register_manager.unlockReg(tmp_lock); |
| | 3611 | |
| | 3612 | const dst_alias = registerAlias(dst_reg, val_abi_size); |
| | 3613 | try self.asmRegisterMemory(.mov, dst_alias, Memory.sib( |
| | 3614 | Memory.PtrSize.fromSize(val_abi_size), |
| | 3615 | .{ .base = ptr_reg, .disp = val_byte_off }, |
| | 3616 | )); |
| | 3617 | try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib( |
| | 3618 | Memory.PtrSize.fromSize(val_abi_size), |
| | 3619 | .{ .base = ptr_reg, .disp = val_byte_off + 1 }, |
| | 3620 | )); |
| | 3621 | try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off)); |
| | 3622 | } |
| | 3623 | |
| | 3624 | if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg); |
| | 3625 | try self.setRegOrMem(val_ty, dst_mcv, .{ .register = dst_reg }); |
| | 3626 | } |
| | 3627 | |
| 3572 | fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { | 3628 | fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { |
| 3573 | const elem_ty = ptr_ty.elemType(); | 3629 | const elem_ty = ptr_ty.elemType(); |
| 3574 | const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*)); | 3630 | const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*)); |
| ... | @@ -3657,12 +3713,84 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3657,12 +3713,84 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 3657 | ptr | 3713 | ptr |
| 3658 | else | 3714 | else |
| 3659 | try self.allocRegOrMem(inst, true); | 3715 | try self.allocRegOrMem(inst, true); |
| 3660 | try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand)); | 3716 | |
| | 3717 | const ptr_ty = self.air.typeOf(ty_op.operand); |
| | 3718 | if (ptr_ty.ptrInfo().data.host_size > 0) { |
| | 3719 | try self.packedLoad(dst_mcv, ptr, ptr_ty); |
| | 3720 | } else { |
| | 3721 | try self.load(dst_mcv, ptr, ptr_ty); |
| | 3722 | } |
| 3661 | break :result dst_mcv; | 3723 | break :result dst_mcv; |
| 3662 | }; | 3724 | }; |
| 3663 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); | 3725 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 3664 | } | 3726 | } |
| 3665 | | 3727 | |
| | 3728 | fn packedStore( |
| | 3729 | self: *Self, |
| | 3730 | ptr_mcv: MCValue, |
| | 3731 | val_mcv: MCValue, |
| | 3732 | ptr_ty: Type, |
| | 3733 | val_ty: Type, |
| | 3734 | ) InnerError!void { |
| | 3735 | const ptr_info = ptr_ty.ptrInfo().data; |
| | 3736 | |
| | 3737 | const limb_abi_size = @min(ptr_info.host_size, 8); |
| | 3738 | const limb_abi_bits = limb_abi_size * 8; |
| | 3739 | |
| | 3740 | const val_bit_size = val_ty.bitSize(self.target.*); |
| | 3741 | const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size); |
| | 3742 | const val_bit_off = ptr_info.bit_offset % limb_abi_bits; |
| | 3743 | |
| | 3744 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); |
| | 3745 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); |
| | 3746 | defer self.register_manager.unlockReg(ptr_lock); |
| | 3747 | |
| | 3748 | var limb_i: u16 = 0; |
| | 3749 | while (limb_i * limb_abi_bits < val_bit_off + val_bit_size) : (limb_i += 1) { |
| | 3750 | const part_bit_off = if (limb_i == 0) val_bit_off else 0; |
| | 3751 | const part_bit_size = |
| | 3752 | @min(val_bit_off + val_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off; |
| | 3753 | const limb_mem = Memory.sib( |
| | 3754 | Memory.PtrSize.fromSize(limb_abi_size), |
| | 3755 | .{ .base = ptr_reg, .disp = val_byte_off + limb_i * limb_abi_bits }, |
| | 3756 | ); |
| | 3757 | |
| | 3758 | const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) << |
| | 3759 | @intCast(u6, part_bit_off); |
| | 3760 | const part_mask_not = part_mask ^ |
| | 3761 | (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits)); |
| | 3762 | if (limb_abi_size <= 4) { |
| | 3763 | try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not)); |
| | 3764 | } else if (math.cast(i32, @bitCast(i64, part_mask_not))) |small| { |
| | 3765 | try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small)); |
| | 3766 | } else { |
| | 3767 | const part_mask_reg = try self.register_manager.allocReg(null, gp); |
| | 3768 | try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not)); |
| | 3769 | try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg); |
| | 3770 | } |
| | 3771 | |
| | 3772 | if (val_bit_size <= 64) { |
| | 3773 | const tmp_reg = try self.register_manager.allocReg(null, gp); |
| | 3774 | const tmp_mcv = MCValue{ .register = tmp_reg }; |
| | 3775 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| | 3776 | defer self.register_manager.unlockReg(tmp_lock); |
| | 3777 | |
| | 3778 | try self.genSetReg(val_ty, tmp_reg, val_mcv); |
| | 3779 | switch (limb_i) { |
| | 3780 | 0 => try self.genShiftBinOpMir(.shl, val_ty, tmp_mcv, .{ .immediate = val_bit_off }), |
| | 3781 | 1 => try self.genShiftBinOpMir(.shr, val_ty, tmp_mcv, .{ |
| | 3782 | .immediate = limb_abi_bits - val_bit_off, |
| | 3783 | }), |
| | 3784 | else => unreachable, |
| | 3785 | } |
| | 3786 | try self.genBinOpMir(.@"and", val_ty, tmp_mcv, .{ .immediate = part_mask }); |
| | 3787 | try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size)); |
| | 3788 | } else return self.fail("TODO: implement packed store of {}", .{ |
| | 3789 | val_ty.fmt(self.bin_file.options.module.?), |
| | 3790 | }); |
| | 3791 | } |
| | 3792 | } |
| | 3793 | |
| 3666 | fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { | 3794 | fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { |
| 3667 | const abi_size = @intCast(u32, value_ty.abiSize(self.target.*)); | 3795 | const abi_size = @intCast(u32, value_ty.abiSize(self.target.*)); |
| 3668 | switch (ptr) { | 3796 | switch (ptr) { |
| ... | @@ -3854,7 +3982,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3854,7 +3982,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index) !void { |
| 3854 | const value = try self.resolveInst(bin_op.rhs); | 3982 | const value = try self.resolveInst(bin_op.rhs); |
| 3855 | const value_ty = self.air.typeOf(bin_op.rhs); | 3983 | const value_ty = self.air.typeOf(bin_op.rhs); |
| 3856 | log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value }); | 3984 | log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value }); |
| 3857 | try self.store(ptr, value, ptr_ty, value_ty); | 3985 | if (ptr_ty.ptrInfo().data.host_size > 0) { |
| | 3986 | try self.packedStore(ptr, value, ptr_ty, value_ty); |
| | 3987 | } else { |
| | 3988 | try self.store(ptr, value, ptr_ty, value_ty); |
| | 3989 | } |
| 3858 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); | 3990 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 3859 | } | 3991 | } |
| 3860 | | 3992 | |
| ... | @@ -5218,7 +5350,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M | ... | @@ -5218,7 +5350,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M |
| 5218 | registerAlias(src_reg, abi_size), | 5350 | registerAlias(src_reg, abi_size), |
| 5219 | ), | 5351 | ), |
| 5220 | .immediate => |imm| { | 5352 | .immediate => |imm| { |
| 5221 | if (std.math.cast(i32, imm)) |small| { | 5353 | if (math.cast(i32, imm)) |small| { |
| 5222 | try self.asmRegisterRegisterImmediate( | 5354 | try self.asmRegisterRegisterImmediate( |
| 5223 | .imul, | 5355 | .imul, |
| 5224 | dst_alias, | 5356 | dst_alias, |
| ... | @@ -6824,7 +6956,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -6824,7 +6956,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
| 6824 | } else if (mem.startsWith(u8, op_str, "$")) { | 6956 | } else if (mem.startsWith(u8, op_str, "$")) { |
| 6825 | if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| { | 6957 | if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| { |
| 6826 | if (mnem_size) |size| { | 6958 | if (mnem_size) |size| { |
| 6827 | const max = @as(u64, std.math.maxInt(u64)) >> | 6959 | const max = @as(u64, math.maxInt(u64)) >> |
| 6828 | @intCast(u6, 64 - (size.bitSize() - 1)); | 6960 | @intCast(u6, 64 - (size.bitSize() - 1)); |
| 6829 | if ((if (s < 0) ~s else s) > max) | 6961 | if ((if (s < 0) ~s else s) > max) |
| 6830 | return self.fail("Invalid immediate size: '{s}'", .{op_str}); | 6962 | return self.fail("Invalid immediate size: '{s}'", .{op_str}); |
| ... | @@ -6832,7 +6964,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -6832,7 +6964,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
| 6832 | op.* = .{ .imm = Immediate.s(s) }; | 6964 | op.* = .{ .imm = Immediate.s(s) }; |
| 6833 | } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| { | 6965 | } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| { |
| 6834 | if (mnem_size) |size| { | 6966 | if (mnem_size) |size| { |
| 6835 | const max = @as(u64, std.math.maxInt(u64)) >> | 6967 | const max = @as(u64, math.maxInt(u64)) >> |
| 6836 | @intCast(u6, 64 - size.bitSize()); | 6968 | @intCast(u6, 64 - size.bitSize()); |
| 6837 | if (u > max) | 6969 | if (u > max) |
| 6838 | return self.fail("Invalid immediate size: '{s}'", .{op_str}); | 6970 | return self.fail("Invalid immediate size: '{s}'", .{op_str}); |
| ... | @@ -7171,7 +7303,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -7171,7 +7303,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 7171 | else => { | 7303 | else => { |
| 7172 | // 64 bit write to memory would take two mov's anyways so we | 7304 | // 64 bit write to memory would take two mov's anyways so we |
| 7173 | // insted just use two 32 bit writes to avoid register allocation | 7305 | // insted just use two 32 bit writes to avoid register allocation |
| 7174 | if (std.math.cast(i32, @bitCast(i64, imm))) |small| { | 7306 | if (math.cast(i32, @bitCast(i64, imm))) |small| { |
| 7175 | try self.asmMemoryImmediate(.mov, Memory.sib( | 7307 | try self.asmMemoryImmediate(.mov, Memory.sib( |
| 7176 | Memory.PtrSize.fromSize(abi_size), | 7308 | Memory.PtrSize.fromSize(abi_size), |
| 7177 | .{ .base = base_reg, .disp = -stack_offset }, | 7309 | .{ .base = base_reg, .disp = -stack_offset }, |
Jacob Young <jacobly@ziglang.org> 2023-04-15 01:59:43-04:00