authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-16 21:40:08+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-16 21:40:08+01:00
log94474ec7c783b94face643f448426bcb91adfc93
treefb70f73bc21f6130e55edd742b4e7df57bd96cea
parent09d468b237e58dd70cfa0a4311efab5ddabc9f95

x64: refactor use of inline memcpy

1 files changed, 84 insertions(+), 213 deletions(-)

src/arch/x86_64/CodeGen.zig+84-213
......@@ -1782,36 +1782,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
17821782 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
17831783 }
17841784
1785 self.register_manager.freezeRegs(&.{ .rax, .rcx });
1786 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
1787
1788 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
1789 const addr_reg = regs[0];
1790 const count_reg = regs[1];
1791 const tmp_reg = regs[2];
1792
1793 _ = try self.addInst(.{
1794 .tag = .mov,
1795 .ops = (Mir.Ops{
1796 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
1797 .reg2 = reg,
1798 }).encode(),
1799 .data = undefined,
1800 });
1801
1802 try self.register_manager.getReg(.rax, null);
1803 try self.register_manager.getReg(.rcx, null);
1804
1805 // TODO allow for abi size to be u64
1806 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
1807
1808 return self.genInlineMemcpy(
1809 -off,
1810 .rbp,
1811 registerAlias(addr_reg, @divExact(reg.size(), 8)),
1812 count_reg.to64(),
1813 tmp_reg.to8(),
1814 );
1785 try self.genInlineMemcpy(off, .rbp, elem_ty, ptr);
18151786 },
18161787 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
18171788 }
......@@ -3626,62 +3597,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
36263597 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
36273598 }
36283599
3629 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3630 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3631
3632 const addr_reg: Register = blk: {
3633 switch (mcv) {
3634 .got_load,
3635 .direct_load,
3636 => |sym_index| {
3637 const flags: u2 = switch (mcv) {
3638 .got_load => 0b00,
3639 .direct_load => 0b01,
3640 else => unreachable,
3641 };
3642 const addr_reg = try self.register_manager.allocReg(null);
3643 _ = try self.addInst(.{
3644 .tag = .lea_pie,
3645 .ops = (Mir.Ops{
3646 .reg1 = addr_reg.to64(),
3647 .flags = flags,
3648 }).encode(),
3649 .data = .{
3650 .load_reloc = .{
3651 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
3652 .sym_index = sym_index,
3653 },
3654 },
3655 });
3656 break :blk addr_reg;
3657 },
3658 .memory => |addr| {
3659 const addr_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = addr });
3660 break :blk addr_reg;
3661 },
3662 else => unreachable,
3663 }
3664 };
3665
3666 self.register_manager.freezeRegs(&.{addr_reg});
3667 defer self.register_manager.unfreezeRegs(&.{addr_reg});
3668
3669 const regs = try self.register_manager.allocRegs(2, .{ null, null });
3670 const count_reg = regs[0];
3671 const tmp_reg = regs[1];
3672
3673 try self.register_manager.getReg(.rax, null);
3674 try self.register_manager.getReg(.rcx, null);
3675
3676 // TODO allow for abi_size to be u64
3677 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3678 try self.genInlineMemcpy(
3679 -stack_offset,
3680 .rsp,
3681 addr_reg.to64(),
3682 count_reg.to64(),
3683 tmp_reg.to8(),
3684 );
3600 try self.genInlineMemcpy(stack_offset, .rsp, ty, mcv);
36853601 },
36863602 .register => |reg| {
36873603 _ = try self.addInst(.{
......@@ -3698,41 +3614,13 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
36983614 const reg = try self.copyToTmpRegister(ty, mcv);
36993615 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
37003616 },
3701 .stack_offset => |off| {
3617 .stack_offset => {
37023618 if (abi_size <= 8) {
37033619 const reg = try self.copyToTmpRegister(ty, mcv);
37043620 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
37053621 }
37063622
3707 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3708 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3709
3710 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
3711 const addr_reg = regs[0];
3712 const count_reg = regs[1];
3713 const tmp_reg = regs[2];
3714
3715 try self.register_manager.getReg(.rax, null);
3716 try self.register_manager.getReg(.rcx, null);
3717
3718 _ = try self.addInst(.{
3719 .tag = .lea,
3720 .ops = (Mir.Ops{
3721 .reg1 = addr_reg.to64(),
3722 .reg2 = .rbp,
3723 }).encode(),
3724 .data = .{ .imm = @bitCast(u32, -off) },
3725 });
3726
3727 // TODO allow for abi_size to be u64
3728 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3729 try self.genInlineMemcpy(
3730 -stack_offset,
3731 .rsp,
3732 addr_reg.to64(),
3733 count_reg.to64(),
3734 tmp_reg.to8(),
3735 );
3623 try self.genInlineMemcpy(stack_offset, .rsp, ty, mcv);
37363624 },
37373625 else => return self.fail("TODO implement args on stack for {}", .{mcv}),
37383626 }
......@@ -3847,65 +3735,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerErro
38473735 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
38483736 }
38493737
3850 try self.register_manager.getReg(.rax, null);
3851 try self.register_manager.getReg(.rcx, null);
3852
3853 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rbp });
3854 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rbp });
3855
3856 const addr_reg: Register = blk: {
3857 switch (mcv) {
3858 .memory => |addr| {
3859 const reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = addr });
3860 break :blk reg;
3861 },
3862 .direct_load,
3863 .got_load,
3864 => |sym_index| {
3865 const flags: u2 = switch (mcv) {
3866 .got_load => 0b00,
3867 .direct_load => 0b01,
3868 else => unreachable,
3869 };
3870 const addr_reg = try self.register_manager.allocReg(null);
3871 _ = try self.addInst(.{
3872 .tag = .lea_pie,
3873 .ops = (Mir.Ops{
3874 .reg1 = addr_reg.to64(),
3875 .flags = flags,
3876 }).encode(),
3877 .data = .{
3878 .load_reloc = .{
3879 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
3880 .sym_index = sym_index,
3881 },
3882 },
3883 });
3884 break :blk addr_reg;
3885 },
3886 else => {
3887 return self.fail("TODO implement memcpy for setting stack from {}", .{mcv});
3888 },
3889 }
3890 };
3891
3892 self.register_manager.freezeRegs(&.{addr_reg});
3893 defer self.register_manager.unfreezeRegs(&.{addr_reg});
3894
3895 const regs = try self.register_manager.allocRegs(2, .{ null, null });
3896 const count_reg = regs[0];
3897 const tmp_reg = regs[1];
3898
3899 // TODO allow for abi_size to be u64
3900 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3901
3902 return self.genInlineMemcpy(
3903 -stack_offset,
3904 .rbp,
3905 addr_reg.to64(),
3906 count_reg.to64(),
3907 tmp_reg.to8(),
3908 );
3738 try self.genInlineMemcpy(stack_offset, .rbp, ty, mcv);
39093739 },
39103740 .ptr_stack_offset => {
39113741 const reg = try self.copyToTmpRegister(ty, mcv);
......@@ -3922,48 +3752,89 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerErro
39223752 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
39233753 }
39243754
3925 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rbp });
3926 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rbp });
3755 try self.genInlineMemcpy(stack_offset, .rbp, ty, mcv);
3756 },
3757 }
3758}
39273759
3928 const regs = try self.register_manager.allocRegs(3, .{ null, null, null });
3929 const addr_reg = regs[0];
3930 const count_reg = regs[1];
3931 const tmp_reg = regs[2];
3760fn genInlineMemcpy(self: *Self, stack_offset: i32, stack_reg: Register, ty: Type, val: MCValue) InnerError!void {
3761 const abi_size = ty.abiSize(self.target.*);
39323762
3933 try self.register_manager.getReg(.rax, null);
3934 try self.register_manager.getReg(.rcx, null);
3763 try self.register_manager.getReg(.rax, null);
3764 try self.register_manager.getReg(.rcx, null);
39353765
3936 _ = try self.addInst(.{
3937 .tag = .lea,
3938 .ops = (Mir.Ops{
3939 .reg1 = addr_reg.to64(),
3940 .reg2 = .rbp,
3941 }).encode(),
3942 .data = .{ .imm = @bitCast(u32, -off) },
3943 });
3766 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rbp });
3767 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rbp });
3768
3769 const addr_reg: Register = blk: {
3770 switch (val) {
3771 .memory => |addr| {
3772 const reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = addr });
3773 break :blk reg;
3774 },
3775 .direct_load,
3776 .got_load,
3777 => |sym_index| {
3778 const flags: u2 = switch (val) {
3779 .got_load => 0b00,
3780 .direct_load => 0b01,
3781 else => unreachable,
3782 };
3783 const addr_reg = (try self.register_manager.allocReg(null)).to64();
3784 _ = try self.addInst(.{
3785 .tag = .lea_pie,
3786 .ops = (Mir.Ops{
3787 .reg1 = addr_reg,
3788 .flags = flags,
3789 }).encode(),
3790 .data = .{
3791 .load_reloc = .{
3792 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
3793 .sym_index = sym_index,
3794 },
3795 },
3796 });
3797 break :blk addr_reg;
3798 },
3799 .stack_offset => |off| {
3800 const addr_reg = (try self.register_manager.allocReg(null)).to64();
3801 _ = try self.addInst(.{
3802 .tag = .lea,
3803 .ops = (Mir.Ops{
3804 .reg1 = addr_reg,
3805 .reg2 = .rbp,
3806 }).encode(),
3807 .data = .{ .imm = @bitCast(u32, -off) },
3808 });
3809 break :blk addr_reg;
3810 },
3811 .register => |reg| {
3812 const addr_reg = try self.register_manager.allocReg(null);
3813 _ = try self.addInst(.{
3814 .tag = .mov,
3815 .ops = (Mir.Ops{
3816 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
3817 .reg2 = reg,
3818 }).encode(),
3819 .data = undefined,
3820 });
3821 break :blk addr_reg.to64();
3822 },
3823 else => {
3824 return self.fail("TODO implement memcpy for setting stack from {}", .{val});
3825 },
3826 }
3827 };
39443828
3945 // TODO allow for abi_size to be u64
3946 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
3829 self.register_manager.freezeRegs(&.{addr_reg});
3830 defer self.register_manager.unfreezeRegs(&.{addr_reg});
39473831
3948 return self.genInlineMemcpy(
3949 -stack_offset,
3950 .rbp,
3951 addr_reg.to64(),
3952 count_reg.to64(),
3953 tmp_reg.to8(),
3954 );
3955 },
3956 }
3957}
3832 const regs = try self.register_manager.allocRegs(2, .{ null, null });
3833 const count_reg = regs[0].to64();
3834 const tmp_reg = regs[1].to8();
3835
3836 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
39583837
3959fn genInlineMemcpy(
3960 self: *Self,
3961 stack_offset: i32,
3962 stack_reg: Register,
3963 addr_reg: Register,
3964 count_reg: Register,
3965 tmp_reg: Register,
3966) InnerError!void {
39673838 // mov rcx, 0
39683839 _ = try self.addInst(.{
39693840 .tag = .mov,
......@@ -4016,7 +3887,7 @@ fn genInlineMemcpy(
40163887 .reg1 = stack_reg,
40173888 .reg2 = tmp_reg.to8(),
40183889 }).encode(),
4019 .data = .{ .imm = @bitCast(u32, stack_offset) },
3890 .data = .{ .imm = @bitCast(u32, -stack_offset) },
40203891 });
40213892
40223893 // add rcx, 1
......@@ -4959,8 +4830,8 @@ fn parseRegName(name: []const u8) ?Register {
49594830 return std.meta.stringToEnum(Register, name);
49604831}
49614832
4833/// Returns register wide enough to hold at least `size_bytes`.
49624834fn registerAlias(reg: Register, size_bytes: u32) Register {
4963 // For x86_64 we have to pick a smaller register alias depending on abi size.
49644835 if (size_bytes == 0) {
49654836 unreachable; // should be comptime known
49664837 } else if (size_bytes <= 1) {