| ... | @@ -327,6 +327,13 @@ pub const MCValue = union(enum) { | ... | @@ -327,6 +327,13 @@ pub const MCValue = union(enum) { |
| 327 | }; | 327 | }; |
| 328 | } | 328 | } |
| 329 | | 329 | |
| | 330 | fn isAddress(mcv: MCValue) bool { |
| | 331 | return switch (mcv) { |
| | 332 | .immediate, .register, .register_offset, .lea_frame => true, |
| | 333 | else => false, |
| | 334 | }; |
| | 335 | } |
| | 336 | |
| 330 | fn address(mcv: MCValue) MCValue { | 337 | fn address(mcv: MCValue) MCValue { |
| 331 | return switch (mcv) { | 338 | return switch (mcv) { |
| 332 | .none, | 339 | .none, |
| ... | @@ -23005,17 +23012,69 @@ fn genSetMem( | ... | @@ -23005,17 +23012,69 @@ fn genSetMem( |
| 23005 | | 23012 | |
| 23006 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); | 23013 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); |
| 23007 | }, | 23014 | }, |
| 23008 | else => try self.genInlineMemcpy( | 23015 | else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }, .{ .no_alias = true }), |
| 23009 | dst_ptr_mcv, | | |
| 23010 | src_mcv.address(), | | |
| 23011 | .{ .immediate = abi_size }, | | |
| 23012 | ), | | |
| 23013 | }, | 23016 | }, |
| 23014 | .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts), | 23017 | .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts), |
| 23015 | } | 23018 | } |
| 23016 | } | 23019 | } |
| 23017 | | 23020 | |
| 23018 | fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void { | 23021 | fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct { |
| | 23022 | no_alias: bool, |
| | 23023 | }) InnerError!void { |
| | 23024 | if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) { |
| | 23025 | else => {}, |
| | 23026 | .immediate => |len_imm| switch (len_imm) { |
| | 23027 | else => {}, |
| | 23028 | 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| | 23029 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte })); |
| | 23030 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8()); |
| | 23031 | return; |
| | 23032 | }, |
| | 23033 | 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| | 23034 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word })); |
| | 23035 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16()); |
| | 23036 | return; |
| | 23037 | }, |
| | 23038 | 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| | 23039 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword })); |
| | 23040 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32()); |
| | 23041 | return; |
| | 23042 | }, |
| | 23043 | 8 => if (self.target.cpu.arch == .x86_64) { |
| | 23044 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| | 23045 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword })); |
| | 23046 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64()); |
| | 23047 | return; |
| | 23048 | } |
| | 23049 | }, |
| | 23050 | 16 => if (self.hasFeature(.avx)) { |
| | 23051 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| | 23052 | try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); |
| | 23053 | try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); |
| | 23054 | return; |
| | 23055 | } |
| | 23056 | } else if (self.hasFeature(.sse2)) { |
| | 23057 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| | 23058 | try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); |
| | 23059 | try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); |
| | 23060 | return; |
| | 23061 | } |
| | 23062 | } else if (self.hasFeature(.sse)) { |
| | 23063 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| | 23064 | try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); |
| | 23065 | try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); |
| | 23066 | return; |
| | 23067 | } |
| | 23068 | }, |
| | 23069 | 32 => if (self.hasFeature(.avx)) { |
| | 23070 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| | 23071 | try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword })); |
| | 23072 | try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256()); |
| | 23073 | return; |
| | 23074 | } |
| | 23075 | }, |
| | 23076 | }, |
| | 23077 | }; |
| 23019 | try self.spillRegisters(&.{ .rsi, .rdi, .rcx }); | 23078 | try self.spillRegisters(&.{ .rsi, .rdi, .rcx }); |
| 23020 | try self.genSetReg(.rsi, .usize, src_ptr, .{}); | 23079 | try self.genSetReg(.rsi, .usize, src_ptr, .{}); |
| 23021 | try self.genSetReg(.rdi, .usize, dst_ptr, .{}); | 23080 | try self.genSetReg(.rdi, .usize, dst_ptr, .{}); |
| ... | @@ -24057,7 +24116,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { | ... | @@ -24057,7 +24116,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 24057 | len_reg, | 24116 | len_reg, |
| 24058 | .s(elem_abi_size), | 24117 | .s(elem_abi_size), |
| 24059 | ); | 24118 | ); |
| 24060 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv); | 24119 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv, .{ .no_alias = false }); |
| 24061 | | 24120 | |
| 24062 | self.performReloc(skip_reloc); | 24121 | self.performReloc(skip_reloc); |
| 24063 | }, | 24122 | }, |
| ... | @@ -24082,7 +24141,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { | ... | @@ -24082,7 +24141,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 24082 | } }, .{}); | 24141 | } }, .{}); |
| 24083 | | 24142 | |
| 24084 | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; | 24143 | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; |
| 24085 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy); | 24144 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy, .{ .no_alias = false }); |
| 24086 | }, | 24145 | }, |
| 24087 | .c, .many => unreachable, | 24146 | .c, .many => unreachable, |
| 24088 | } | 24147 | } |
| ... | @@ -24165,7 +24224,7 @@ fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -24165,7 +24224,7 @@ fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 24165 | else => src, | 24224 | else => src, |
| 24166 | }; | 24225 | }; |
| 24167 | | 24226 | |
| 24168 | try self.genInlineMemcpy(dst_ptr, src_ptr, len); | 24227 | try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{ .no_alias = true }); |
| 24169 | | 24228 | |
| 24170 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); | 24229 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 24171 | } | 24230 | } |
Jacob Young <jacobly@ziglang.org> 2025-01-02 04:42:27-05:00
src