| ... | ... | @@ -327,6 +327,13 @@ pub const MCValue = union(enum) { |
| 327 | 327 | }; |
| 328 | 328 | } |
| 329 | 329 | |
| 330 | fn isAddress(mcv: MCValue) bool { |
| 331 | return switch (mcv) { |
| 332 | .immediate, .register, .register_offset, .lea_frame => true, |
| 333 | else => false, |
| 334 | }; |
| 335 | } |
| 336 | |
| 330 | 337 | fn address(mcv: MCValue) MCValue { |
| 331 | 338 | return switch (mcv) { |
| 332 | 339 | .none, |
| ... | ... | @@ -23005,17 +23012,69 @@ fn genSetMem( |
| 23005 | 23012 | |
| 23006 | 23013 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); |
| 23007 | 23014 | }, |
| 23008 | | else => try self.genInlineMemcpy( |
| 23009 | | dst_ptr_mcv, |
| 23010 | | src_mcv.address(), |
| 23011 | | .{ .immediate = abi_size }, |
| 23012 | | ), |
| 23015 | else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }, .{ .no_alias = true }), |
| 23013 | 23016 | }, |
| 23014 | 23017 | .air_ref => |src_ref| try self.genSetMem(base, disp, ty, try self.resolveInst(src_ref), opts), |
| 23015 | 23018 | } |
| 23016 | 23019 | } |
| 23017 | 23020 | |
| 23018 | | fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void { |
| 23021 | fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct { |
| 23022 | no_alias: bool, |
| 23023 | }) InnerError!void { |
| 23024 | if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) { |
| 23025 | else => {}, |
| 23026 | .immediate => |len_imm| switch (len_imm) { |
| 23027 | else => {}, |
| 23028 | 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| 23029 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte })); |
| 23030 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8()); |
| 23031 | return; |
| 23032 | }, |
| 23033 | 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| 23034 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word })); |
| 23035 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16()); |
| 23036 | return; |
| 23037 | }, |
| 23038 | 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| 23039 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword })); |
| 23040 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32()); |
| 23041 | return; |
| 23042 | }, |
| 23043 | 8 => if (self.target.cpu.arch == .x86_64) { |
| 23044 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| { |
| 23045 | try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword })); |
| 23046 | try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64()); |
| 23047 | return; |
| 23048 | } |
| 23049 | }, |
| 23050 | 16 => if (self.hasFeature(.avx)) { |
| 23051 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| 23052 | try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); |
| 23053 | try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); |
| 23054 | return; |
| 23055 | } |
| 23056 | } else if (self.hasFeature(.sse2)) { |
| 23057 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| 23058 | try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); |
| 23059 | try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); |
| 23060 | return; |
| 23061 | } |
| 23062 | } else if (self.hasFeature(.sse)) { |
| 23063 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| 23064 | try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword })); |
| 23065 | try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128()); |
| 23066 | return; |
| 23067 | } |
| 23068 | }, |
| 23069 | 32 => if (self.hasFeature(.avx)) { |
| 23070 | if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| { |
| 23071 | try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword })); |
| 23072 | try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256()); |
| 23073 | return; |
| 23074 | } |
| 23075 | }, |
| 23076 | }, |
| 23077 | }; |
| 23019 | 23078 | try self.spillRegisters(&.{ .rsi, .rdi, .rcx }); |
| 23020 | 23079 | try self.genSetReg(.rsi, .usize, src_ptr, .{}); |
| 23021 | 23080 | try self.genSetReg(.rdi, .usize, dst_ptr, .{}); |
| ... | ... | @@ -24057,7 +24116,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 24057 | 24116 | len_reg, |
| 24058 | 24117 | .s(elem_abi_size), |
| 24059 | 24118 | ); |
| 24060 | | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv); |
| 24119 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv, .{ .no_alias = false }); |
| 24061 | 24120 | |
| 24062 | 24121 | self.performReloc(skip_reloc); |
| 24063 | 24122 | }, |
| ... | ... | @@ -24082,7 +24141,7 @@ fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 24082 | 24141 | } }, .{}); |
| 24083 | 24142 | |
| 24084 | 24143 | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; |
| 24085 | | try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy); |
| 24144 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy, .{ .no_alias = false }); |
| 24086 | 24145 | }, |
| 24087 | 24146 | .c, .many => unreachable, |
| 24088 | 24147 | } |
| ... | ... | @@ -24165,7 +24224,7 @@ fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 24165 | 24224 | else => src, |
| 24166 | 24225 | }; |
| 24167 | 24226 | |
| 24168 | | try self.genInlineMemcpy(dst_ptr, src_ptr, len); |
| 24227 | try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{ .no_alias = true }); |
| 24169 | 24228 | |
| 24170 | 24229 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 24171 | 24230 | } |
Jacob Young <jacobly@ziglang.org> 2025-01-02 04:42:27-05:00