authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-23 20:52:43-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-25 11:23:41-07:00
log881e931ee1758dee469c3f87700413bb321c9f43
treece3f2e3982362f9fcde2edf3719a3b40736e3bb6
parent7c56145a7651124dafdc87176ba85c814cd97d2d

x86_64 backend: implement `@memset` for element ABI size > 1

* make memset and memset_safe guarantee that if the length is comptime-known then it will be nonzero.

4 files changed, 74 insertions(+), 25 deletions(-)

src/Air.zig+5
......@@ -641,6 +641,8 @@ pub const Inst = struct {
641641 /// The element value may be undefined, in which case the destination
642642 /// memory region has undefined bytes after this function executes. In
643643 /// such case ignoring this instruction is legal lowering.
644 /// If the length is compile-time known (due to the destination being a
645 /// pointer-to-array), then it is guaranteed to be greater than zero.
644646 memset,
645647 /// Same as `memset`, except if the element value is undefined, the memory region
646648 /// should be filled with 0xaa bytes, and any other safety metadata such as Valgrind
......@@ -654,6 +656,9 @@ pub const Inst = struct {
654656 /// The two memory regions must not overlap.
655657 /// Result type is always void.
656658 /// Uses the `bin_op` field. LHS is the dest slice. RHS is the source pointer.
659 /// If the length is compile-time known (due to the destination or
660 /// source being a pointer-to-array), then it is guaranteed to be
661 /// greater than zero.
657662 memcpy,
658663
659664 /// Uses the `ty_pl` field with payload `Cmpxchg`.
src/Sema.zig+16-7
......@@ -21918,8 +21918,6 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
2191821918 } else break :rs src_src;
2191921919 } else dest_src;
2192021920
21921 try sema.requireRuntimeBlock(block, src, runtime_src);
21922
2192321921 const dest_ty = sema.typeOf(dest_ptr);
2192421922 const src_ty = sema.typeOf(src_ptr);
2192521923
......@@ -21946,10 +21944,16 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
2194621944 var new_src_ptr = src_ptr;
2194721945 if (len_val) |val| {
2194821946 const len = val.toUnsignedInt(target);
21947 if (len == 0) {
21948 // This AIR instruction guarantees length > 0 if it is comptime-known.
21949 return;
21950 }
2194921951 new_dest_ptr = try upgradeToArrayPtr(sema, block, dest_ptr, len);
2195021952 new_src_ptr = try upgradeToArrayPtr(sema, block, src_ptr, len);
2195121953 }
2195221954
21955 try sema.requireRuntimeBlock(block, src, runtime_src);
21956
2195321957 // Aliasing safety check.
2195421958 if (block.wantSafety()) {
2195521959 const dest_int = try block.addUnOp(.ptrtoint, new_dest_ptr);
......@@ -21995,13 +21999,18 @@ fn zirMemset(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
2199521999 const target = sema.mod.getTarget();
2199622000
2199722001 const runtime_src = if (try sema.resolveDefinedValue(block, dest_src, dest_ptr)) |ptr_val| rs: {
22002 const len_air_ref = try sema.fieldVal(block, src, dest_ptr, "len", dest_src);
22003 const len_val = (try sema.resolveDefinedValue(block, dest_src, len_air_ref)) orelse
22004 break :rs dest_src;
22005 const len_u64 = (try len_val.getUnsignedIntAdvanced(target, sema)).?;
22006 const len = try sema.usizeCast(block, dest_src, len_u64);
22007 if (len == 0) {
22008 // This AIR instruction guarantees length > 0 if it is comptime-known.
22009 return;
22010 }
22011
2199822012 if (!ptr_val.isComptimeMutablePtr()) break :rs dest_src;
2199922013 if (try sema.resolveMaybeUndefVal(uncoerced_elem)) |_| {
22000 const len_air_ref = try sema.fieldVal(block, src, dest_ptr, "len", dest_src);
22001 const len_val = (try sema.resolveDefinedValue(block, dest_src, len_air_ref)) orelse
22002 break :rs dest_src;
22003 const len_u64 = (try len_val.getUnsignedIntAdvanced(target, sema)).?;
22004 const len = try sema.usizeCast(block, dest_src, len_u64);
2200522014 for (0..len) |i| {
2200622015 const elem_index = try sema.addIntUnsigned(Type.usize, i);
2200722016 const elem_ptr = try sema.elemPtr(
src/arch/x86_64/CodeGen.zig+53-14
......@@ -8175,23 +8175,62 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
81758175 };
81768176 defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock);
81778177
8178 if (elem_ty.abiSize(self.target.*) != 1) {
8179 return self.fail("TODO implement airMemset when element ABI size > 1", .{});
8178 if (elem_ty.abiSize(self.target.*) == 1) {
8179 const len = switch (dst_ptr_ty.ptrSize()) {
8180 // TODO: this only handles slices stored in the stack
8181 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8182 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
8183 .C, .Many => unreachable,
8184 };
8185 const len_lock: ?RegisterLock = switch (len) {
8186 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
8187 else => null,
8188 };
8189 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
8190
8191 // TODO: dst_ptr could be a slice rather than raw pointer
8192 try self.genInlineMemset(dst_ptr, src_val, len, .{});
8193 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
81808194 }
81818195
8182 const len = switch (dst_ptr_ty.ptrSize()) {
8183 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8184 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
8185 .C, .Many => unreachable,
8186 };
8187 const len_lock: ?RegisterLock = switch (len) {
8188 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
8189 else => null,
8190 };
8191 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
8196 // Store the first element, and then rely on memcpy copying forwards.
8197 // Length zero requires a runtime check - so we handle arrays specially
8198 // here to elide it.
8199 switch (dst_ptr_ty.ptrSize()) {
8200 .Slice => {
8201 // TODO: this only handles slices stored in the stack
8202 const ptr = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 });
8203 const len = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 });
8204 _ = ptr;
8205 _ = len;
8206 return self.fail("TODO implement airMemset for x86_64 with ABI size > 1 using a slice", .{});
8207 },
8208 .One => {
8209 const len = dst_ptr_ty.childType().arrayLen();
8210 assert(len != 0); // prevented by Sema
8211 try self.store(dst_ptr, src_val, dst_ptr_ty, elem_ty);
8212
8213 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8214 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8215 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8216 defer self.register_manager.unlockReg(second_elem_ptr_lock);
81928217
8193 // TODO: dst_ptr could be a slice rather than raw pointer
8194 try self.genInlineMemset(dst_ptr, src_val, len, .{});
8218 const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));
8219
8220 try self.asmRegisterMemory(
8221 .lea,
8222 second_elem_ptr_reg,
8223 Memory.sib(.qword, .{
8224 .base = try self.copyToTmpRegister(Type.usize, dst_ptr),
8225 .disp = elem_abi_size,
8226 }),
8227 );
8228
8229 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
8230 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy, .{});
8231 },
8232 .C, .Many => unreachable,
8233 }
81958234
81968235 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
81978236}
test/behavior/basic.zig-4
......@@ -361,10 +361,6 @@ test "@memset on array pointers" {
361361 // TODO: implement memset when element ABI size > 1
362362 return error.SkipZigTest;
363363 }
364 if (builtin.zig_backend == .stage2_x86_64) {
365 // TODO: implement memset when element ABI size > 1
366 return error.SkipZigTest;
367 }
368364
369365 try testMemsetArray();
370366 try comptime testMemsetArray();