authorgravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-24 00:02:30+02:00
committergravatar for paul.verigo@gmail.comPavel Verigo <paul.verigo@gmail.com> 2026-08-24 00:02:30+02:00
log93510cc95c3a07fc2765f729d149d8e76b2666e6
treed14fdabc3ae96c5ee411b4d91f4841a93fca8fd6
parent40ebd816226f5bd74f52399cda6d1c845e4e4c26
parent7def86c2f2a721815bd232b8d4fba7b6682345ae

Merge pull request 'sema: dont use aggregate_init for runtime splat on arrays, reuse AIR .splat' (#36524) from pavelverigo/zig:air-splat-arrays into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36524

13 files changed, 139 insertions(+), 42 deletions(-)

lib/compiler_rt.zig+8
......@@ -602,6 +602,8 @@ inline fn negXi2(comptime T: type, a: T) T {
602602}
603603
604604fn memsetSmallPowerOf2(d: [*]u8, b: u8, comptime size: usize) void {
605 @disableIntrinsics();
606
605607 if (size > @sizeOf(usize)) {
606608 d[0..size].* = @splat(b);
607609 } else {
......@@ -619,6 +621,8 @@ fn shortMemset(
619621 b: u8,
620622 len: usize,
621623) void {
624 @disableIntrinsics();
625
622626 if (log_min + 1 != log_max) {
623627 const mid = (log_min + log_max) / 2;
624628 if (len > 1 << mid) {
......@@ -635,6 +639,8 @@ fn shortMemset(
635639}
636640
637641fn fastMemset(dest: ?[*]u8, c: c_int, len: usize) callconv(.c) ?[*]u8 {
642 @disableIntrinsics();
643
638644 const b: u8 = @truncate(@as(c_uint, @bitCast(c)));
639645 const n = std.simd.suggestVectorLength(u8) orelse @sizeOf(usize);
640646
......@@ -668,6 +674,8 @@ fn fastMemset(dest: ?[*]u8, c: c_int, len: usize) callconv(.c) ?[*]u8 {
668674}
669675
670676fn smallMemset(dest: ?[*]u8, c: c_int, len: usize) callconv(.c) ?[*]u8 {
677 @disableIntrinsics();
678
671679 const b: u8 = @truncate(@as(c_uint, @bitCast(c)));
672680
673681 if (len != 0) {
src/Air.zig+2-2
......@@ -780,8 +780,8 @@ pub const Inst = struct {
780780 reduce,
781781 /// Same as `reduce` with optimized float mode.
782782 reduce_optimized,
783 /// Given an integer, bool, float, or pointer operand, return a vector with all elements
784 /// equal to the scalar value.
783 /// Given an operand, return a vector or array with all elements equal to the operand.
784 /// For a sentinel-terminated array, the sentinel is derived from the result type.
785785 /// Uses the `ty_op` field.
786786 splat,
787787 /// Constructs a vector by selecting elements from a single vector based on a mask. Each
src/Air/Legalize.zig+22-8
......@@ -200,6 +200,8 @@ pub const Feature = enum {
200200 expand_packed_agg_field_val,
201201 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
202202 expand_packed_aggregate_init,
203 /// Replace `splat` of an array with an `aggregate_init`.
204 expand_array_splat,
203205 /// Replace `array_to_vector` with an `array_elem_val` per element followed by an `aggregate_init`.
204206 expand_array_to_vector,
205207
......@@ -891,15 +893,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
891893 .soft_float => unreachable, // the operand is not a scalar
892894 }
893895 },
894 .splat => if (l.features.has(.splat_one_elem_to_bit_cast)) {
896 .splat => {
895897 const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
896 switch (ty_op.ty.vectorLen(zcu)) {
897 0 => unreachable,
898 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
899 .ty = ty_op.ty,
900 .operand = ty_op.operand,
901 } }),
902 else => {},
898 switch (ty_op.ty.zigTypeTag(zcu)) {
899 .vector => switch (ty_op.ty.vectorLen(zcu)) {
900 0 => unreachable,
901 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
902 .ty = ty_op.ty,
903 .operand = ty_op.operand,
904 } }),
905 else => {},
906 },
907 .array => if (l.features.has(.expand_array_splat)) {
908 const len: usize = @intCast(ty_op.ty.arrayLen(zcu));
909 const elems_start: u32 = @intCast(l.air_extra.items.len);
910 try l.air_extra.appendNTimes(l.pt.zcu.gpa, @backingInt(ty_op.operand), len);
911 continue :inst l.replaceInst(inst, .aggregate_init, .{ .ty_pl = .{
912 .ty = ty_op.ty,
913 .payload = elems_start,
914 } });
915 },
916 else => unreachable,
903917 }
904918 },
905919 .shuffle_one => {
src/Sema.zig+1-7
......@@ -23271,13 +23271,7 @@ fn zirSplat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.I
2327123271 try sema.requireRuntimeBlock(block, src, scalar_src);
2327223272
2327323273 switch (dest_ty.zigTypeTag(zcu)) {
23274 .array => {
23275 const elems = try sema.arena.alloc(Air.Inst.Ref, len + @intFromBool(maybe_sentinel != null));
23276 @memset(elems[0..len], scalar);
23277 if (maybe_sentinel) |s| elems[len] = Air.internedToRef(s.toIntern());
23278 return block.addAggregateInit(dest_ty, elems);
23279 },
23280 .vector => return block.addTyOp(.splat, dest_ty, scalar),
23274 .vector, .array => return block.addTyOp(.splat, dest_ty, scalar),
2328123275 else => unreachable,
2328223276 }
2328323277}
src/codegen/aarch64.zig+1
......@@ -8,6 +8,7 @@ pub const Select = @import("aarch64/Select.zig");
88pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
99 return comptime &.initMany(&.{
1010 .expand_bit_cast_safe,
11 .expand_array_splat,
1112 .expand_array_to_vector,
1213 });
1314}
src/codegen/c.zig+1
......@@ -39,6 +39,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
3939 .expand_packed_store = true,
4040 .expand_packed_agg_field_val = true,
4141 .expand_packed_aggregate_init = true,
42 .expand_array_splat = true,
4243 .expand_array_to_vector = true,
4344
4445 .scalarize_bit_cast_array = true,
src/codegen/llvm/FuncGen.zig+77-25
......@@ -5590,8 +5590,36 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
55905590 .slice => null,
55915591 .many, .c => unreachable,
55925592 });
5593 const len_bytes = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
55935594
5594 if (allow_byte_memset) if (bin_op.rhs.toInterned()) |elem_ip_index| {
5595 try self.lowerMemset(
5596 dest_ptr,
5597 dest_ptr_align,
5598 bin_op.rhs,
5599 elem_ty,
5600 len_bytes,
5601 access_kind,
5602 safety,
5603 allow_byte_memset,
5604 );
5605 return .none;
5606}
5607
5608fn lowerMemset(
5609 self: *FuncGen,
5610 dest_ptr: Builder.Value,
5611 dest_ptr_align: InternPool.Alignment,
5612 elem_ref: Air.Inst.Ref,
5613 elem_ty: Type,
5614 len_bytes: Builder.Value,
5615 access_kind: Builder.MemoryAccessKind,
5616 safety: bool,
5617 allow_byte_memset: bool,
5618) Allocator.Error!void {
5619 const o = self.object;
5620 const zcu = o.zcu;
5621
5622 if (allow_byte_memset) if (elem_ref.toInterned()) |elem_ip_index| {
55955623 const elem_val: Value = .fromInterned(elem_ip_index);
55965624 if (elem_val.isUndef(zcu)) {
55975625 // Even if safety is disabled, we still emit a memset to undefined since it conveys
......@@ -5601,20 +5629,19 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
56015629 try o.builder.intValue(.i8, 0xaa)
56025630 else
56035631 try o.builder.undefValue(.i8);
5604 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
56055632 _ = try self.wip.callMemSet(
56065633 dest_ptr,
56075634 dest_ptr_align.toLlvm(),
56085635 fill_byte,
5609 len,
5636 len_bytes,
56105637 access_kind,
56115638 self.disable_intrinsics,
56125639 );
56135640 const owner_mod = self.ownerModule();
56145641 if (safety and owner_mod.valgrind) {
5615 try self.valgrindMarkUndef(dest_ptr, len);
5642 try self.valgrindMarkUndef(dest_ptr, len_bytes);
56165643 }
5617 return .none;
5644 return;
56185645 }
56195646
56205647 // Test if the element value is compile-time known to be a
......@@ -5623,20 +5650,19 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
56235650 // intrinsic can be used.
56245651 if (try elem_val.hasRepeatedByteRepr(zcu)) |byte_val| {
56255652 const fill_byte = try o.builder.intValue(.i8, byte_val);
5626 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
56275653 _ = try self.wip.callMemSet(
56285654 dest_ptr,
56295655 dest_ptr_align.toLlvm(),
56305656 fill_byte,
5631 len,
5657 len_bytes,
56325658 access_kind,
56335659 self.disable_intrinsics,
56345660 );
5635 return .none;
5661 return;
56365662 }
56375663 };
56385664
5639 const value = try self.resolveInst(bin_op.rhs);
5665 const value = try self.resolveInst(elem_ref);
56405666 const elem_abi_size = elem_ty.abiSize(zcu);
56415667
56425668 intrinsic: {
......@@ -5660,16 +5686,15 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
56605686 break :intrinsic;
56615687 };
56625688 // Great, we can use the intrinsic!
5663 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
56645689 _ = try self.wip.callMemSet(
56655690 dest_ptr,
56665691 dest_ptr_align.toLlvm(),
56675692 fill_byte,
5668 len,
5693 len_bytes,
56695694 access_kind,
56705695 self.disable_intrinsics,
56715696 );
5672 return .none;
5697 return;
56735698 }
56745699
56755700 // non-byte-sized element. lower with a loop. something like this:
......@@ -5693,15 +5718,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
56935718 const body_block = try self.wip.block(1, "InlineMemsetBody");
56945719 const end_block = try self.wip.block(1, "InlineMemsetEnd");
56955720
5696 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
5697 .slice => try self.ptraddScaled(
5698 dest_ptr,
5699 try self.wip.extractValue(dest_slice, &.{1}, ""),
5700 elem_abi_size,
5701 ),
5702 .one => try self.ptraddConst(dest_ptr, ptr_ty.childType(zcu).abiSize(zcu)),
5703 .many, .c => unreachable,
5704 };
5721 const end_ptr = try self.ptraddScaled(dest_ptr, len_bytes, 1);
57055722 _ = try self.wip.br(loop_block);
57065723
57075724 self.wip.cursor = .{ .block = loop_block };
......@@ -5718,7 +5735,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
57185735
57195736 self.wip.cursor = .{ .block = end_block };
57205737 it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
5721 return .none;
5738 return;
57225739}
57235740
57245741fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -5980,10 +5997,45 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
59805997}
59815998
59825999fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6000 const o = self.object;
6001 const zcu = o.zcu;
59836002 const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5984 const scalar = try self.resolveInst(ty_op.operand);
5985 const vector_ty = self.typeOfIndex(inst);
5986 return self.wip.splatVector(try self.object.lowerType(vector_ty, .as_value), scalar, "");
6003 const result_ty = self.typeOfIndex(inst);
6004 switch (result_ty.zigTypeTag(zcu)) {
6005 .vector => {
6006 const scalar = try self.resolveInst(ty_op.operand);
6007 return self.wip.splatVector(try o.lowerType(result_ty, .as_value), scalar, "");
6008 },
6009 .array => {
6010 assert(isByRef(result_ty, zcu));
6011
6012 const result_ptr = try self.buildZigAlloca(result_ty, .none);
6013 const array_info = result_ty.arrayInfo(zcu);
6014 const elem_size = array_info.elem_type.abiSize(zcu);
6015 const len_bytes = array_info.len * elem_size;
6016 const len_bytes_llvm = try o.builder.intValue(try o.lowerType(.usize, .as_value), len_bytes);
6017
6018 try self.lowerMemset(
6019 result_ptr,
6020 result_ty.abiAlignment(zcu),
6021 ty_op.operand,
6022 array_info.elem_type,
6023 len_bytes_llvm,
6024 .normal,
6025 false,
6026 !self.needMemsetWorkaround(len_bytes),
6027 );
6028
6029 if (array_info.sentinel) |sent_val| {
6030 const sent_ptr = try self.ptraddConst(result_ptr, len_bytes);
6031 const sent_elem = try self.resolveValue(sent_val);
6032 try self.store(sent_ptr, .none, sent_elem.toValue(), array_info.elem_type, .normal);
6033 }
6034
6035 return result_ptr;
6036 },
6037 else => unreachable,
6038 }
59876039}
59886040
59896041fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
src/codegen/riscv64/CodeGen.zig+1
......@@ -59,6 +59,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
5959 .expand_sub_safe,
6060 .expand_mul_safe,
6161
62 .expand_array_splat,
6263 .expand_array_to_vector,
6364 });
6465}
src/codegen/sparc64/CodeGen.zig+1
......@@ -42,6 +42,7 @@ const InnerError = codegen.Error || error{OutOfRegisters};
4242
4343pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
4444 return comptime &.initMany(&.{
45 .expand_array_splat,
4546 .expand_array_to_vector,
4647 });
4748}
src/codegen/spirv/CodeGen.zig+1
......@@ -129,6 +129,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
129129 .expand_sub_safe,
130130 .expand_mul_safe,
131131
132 .expand_array_splat,
132133 .expand_array_to_vector,
133134 });
134135}
src/codegen/wasm/CodeGen.zig+1
......@@ -38,6 +38,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
3838 .expand_packed_store,
3939 .expand_packed_agg_field_val,
4040 .expand_packed_aggregate_init,
41 .expand_array_splat,
4142 .expand_array_to_vector,
4243
4344 .scalarize_add,
src/codegen/x86_64/CodeGen.zig+1
......@@ -78,6 +78,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
7878 .expand_packed_store,
7979 .expand_packed_agg_field_val,
8080 .expand_packed_aggregate_init,
81 .expand_array_splat,
8182 .expand_array_to_vector,
8283 });
8384}
test/behavior/memset.zig+22
......@@ -30,6 +30,28 @@ fn testMemsetArray() !void {
3030 }
3131}
3232
33test "@memset preserves array sentinel" {
34 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
35 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
36 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
37
38 try testMemsetArraySentinel();
39 try comptime testMemsetArraySentinel();
40}
41
42fn testMemsetArraySentinel() !void {
43 var value: u32 = 42;
44 _ = &value;
45 var array: [3:0]u32 = .{ 1, 2, 3 };
46
47 @memset(&array, value);
48
49 try expect(array[0] == value);
50 try expect(array[1] == value);
51 try expect(array[2] == value);
52 try expect(array[3] == 0);
53}
54
3355test "@memset on slices" {
3456 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
3557 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;