authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-26 13:41:02-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-28 13:24:42-07:00
log9295355985202c267b4326b5a6e2ad5158b48e5d
tree31fd1162358b568f9adb34f11cc3e8e2074c6987
parent51adbf472bcf9eacc0099e39778a6f9177fea023

LLVM backend: optimize memset with comptime-known element

When the element is comptime-known, we can check if it has a repeated byte representation. In this case, `@memset` can be lowered with the LLVM intrinsic rather than with a loop.

4 files changed, 83 insertions(+), 19 deletions(-)

src/Sema.zig+3
...@@ -26953,9 +26953,11 @@ fn storePtrVal(...@@ -26953,9 +26953,11 @@ fn storePtrVal(
26953 defer sema.gpa.free(buffer);26953 defer sema.gpa.free(buffer);
26954 reinterpret.val_ptr.*.writeToMemory(mut_kit.ty, sema.mod, buffer) catch |err| switch (err) {26954 reinterpret.val_ptr.*.writeToMemory(mut_kit.ty, sema.mod, buffer) catch |err| switch (err) {
26955 error.ReinterpretDeclRef => unreachable,26955 error.ReinterpretDeclRef => unreachable,
26956 error.IllDefinedMemoryLayout => unreachable, // Sema was supposed to emit a compile error already
26956 };26957 };
26957 operand_val.writeToMemory(operand_ty, sema.mod, buffer[reinterpret.byte_offset..]) catch |err| switch (err) {26958 operand_val.writeToMemory(operand_ty, sema.mod, buffer[reinterpret.byte_offset..]) catch |err| switch (err) {
26958 error.ReinterpretDeclRef => unreachable,26959 error.ReinterpretDeclRef => unreachable,
26960 error.IllDefinedMemoryLayout => unreachable, // Sema was supposed to emit a compile error already
26959 };26961 };
2696026962
26961 const arena = mut_kit.beginArena(sema.mod);26963 const arena = mut_kit.beginArena(sema.mod);
...@@ -27905,6 +27907,7 @@ fn bitCastVal(...@@ -27905,6 +27907,7 @@ fn bitCastVal(
27905 defer sema.gpa.free(buffer);27907 defer sema.gpa.free(buffer);
27906 val.writeToMemory(old_ty, sema.mod, buffer) catch |err| switch (err) {27908 val.writeToMemory(old_ty, sema.mod, buffer) catch |err| switch (err) {
27907 error.ReinterpretDeclRef => return null,27909 error.ReinterpretDeclRef => return null,
27910 error.IllDefinedMemoryLayout => unreachable, // Sema was supposed to emit a compile error already
27908 };27911 };
27909 return try Value.readFromMemory(new_ty, sema.mod, buffer[buffer_offset..], sema.arena);27912 return try Value.readFromMemory(new_ty, sema.mod, buffer[buffer_offset..], sema.arena);
27910}27913}
src/codegen/llvm.zig+32-15
...@@ -8424,28 +8424,45 @@ pub const FuncGen = struct {...@@ -8424,28 +8424,45 @@ pub const FuncGen = struct {
8424 const dest_slice = try self.resolveInst(bin_op.lhs);8424 const dest_slice = try self.resolveInst(bin_op.lhs);
8425 const ptr_ty = self.air.typeOf(bin_op.lhs);8425 const ptr_ty = self.air.typeOf(bin_op.lhs);
8426 const elem_ty = self.air.typeOf(bin_op.rhs);8426 const elem_ty = self.air.typeOf(bin_op.rhs);
8427 const target = self.dg.module.getTarget();8427 const module = self.dg.module;
8428 const val_is_undef = if (self.air.value(bin_op.rhs)) |val| val.isUndefDeep() else false;8428 const target = module.getTarget();
8429 const dest_ptr_align = ptr_ty.ptrAlignment(target);8429 const dest_ptr_align = ptr_ty.ptrAlignment(target);
8430 const u8_llvm_ty = self.context.intType(8);8430 const u8_llvm_ty = self.context.intType(8);
8431 const dest_ptr = self.sliceOrArrayPtr(dest_slice, ptr_ty);8431 const dest_ptr = self.sliceOrArrayPtr(dest_slice, ptr_ty);
8432 const is_volatile = ptr_ty.isVolatilePtr();8432 const is_volatile = ptr_ty.isVolatilePtr();
84338433
8434 if (val_is_undef) {8434 if (self.air.value(bin_op.rhs)) |elem_val| {
8435 // Even if safety is disabled, we still emit a memset to undefined since it conveys8435 if (elem_val.isUndefDeep()) {
8436 // extra information to LLVM. However, safety makes the difference between using8436 // Even if safety is disabled, we still emit a memset to undefined since it conveys
8437 // 0xaa or actual undefined for the fill byte.8437 // extra information to LLVM. However, safety makes the difference between using
8438 const fill_byte = if (safety)8438 // 0xaa or actual undefined for the fill byte.
8439 u8_llvm_ty.constInt(0xaa, .False)8439 const fill_byte = if (safety)
8440 else8440 u8_llvm_ty.constInt(0xaa, .False)
8441 u8_llvm_ty.getUndef();8441 else
8442 const len = self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);8442 u8_llvm_ty.getUndef();
8443 _ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, is_volatile);8443 const len = self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
8444 _ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, is_volatile);
84448445
8445 if (safety and self.dg.module.comp.bin_file.options.valgrind) {8446 if (safety and module.comp.bin_file.options.valgrind) {
8446 self.valgrindMarkUndef(dest_ptr, len);8447 self.valgrindMarkUndef(dest_ptr, len);
8448 }
8449 return null;
8450 }
8451
8452 // Test if the element value is compile-time known to be a
8453 // repeating byte pattern, for example, `@as(u64, 0)` has a
8454 // repeating byte pattern of 0 bytes. In such case, the memset
8455 // intrinsic can be used.
8456 var value_buffer: Value.Payload.U64 = undefined;
8457 if (try elem_val.hasRepeatedByteRepr(elem_ty, module, &value_buffer)) |byte_val| {
8458 const fill_byte = try self.resolveValue(.{
8459 .ty = Type.u8,
8460 .val = byte_val,
8461 });
8462 const len = self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
8463 _ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, is_volatile);
8464 return null;
8447 }8465 }
8448 return null;
8449 }8466 }
84508467
8451 const value = try self.resolveInst(bin_op.rhs);8468 const value = try self.resolveInst(bin_op.rhs);
src/value.zig+35-3
...@@ -1278,7 +1278,10 @@ pub const Value = extern union {...@@ -1278,7 +1278,10 @@ pub const Value = extern union {
1278 ///1278 ///
1279 /// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past1279 /// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
1280 /// the end of the value in memory.1280 /// the end of the value in memory.
1281 pub fn writeToMemory(val: Value, ty: Type, mod: *Module, buffer: []u8) error{ReinterpretDeclRef}!void {1281 pub fn writeToMemory(val: Value, ty: Type, mod: *Module, buffer: []u8) error{
1282 ReinterpretDeclRef,
1283 IllDefinedMemoryLayout,
1284 }!void {
1282 const target = mod.getTarget();1285 const target = mod.getTarget();
1283 const endian = target.cpu.arch.endian();1286 const endian = target.cpu.arch.endian();
1284 if (val.isUndef()) {1287 if (val.isUndef()) {
...@@ -1345,7 +1348,7 @@ pub const Value = extern union {...@@ -1345,7 +1348,7 @@ pub const Value = extern union {
1345 return writeToPackedMemory(val, ty, mod, buffer[0..byte_count], 0);1348 return writeToPackedMemory(val, ty, mod, buffer[0..byte_count], 0);
1346 },1349 },
1347 .Struct => switch (ty.containerLayout()) {1350 .Struct => switch (ty.containerLayout()) {
1348 .Auto => unreachable, // Sema is supposed to have emitted a compile error already1351 .Auto => return error.IllDefinedMemoryLayout,
1349 .Extern => {1352 .Extern => {
1350 const fields = ty.structFields().values();1353 const fields = ty.structFields().values();
1351 const field_vals = val.castTag(.aggregate).?.data;1354 const field_vals = val.castTag(.aggregate).?.data;
...@@ -1366,7 +1369,7 @@ pub const Value = extern union {...@@ -1366,7 +1369,7 @@ pub const Value = extern union {
1366 std.mem.writeInt(Int, buffer[0..@sizeOf(Int)], @intCast(Int, int), endian);1369 std.mem.writeInt(Int, buffer[0..@sizeOf(Int)], @intCast(Int, int), endian);
1367 },1370 },
1368 .Union => switch (ty.containerLayout()) {1371 .Union => switch (ty.containerLayout()) {
1369 .Auto => unreachable,1372 .Auto => return error.IllDefinedMemoryLayout,
1370 .Extern => @panic("TODO implement writeToMemory for extern unions"),1373 .Extern => @panic("TODO implement writeToMemory for extern unions"),
1371 .Packed => {1374 .Packed => {
1372 const byte_count = (@intCast(usize, ty.bitSize(target)) + 7) / 8;1375 const byte_count = (@intCast(usize, ty.bitSize(target)) + 7) / 8;
...@@ -5381,6 +5384,35 @@ pub const Value = extern union {...@@ -5381,6 +5384,35 @@ pub const Value = extern union {
5381 }5384 }
5382 }5385 }
53835386
5387 /// If the value is represented in-memory as a series of bytes that all
5388 /// have the same value, return that byte value, otherwise null.
5389 pub fn hasRepeatedByteRepr(val: Value, ty: Type, mod: *Module, value_buffer: *Payload.U64) !?Value {
5390 const target = mod.getTarget();
5391 const abi_size = ty.abiSize(target);
5392 assert(abi_size >= 1);
5393 const byte_buffer = try mod.gpa.alloc(u8, abi_size);
5394 defer mod.gpa.free(byte_buffer);
5395
5396 writeToMemory(val, ty, mod, byte_buffer) catch |err| switch (err) {
5397 error.ReinterpretDeclRef => return null,
5398 // TODO: The writeToMemory function was originally created for the purpose
5399 // of comptime pointer casting. However, it is now additionally being used
5400 // for checking the actual memory layout that will be generated by machine
5401 // code late in compilation. So, this error handling is too aggressive and
5402 // causes some false negatives, causing less-than-ideal code generation.
5403 error.IllDefinedMemoryLayout => return null,
5404 };
5405 const first_byte = byte_buffer[0];
5406 for (byte_buffer[1..]) |byte| {
5407 if (byte != first_byte) return null;
5408 }
5409 value_buffer.* = .{
5410 .base = .{ .tag = .int_u64 },
5411 .data = first_byte,
5412 };
5413 return initPayload(&value_buffer.base);
5414 }
5415
5384 /// This type is not copyable since it may contain pointers to its inner data.5416 /// This type is not copyable since it may contain pointers to its inner data.
5385 pub const Payload = struct {5417 pub const Payload = struct {
5386 tag: Tag,5418 tag: Tag,
test/behavior/memset.zig+13-1
...@@ -94,7 +94,7 @@ test "memset with 1-byte array element" {...@@ -94,7 +94,7 @@ test "memset with 1-byte array element" {
94 try expect(buf[4][0]);94 try expect(buf[4][0]);
95}95}
9696
97test "memset with large array element" {97test "memset with large array element, runtime known" {
98 const A = [128]u64;98 const A = [128]u64;
99 var buf: [5]A = undefined;99 var buf: [5]A = undefined;
100 var runtime_known_element = [_]u64{0} ** 128;100 var runtime_known_element = [_]u64{0} ** 128;
...@@ -106,6 +106,18 @@ test "memset with large array element" {...@@ -106,6 +106,18 @@ test "memset with large array element" {
106 for (buf[4]) |elem| try expect(elem == 0);106 for (buf[4]) |elem| try expect(elem == 0);
107}107}
108108
109test "memset with large array element, comptime known" {
110 const A = [128]u64;
111 var buf: [5]A = undefined;
112 const comptime_known_element = [_]u64{0} ** 128;
113 @memset(&buf, comptime_known_element);
114 for (buf[0]) |elem| try expect(elem == 0);
115 for (buf[1]) |elem| try expect(elem == 0);
116 for (buf[2]) |elem| try expect(elem == 0);
117 for (buf[3]) |elem| try expect(elem == 0);
118 for (buf[4]) |elem| try expect(elem == 0);
119}
120
109test "memcpy and memset intrinsics" {121test "memcpy and memset intrinsics" {
110 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;122 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
111 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;123 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;