authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-21 23:05:36-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-25 11:23:41-07:00
log057c950093085e392fcdd6d6c8e7fb4356dd9959
treef47a5292c6a6cba0800a0ab6ee8ed6cfdfd7c6bc
parent25d11283b7b79edd383163cb2a72bd79dcf02dba

LLVM backend: support non-byte-sized memset

Also introduce memset_safe AIR tag and support it in C backend and LLVM backend.

14 files changed, 191 insertions(+), 30 deletions(-)

src/Air.zig+9
......@@ -638,7 +638,14 @@ pub const Inst = struct {
638638 /// The element type may be any type, and the slice may have any alignment.
639639 /// Result type is always void.
640640 /// Uses the `bin_op` field. LHS is the dest slice. RHS is the element value.
641 /// The element value may be undefined, in which case the destination
642 /// memory region has undefined bytes after this function executes. In
643 /// such case ignoring this instruction is legal lowering.
641644 memset,
645 /// Same as `memset`, except if the element value is undefined, the memory region
646 /// should be filled with 0xaa bytes, and any other safety metadata such as Valgrind
647 /// integrations should be notified of this memory region being undefined.
648 memset_safe,
642649 /// Given dest pointer and source pointer, copy elements from source to dest.
643650 /// Dest pointer is either a slice or a pointer to array.
644651 /// The dest element type may be any type.
......@@ -1236,6 +1243,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
12361243 .atomic_store_release,
12371244 .atomic_store_seq_cst,
12381245 .memset,
1246 .memset_safe,
12391247 .memcpy,
12401248 .set_union_tag,
12411249 .prefetch,
......@@ -1415,6 +1423,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index) bool {
14151423 .errunion_payload_ptr_set,
14161424 .set_union_tag,
14171425 .memset,
1426 .memset_safe,
14181427 .memcpy,
14191428 .cmpxchg_weak,
14201429 .cmpxchg_strong,
src/Liveness.zig+2
......@@ -305,6 +305,7 @@ pub fn categorizeOperand(
305305 .atomic_store_seq_cst,
306306 .set_union_tag,
307307 .memset,
308 .memset_safe,
308309 .memcpy,
309310 => {
310311 const o = air_datas[inst].bin_op;
......@@ -980,6 +981,7 @@ fn analyzeInst(
980981 .min,
981982 .max,
982983 .memset,
984 .memset_safe,
983985 .memcpy,
984986 => {
985987 const o = inst_datas[inst].bin_op;
src/Liveness/Verify.zig+1
......@@ -255,6 +255,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
255255 .min,
256256 .max,
257257 .memset,
258 .memset_safe,
258259 .memcpy,
259260 => {
260261 const bin_op = data[inst].bin_op;
src/Sema.zig+1-1
......@@ -21972,7 +21972,7 @@ fn zirMemset(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
2197221972
2197321973 try sema.requireRuntimeBlock(block, src, runtime_src);
2197421974 _ = try block.addInst(.{
21975 .tag = .memset,
21975 .tag = if (block.wantSafety()) .memset_safe else .memset,
2197621976 .data = .{ .bin_op = .{
2197721977 .lhs = dest_ptr,
2197821978 .rhs = elem,
src/arch/aarch64/CodeGen.zig+8-2
......@@ -775,7 +775,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
775775 .atomic_rmw => try self.airAtomicRmw(inst),
776776 .atomic_load => try self.airAtomicLoad(inst),
777777 .memcpy => try self.airMemcpy(inst),
778 .memset => try self.airMemset(inst),
778 .memset => try self.airMemset(inst, false),
779 .memset_safe => try self.airMemset(inst, true),
779780 .set_union_tag => try self.airSetUnionTag(inst),
780781 .get_union_tag => try self.airGetUnionTag(inst),
781782 .clz => try self.airClz(inst),
......@@ -5975,8 +5976,13 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
59755976 return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
59765977}
59775978
5978fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
5979fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
59795980 _ = inst;
5981 if (safety) {
5982 // TODO if the value is undef, write 0xaa bytes to dest
5983 } else {
5984 // TODO if the value is undef, don't lower this instruction
5985 }
59805986 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
59815987}
59825988
src/arch/arm/CodeGen.zig+8-2
......@@ -759,7 +759,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
759759 .atomic_rmw => try self.airAtomicRmw(inst),
760760 .atomic_load => try self.airAtomicLoad(inst),
761761 .memcpy => try self.airMemcpy(inst),
762 .memset => try self.airMemset(inst),
762 .memset => try self.airMemset(inst, false),
763 .memset_safe => try self.airMemset(inst, true),
763764 .set_union_tag => try self.airSetUnionTag(inst),
764765 .get_union_tag => try self.airGetUnionTag(inst),
765766 .clz => try self.airClz(inst),
......@@ -5921,7 +5922,12 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
59215922 return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
59225923}
59235924
5924fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
5925fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
5926 if (safety) {
5927 // TODO if the value is undef, write 0xaa bytes to dest
5928 } else {
5929 // TODO if the value is undef, don't lower this instruction
5930 }
59255931 _ = inst;
59265932 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
59275933}
src/arch/riscv64/CodeGen.zig+8-2
......@@ -589,7 +589,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
589589 .atomic_rmw => try self.airAtomicRmw(inst),
590590 .atomic_load => try self.airAtomicLoad(inst),
591591 .memcpy => try self.airMemcpy(inst),
592 .memset => try self.airMemset(inst),
592 .memset => try self.airMemset(inst, false),
593 .memset_safe => try self.airMemset(inst, true),
593594 .set_union_tag => try self.airSetUnionTag(inst),
594595 .get_union_tag => try self.airGetUnionTag(inst),
595596 .clz => try self.airClz(inst),
......@@ -2421,8 +2422,13 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
24212422 return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
24222423}
24232424
2424fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
2425fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
24252426 _ = inst;
2427 if (safety) {
2428 // TODO if the value is undef, write 0xaa bytes to dest
2429 } else {
2430 // TODO if the value is undef, don't lower this instruction
2431 }
24262432 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
24272433}
24282434
src/arch/sparc64/CodeGen.zig+8-2
......@@ -605,7 +605,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
605605 .atomic_rmw => try self.airAtomicRmw(inst),
606606 .atomic_load => try self.airAtomicLoad(inst),
607607 .memcpy => @panic("TODO try self.airMemcpy(inst)"),
608 .memset => try self.airMemset(inst),
608 .memset => try self.airMemset(inst, false),
609 .memset_safe => try self.airMemset(inst, true),
609610 .set_union_tag => try self.airSetUnionTag(inst),
610611 .get_union_tag => try self.airGetUnionTag(inst),
611612 .clz => try self.airClz(inst),
......@@ -1764,7 +1765,12 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
17641765 return self.finishAirBookkeeping();
17651766}
17661767
1767fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
1768fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1769 if (safety) {
1770 // TODO if the value is undef, write 0xaa bytes to dest
1771 } else {
1772 // TODO if the value is undef, don't lower this instruction
1773 }
17681774 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
17691775 const extra = self.air.extraData(Air.Bin, pl_op.payload);
17701776
src/arch/wasm/CodeGen.zig+2-1
......@@ -1883,7 +1883,8 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
18831883
18841884 .load => func.airLoad(inst),
18851885 .loop => func.airLoop(inst),
1886 .memset => func.airMemset(inst),
1886 // TODO: elide memset when writing undef without safety
1887 .memset, .memset_safe => func.airMemset(inst),
18871888 .not => func.airNot(inst),
18881889 .optional_payload => func.airOptionalPayload(inst),
18891890 .optional_payload_ptr => func.airOptionalPayloadPtr(inst),
src/arch/x86_64/CodeGen.zig+9-2
......@@ -1046,7 +1046,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
10461046 .atomic_rmw => try self.airAtomicRmw(inst),
10471047 .atomic_load => try self.airAtomicLoad(inst),
10481048 .memcpy => try self.airMemcpy(inst),
1049 .memset => try self.airMemset(inst),
1049 .memset => try self.airMemset(inst, false),
1050 .memset_safe => try self.airMemset(inst, true),
10501051 .set_union_tag => try self.airSetUnionTag(inst),
10511052 .get_union_tag => try self.airGetUnionTag(inst),
10521053 .clz => try self.airClz(inst),
......@@ -8149,7 +8150,13 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
81498150 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
81508151}
81518152
8152fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
8153fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8154 if (safety) {
8155 // TODO if the value is undef, write 0xaa bytes to dest
8156 } else {
8157 // TODO if the value is undef, don't lower this instruction
8158 }
8159
81538160 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
81548161
81558162 const dst_ptr = try self.resolveInst(bin_op.lhs);
src/codegen/c.zig+9-4
......@@ -2925,7 +2925,8 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
29252925 .cmpxchg_strong => try airCmpxchg(f, inst, "strong"),
29262926 .atomic_rmw => try airAtomicRmw(f, inst),
29272927 .atomic_load => try airAtomicLoad(f, inst),
2928 .memset => try airMemset(f, inst),
2928 .memset => try airMemset(f, inst, false),
2929 .memset_safe => try airMemset(f, inst, true),
29292930 .memcpy => try airMemcpy(f, inst),
29302931 .set_union_tag => try airSetUnionTag(f, inst),
29312932 .get_union_tag => try airGetUnionTag(f, inst),
......@@ -6189,7 +6190,7 @@ fn writeSliceOrPtr(f: *Function, writer: anytype, ptr: CValue, ptr_ty: Type) !vo
61896190 }
61906191}
61916192
6192fn airMemset(f: *Function, inst: Air.Inst.Index) !CValue {
6193fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
61936194 const bin_op = f.air.instructions.items(.data)[inst].bin_op;
61946195 const dest_ty = f.air.typeOf(bin_op.lhs);
61956196 const dest_slice = try f.resolveInst(bin_op.lhs);
......@@ -6201,6 +6202,11 @@ fn airMemset(f: *Function, inst: Air.Inst.Index) !CValue {
62016202 const writer = f.object.writer();
62026203
62036204 if (val_is_undef) {
6205 if (!safety) {
6206 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
6207 return .none;
6208 }
6209
62046210 try writer.writeAll("memset(");
62056211 switch (dest_ty.ptrSize()) {
62066212 .Slice => {
......@@ -6242,8 +6248,7 @@ fn airMemset(f: *Function, inst: Air.Inst.Index) !CValue {
62426248 },
62436249 .One => {
62446250 const array_ty = dest_ty.childType();
6245 const len = array_ty.arrayLen() * elem_abi_size;
6246 try writer.print("{d}", .{len});
6251 try writer.print("{d}", .{array_ty.arrayLen()});
62476252 },
62486253 .Many, .C => unreachable,
62496254 }
src/codegen/llvm.zig+81-14
......@@ -4672,7 +4672,8 @@ pub const FuncGen = struct {
46724672 .fence => try self.airFence(inst),
46734673 .atomic_rmw => try self.airAtomicRmw(inst),
46744674 .atomic_load => try self.airAtomicLoad(inst),
4675 .memset => try self.airMemset(inst),
4675 .memset => try self.airMemset(inst, false),
4676 .memset_safe => try self.airMemset(inst, true),
46764677 .memcpy => try self.airMemcpy(inst),
46774678 .set_union_tag => try self.airSetUnionTag(inst),
46784679 .get_union_tag => try self.airGetUnionTag(inst),
......@@ -8405,29 +8406,95 @@ pub const FuncGen = struct {
84058406 return null;
84068407 }
84078408
8408 fn airMemset(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value {
8409 fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !?*llvm.Value {
84098410 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
84108411 const dest_slice = try self.resolveInst(bin_op.lhs);
84118412 const ptr_ty = self.air.typeOf(bin_op.lhs);
8412 const value = try self.resolveInst(bin_op.rhs);
84138413 const elem_ty = self.air.typeOf(bin_op.rhs);
84148414 const target = self.dg.module.getTarget();
84158415 const val_is_undef = if (self.air.value(bin_op.rhs)) |val| val.isUndefDeep() else false;
8416 const len = self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
8417 const dest_ptr = self.sliceOrArrayPtr(dest_slice, ptr_ty);
8416 const dest_ptr_align = ptr_ty.ptrAlignment(target);
84188417 const u8_llvm_ty = self.context.intType(8);
8419 const fill_byte = if (val_is_undef) u8_llvm_ty.constInt(0xaa, .False) else b: {
8420 if (elem_ty.abiSize(target) != 1) {
8421 return self.dg.todo("implement @memset for non-byte-sized element type", .{});
8418 const dest_ptr = self.sliceOrArrayPtr(dest_slice, ptr_ty);
8419
8420 if (val_is_undef) {
8421 // Even if safety is disabled, we still emit a memset to undefined since it conveys
8422 // extra information to LLVM. However, safety makes the difference between using
8423 // 0xaa or actual undefined for the fill byte.
8424 const fill_byte = if (safety)
8425 u8_llvm_ty.constInt(0xaa, .False)
8426 else
8427 u8_llvm_ty.getUndef();
8428 const len = self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
8429 _ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, ptr_ty.isVolatilePtr());
8430
8431 if (safety and self.dg.module.comp.bin_file.options.valgrind) {
8432 self.valgrindMarkUndef(dest_ptr, len);
84228433 }
8423 break :b self.builder.buildBitCast(value, u8_llvm_ty, "");
8424 };
8425 const dest_ptr_align = ptr_ty.ptrAlignment(target);
8426 _ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, ptr_ty.isVolatilePtr());
8434 return null;
8435 }
8436
8437 const value = try self.resolveInst(bin_op.rhs);
8438 const elem_abi_size = elem_ty.abiSize(target);
84278439
8428 if (val_is_undef and self.dg.module.comp.bin_file.options.valgrind) {
8429 self.valgrindMarkUndef(dest_ptr, len);
8440 if (elem_abi_size == 1) {
8441 // In this case we can take advantage of LLVM's intrinsic.
8442 const fill_byte = self.builder.buildBitCast(value, u8_llvm_ty, "");
8443 const len = self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
8444 _ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, ptr_ty.isVolatilePtr());
8445 return null;
84308446 }
8447
8448 // non-byte-sized element. lower with a loop. something like this:
8449
8450 // entry:
8451 // ...
8452 // %end_ptr = getelementptr %ptr, %len
8453 // br loop
8454 // loop:
8455 // %it_ptr = phi body %next_ptr, entry %ptr
8456 // %end = cmp eq %it_ptr, %end_ptr
8457 // cond_br %end body, end
8458 // body:
8459 // store %it_ptr, %value
8460 // %next_ptr = getelementptr %it_ptr, 1
8461 // br loop
8462 // end:
8463 // ...
8464 const entry_block = self.builder.getInsertBlock();
8465 const loop_block = self.context.appendBasicBlock(self.llvm_func, "InlineMemsetLoop");
8466 const body_block = self.context.appendBasicBlock(self.llvm_func, "InlineMemsetBody");
8467 const end_block = self.context.appendBasicBlock(self.llvm_func, "InlineMemsetEnd");
8468
8469 const llvm_usize_ty = self.context.intType(target.cpu.arch.ptrBitWidth());
8470 const len = switch (ptr_ty.ptrSize()) {
8471 .Slice => self.builder.buildExtractValue(dest_slice, 1, ""),
8472 .One => llvm_usize_ty.constInt(ptr_ty.childType().arrayLen(), .False),
8473 .Many, .C => unreachable,
8474 };
8475 const elem_llvm_ty = try self.dg.lowerType(elem_ty);
8476 const len_gep = [_]*llvm.Value{len};
8477 const end_ptr = self.builder.buildInBoundsGEP(elem_llvm_ty, dest_ptr, &len_gep, len_gep.len, "");
8478 _ = self.builder.buildBr(loop_block);
8479
8480 self.builder.positionBuilderAtEnd(loop_block);
8481 const it_ptr = self.builder.buildPhi(self.context.pointerType(0), "");
8482 const end = self.builder.buildICmp(.NE, it_ptr, end_ptr, "");
8483 _ = self.builder.buildCondBr(end, body_block, end_block);
8484
8485 self.builder.positionBuilderAtEnd(body_block);
8486 const store_inst = self.builder.buildStore(value, it_ptr);
8487 store_inst.setAlignment(@min(elem_ty.abiAlignment(target), dest_ptr_align));
8488 const one_gep = [_]*llvm.Value{llvm_usize_ty.constInt(1, .False)};
8489 const next_ptr = self.builder.buildInBoundsGEP(elem_llvm_ty, it_ptr, &one_gep, one_gep.len, "");
8490 _ = self.builder.buildBr(loop_block);
8491
8492 self.builder.positionBuilderAtEnd(end_block);
8493
8494 const incoming_values: [2]*llvm.Value = .{ next_ptr, dest_ptr };
8495 const incoming_blocks: [2]*llvm.BasicBlock = .{ body_block, entry_block };
8496 it_ptr.addIncoming(&incoming_values, &incoming_blocks, 2);
8497
84318498 return null;
84328499 }
84338500
src/print_air.zig+1
......@@ -171,6 +171,7 @@ const Writer = struct {
171171 .cmp_neq_optimized,
172172 .memcpy,
173173 .memset,
174 .memset_safe,
174175 => try w.writeBinOp(s, inst),
175176
176177 .is_null,
test/behavior/basic.zig+44
......@@ -353,6 +353,50 @@ fn f2(x: bool) []const u8 {
353353 return (if (x) &fA else &fB)();
354354}
355355
356test "@memset on array pointers" {
357 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
358 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
359 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
360
361 try testMemsetArray();
362 // TODO this doesn't pass yet
363 // try comptime testMemsetArray();
364}
365
366fn testMemsetArray() !void {
367 {
368 // memset array to non-undefined, ABI size == 1
369 var foo: [20]u8 = undefined;
370 @memset(&foo, 'A');
371 try expect(foo[0] == 'A');
372 try expect(foo[11] == 'A');
373 try expect(foo[19] == 'A');
374
375 // memset array to undefined, ABI size == 1
376 @setRuntimeSafety(true);
377 @memset(&foo, undefined);
378 try expect(foo[0] == 0xaa);
379 try expect(foo[11] == 0xaa);
380 try expect(foo[19] == 0xaa);
381 }
382
383 {
384 // memset array to non-undefined, ABI size > 1
385 var foo: [20]u32 = undefined;
386 @memset(&foo, 1234);
387 try expect(foo[0] == 1234);
388 try expect(foo[11] == 1234);
389 try expect(foo[19] == 1234);
390
391 // memset array to undefined, ABI size > 1
392 @setRuntimeSafety(true);
393 @memset(&foo, undefined);
394 try expect(foo[0] == 0xaaaaaaaa);
395 try expect(foo[11] == 0xaaaaaaaa);
396 try expect(foo[19] == 0xaaaaaaaa);
397 }
398}
399
356400test "memcpy and memset intrinsics" {
357401 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
358402 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;