authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-05-18 17:26:20+02:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-05-19 20:22:45+02:00
log4a33aa922e90b76248b259a89be86966eb4898c2
tree9e0320d8d8cc8e4d63fc341b6f1db591a7d632c0
parent55a260c968aed32001df5355596331db38b13729
signature Commit is signed but in an unrecognized format.

wasm: support `memset` for elem abi size > 1

Previously we incorrectly assumed all memset's to have its element abi-size be 1 byte. This would set the region of memory incorrectly. We now have a more efficient loop, as well as support any element type by re-using the `store` function for each element and moving the pointer by 1 element.

2 files changed, 108 insertions(+), 94 deletions(-)

src/arch/wasm/CodeGen.zig+108-80
...@@ -1605,10 +1605,16 @@ fn memcpy(func: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {...@@ -1605,10 +1605,16 @@ fn memcpy(func: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1605 else => {},1605 else => {},
1606 }1606 }
16071607
1608 // TODO: We should probably lower this to a call to compiler_rt1608 // allocate a local for the offset, and set it to 0.
1609 // But for now, we implement it manually1609 // This to ensure that inside loops we correctly re-set the counter.
1610 var offset = try func.ensureAllocLocal(Type.usize); // local for counter1610 var offset = try func.allocLocal(Type.usize); // local for counter
1611 defer offset.free(func);1611 defer offset.free(func);
1612 switch (func.arch()) {
1613 .wasm32 => try func.addImm32(0),
1614 .wasm64 => try func.addImm64(0),
1615 else => unreachable,
1616 }
1617 try func.addLabel(.local_set, offset.local.value);
16121618
1613 // outer block to jump to when loop is done1619 // outer block to jump to when loop is done
1614 try func.startBlock(.block, wasm.block_empty);1620 try func.startBlock(.block, wasm.block_empty);
...@@ -3301,19 +3307,23 @@ fn airCondBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -3301,19 +3307,23 @@ fn airCondBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3301 {3307 {
3302 func.branches.appendAssumeCapacity(.{});3308 func.branches.appendAssumeCapacity(.{});
3303 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_condbr.else_deaths.len));3309 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_condbr.else_deaths.len));
3310 defer {
3311 var else_stack = func.branches.pop();
3312 else_stack.deinit(func.gpa);
3313 }
3304 try func.genBody(else_body);3314 try func.genBody(else_body);
3305 try func.endBlock();3315 try func.endBlock();
3306 var else_stack = func.branches.pop();
3307 else_stack.deinit(func.gpa);
3308 }3316 }
33093317
3310 // Outer block that matches the condition3318 // Outer block that matches the condition
3311 {3319 {
3312 func.branches.appendAssumeCapacity(.{});3320 func.branches.appendAssumeCapacity(.{});
3313 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_condbr.then_deaths.len));3321 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, @intCast(u32, liveness_condbr.then_deaths.len));
3322 defer {
3323 var then_stack = func.branches.pop();
3324 then_stack.deinit(func.gpa);
3325 }
3314 try func.genBody(then_body);3326 try func.genBody(then_body);
3315 var then_stack = func.branches.pop();
3316 then_stack.deinit(func.gpa);
3317 }3327 }
33183328
3319 func.finishAir(inst, .none, &.{});3329 func.finishAir(inst, .none, &.{});
...@@ -3829,20 +3839,24 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -3829,20 +3839,24 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3829 }3839 }
3830 func.branches.appendAssumeCapacity(.{});3840 func.branches.appendAssumeCapacity(.{});
3831 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.deaths[index].len);3841 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.deaths[index].len);
3842 defer {
3843 var case_branch = func.branches.pop();
3844 case_branch.deinit(func.gpa);
3845 }
3832 try func.genBody(case.body);3846 try func.genBody(case.body);
3833 try func.endBlock();3847 try func.endBlock();
3834 var case_branch = func.branches.pop();
3835 case_branch.deinit(func.gpa);
3836 }3848 }
38373849
3838 if (has_else_body) {3850 if (has_else_body) {
3839 func.branches.appendAssumeCapacity(.{});3851 func.branches.appendAssumeCapacity(.{});
3840 const else_deaths = liveness.deaths.len - 1;3852 const else_deaths = liveness.deaths.len - 1;
3841 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.deaths[else_deaths].len);3853 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.deaths[else_deaths].len);
3854 defer {
3855 var else_branch = func.branches.pop();
3856 else_branch.deinit(func.gpa);
3857 }
3842 try func.genBody(else_body);3858 try func.genBody(else_body);
3843 try func.endBlock();3859 try func.endBlock();
3844 var else_branch = func.branches.pop();
3845 else_branch.deinit(func.gpa);
3846 }3860 }
3847 func.finishAir(inst, .none, &.{});3861 func.finishAir(inst, .none, &.{});
3848}3862}
...@@ -3971,7 +3985,7 @@ fn airWrapErrUnionErr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -3971,7 +3985,7 @@ fn airWrapErrUnionErr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3971 // write 'undefined' to the payload3985 // write 'undefined' to the payload
3972 const payload_ptr = try func.buildPointerOffset(err_union, @intCast(u32, errUnionPayloadOffset(pl_ty, func.target)), .new);3986 const payload_ptr = try func.buildPointerOffset(err_union, @intCast(u32, errUnionPayloadOffset(pl_ty, func.target)), .new);
3973 const len = @intCast(u32, err_ty.errorUnionPayload().abiSize(func.target));3987 const len = @intCast(u32, err_ty.errorUnionPayload().abiSize(func.target));
3974 try func.memset(payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaaaaaaaa });3988 try func.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
39753989
3976 break :result err_union;3990 break :result err_union;
3977 };3991 };
...@@ -4466,8 +4480,13 @@ fn airMemset(func: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void...@@ -4466,8 +4480,13 @@ fn airMemset(func: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void
4466 .C, .Many => unreachable,4480 .C, .Many => unreachable,
4467 };4481 };
44684482
4483 const elem_ty = if (ptr_ty.ptrSize() == .One)
4484 ptr_ty.childType().childType()
4485 else
4486 ptr_ty.childType();
4487
4469 const dst_ptr = try func.sliceOrArrayPtr(ptr, ptr_ty);4488 const dst_ptr = try func.sliceOrArrayPtr(ptr, ptr_ty);
4470 try func.memset(dst_ptr, len, value);4489 try func.memset(elem_ty, dst_ptr, len, value);
44714490
4472 func.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });4491 func.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
4473}4492}
...@@ -4476,10 +4495,12 @@ fn airMemset(func: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void...@@ -4476,10 +4495,12 @@ fn airMemset(func: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void
4476/// When the user has enabled the bulk_memory feature, we lower4495/// When the user has enabled the bulk_memory feature, we lower
4477/// this to wasm's memset instruction. When the feature is not present,4496/// this to wasm's memset instruction. When the feature is not present,
4478/// we implement it manually.4497/// we implement it manually.
4479fn memset(func: *CodeGen, ptr: WValue, len: WValue, value: WValue) InnerError!void {4498fn memset(func: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
4499 const abi_size = @intCast(u32, elem_ty.abiSize(func.target));
4500
4480 // When bulk_memory is enabled, we lower it to wasm's memset instruction.4501 // When bulk_memory is enabled, we lower it to wasm's memset instruction.
4481 // If not, we lower it ourselves4502 // If not, we lower it ourselves.
4482 if (std.Target.wasm.featureSetHas(func.target.cpu.features, .bulk_memory)) {4503 if (std.Target.wasm.featureSetHas(func.target.cpu.features, .bulk_memory) and abi_size == 1) {
4483 try func.lowerToStack(ptr);4504 try func.lowerToStack(ptr);
4484 try func.emitWValue(value);4505 try func.emitWValue(value);
4485 try func.emitWValue(len);4506 try func.emitWValue(len);
...@@ -4487,74 +4508,79 @@ fn memset(func: *CodeGen, ptr: WValue, len: WValue, value: WValue) InnerError!vo...@@ -4487,74 +4508,79 @@ fn memset(func: *CodeGen, ptr: WValue, len: WValue, value: WValue) InnerError!vo
4487 return;4508 return;
4488 }4509 }
44894510
4490 // When the length is comptime-known we do the loop at codegen, rather4511 const final_len = switch (len) {
4491 // than emitting a runtime loop into the binary4512 .imm32 => |val| WValue{ .imm32 = val * abi_size },
4492 switch (len) {4513 .imm64 => |val| WValue{ .imm64 = val * abi_size },
4493 .imm32, .imm64 => {4514 else => if (abi_size != 1) blk: {
4494 const length = switch (len) {4515 const new_len = try func.ensureAllocLocal(Type.usize);
4495 .imm32 => |val| val,
4496 .imm64 => |val| val,
4497 else => unreachable,
4498 };
4499
4500 var offset: u32 = 0;
4501 const base = ptr.offset();
4502 while (offset < length) : (offset += 1) {
4503 try func.emitWValue(ptr);
4504 try func.emitWValue(value);
4505 switch (func.arch()) {
4506 .wasm32 => {
4507 try func.addMemArg(.i32_store8, .{ .offset = base + offset, .alignment = 1 });
4508 },
4509 .wasm64 => {
4510 try func.addMemArg(.i64_store8, .{ .offset = base + offset, .alignment = 1 });
4511 },
4512 else => unreachable,
4513 }
4514 }
4515 },
4516 else => {
4517 // TODO: We should probably lower this to a call to compiler_rt
4518 // But for now, we implement it manually
4519 const offset = try func.ensureAllocLocal(Type.usize); // local for counter
4520 // outer block to jump to when loop is done
4521 try func.startBlock(.block, wasm.block_empty);
4522 try func.startBlock(.loop, wasm.block_empty);
4523 try func.emitWValue(offset);
4524 try func.emitWValue(len);4516 try func.emitWValue(len);
4525 switch (func.arch()) {4517 switch (func.arch()) {
4526 .wasm32 => try func.addTag(.i32_eq),4518 .wasm32 => {
4527 .wasm64 => try func.addTag(.i64_eq),4519 try func.emitWValue(.{ .imm32 = abi_size });
4528 else => unreachable,4520 try func.addTag(.i32_mul);
4529 }4521 },
4530 try func.addLabel(.br_if, 1); // jump out of loop into outer block (finished)4522 .wasm64 => {
4531 try func.emitWValue(ptr);4523 try func.emitWValue(.{ .imm64 = abi_size });
4532 try func.emitWValue(offset);4524 try func.addTag(.i64_mul);
4533 switch (func.arch()) {4525 },
4534 .wasm32 => try func.addTag(.i32_add),
4535 .wasm64 => try func.addTag(.i64_add),
4536 else => unreachable,
4537 }
4538 try func.emitWValue(value);
4539 const mem_store_op: Mir.Inst.Tag = switch (func.arch()) {
4540 .wasm32 => .i32_store8,
4541 .wasm64 => .i64_store8,
4542 else => unreachable,
4543 };
4544 try func.addMemArg(mem_store_op, .{ .offset = ptr.offset(), .alignment = 1 });
4545 try func.emitWValue(offset);
4546 try func.addImm32(1);
4547 switch (func.arch()) {
4548 .wasm32 => try func.addTag(.i32_add),
4549 .wasm64 => try func.addTag(.i64_add),
4550 else => unreachable,4526 else => unreachable,
4551 }4527 }
4552 try func.addLabel(.local_set, offset.local.value);4528 try func.addLabel(.local_set, new_len.local.value);
4553 try func.addLabel(.br, 0); // jump to start of loop4529 break :blk new_len;
4554 try func.endBlock();4530 } else len,
4555 try func.endBlock();4531 };
4532
4533 var end_ptr = try func.allocLocal(Type.usize);
4534 defer end_ptr.free(func);
4535 var new_ptr = try func.buildPointerOffset(ptr, 0, .new);
4536 defer new_ptr.free(func);
4537
4538 // get the loop conditional: if current pointer address equals final pointer's address
4539 try func.lowerToStack(ptr);
4540 try func.emitWValue(final_len);
4541 switch (func.arch()) {
4542 .wasm32 => try func.addTag(.i32_add),
4543 .wasm64 => try func.addTag(.i64_add),
4544 else => unreachable,
4545 }
4546 try func.addLabel(.local_set, end_ptr.local.value);
4547
4548 // outer block to jump to when loop is done
4549 try func.startBlock(.block, wasm.block_empty);
4550 try func.startBlock(.loop, wasm.block_empty);
4551
4552 // check for codition for loop end
4553 try func.emitWValue(new_ptr);
4554 try func.emitWValue(end_ptr);
4555 switch (func.arch()) {
4556 .wasm32 => try func.addTag(.i32_eq),
4557 .wasm64 => try func.addTag(.i64_eq),
4558 else => unreachable,
4559 }
4560 try func.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
4561
4562 // store the value at the current position of the pointer
4563 try func.store(new_ptr, value, elem_ty, 0);
4564
4565 // move the pointer to the next element
4566 try func.emitWValue(new_ptr);
4567 switch (func.arch()) {
4568 .wasm32 => {
4569 try func.emitWValue(.{ .imm32 = abi_size });
4570 try func.addTag(.i32_add);
4556 },4571 },
4572 .wasm64 => {
4573 try func.emitWValue(.{ .imm64 = abi_size });
4574 try func.addTag(.i64_add);
4575 },
4576 else => unreachable,
4557 }4577 }
4578 try func.addLabel(.local_set, new_ptr.local.value);
4579
4580 // end of loop
4581 try func.addLabel(.br, 0); // jump to start of loop
4582 try func.endBlock();
4583 try func.endBlock();
4558}4584}
45594585
4560fn airArrayElemVal(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {4586fn airArrayElemVal(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...@@ -6007,10 +6033,12 @@ fn lowerTry(...@@ -6007,10 +6033,12 @@ fn lowerTry(
6007 const liveness = func.liveness.getCondBr(inst);6033 const liveness = func.liveness.getCondBr(inst);
6008 try func.branches.append(func.gpa, .{});6034 try func.branches.append(func.gpa, .{});
6009 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.else_deaths.len + liveness.then_deaths.len);6035 try func.currentBranch().values.ensureUnusedCapacity(func.gpa, liveness.else_deaths.len + liveness.then_deaths.len);
6036 defer {
6037 var branch = func.branches.pop();
6038 branch.deinit(func.gpa);
6039 }
6010 try func.genBody(body);6040 try func.genBody(body);
6011 try func.endBlock();6041 try func.endBlock();
6012 var branch = func.branches.pop();
6013 branch.deinit(func.gpa);
6014 }6042 }
60156043
6016 // if we reach here it means error was not set, and we want the payload6044 // if we reach here it means error was not set, and we want the payload
test/behavior/memset.zig-14
...@@ -7,10 +7,6 @@ test "@memset on array pointers" {...@@ -7,10 +7,6 @@ test "@memset on array pointers" {
7 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;7 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
8 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;8 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
9 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;9 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
10 if (builtin.zig_backend == .stage2_wasm) {
11 // TODO: implement memset when element ABI size > 1
12 return error.SkipZigTest;
13 }
1410
15 try testMemsetArray();11 try testMemsetArray();
16 try comptime testMemsetArray();12 try comptime testMemsetArray();
...@@ -40,11 +36,6 @@ test "@memset on slices" {...@@ -40,11 +36,6 @@ test "@memset on slices" {
40 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;36 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
41 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;37 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
42 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;38 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
43 if (builtin.zig_backend == .stage2_wasm) {
44 // TODO: implement memset when element ABI size > 1
45 // TODO: implement memset on slices
46 return error.SkipZigTest;
47 }
4839
49 try testMemsetSlice();40 try testMemsetSlice();
50 try comptime testMemsetSlice();41 try comptime testMemsetSlice();
...@@ -78,7 +69,6 @@ test "memset with bool element" {...@@ -78,7 +69,6 @@ test "memset with bool element" {
78 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;69 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
79 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;70 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
80 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;71 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
81 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
8272
83 var buf: [5]bool = undefined;73 var buf: [5]bool = undefined;
84 @memset(&buf, true);74 @memset(&buf, true);
...@@ -91,7 +81,6 @@ test "memset with 1-byte struct element" {...@@ -91,7 +81,6 @@ test "memset with 1-byte struct element" {
91 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;81 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
92 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;82 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
93 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;83 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
94 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
9584
96 const S = struct { x: bool };85 const S = struct { x: bool };
97 var buf: [5]S = undefined;86 var buf: [5]S = undefined;
...@@ -105,7 +94,6 @@ test "memset with 1-byte array element" {...@@ -105,7 +94,6 @@ test "memset with 1-byte array element" {
105 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;94 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
106 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;95 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
107 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;96 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
108 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
10997
110 const A = [1]bool;98 const A = [1]bool;
111 var buf: [5]A = undefined;99 var buf: [5]A = undefined;
...@@ -119,7 +107,6 @@ test "memset with large array element, runtime known" {...@@ -119,7 +107,6 @@ test "memset with large array element, runtime known" {
119 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;107 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
120 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;108 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
121 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;109 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
122 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
123110
124 const A = [128]u64;111 const A = [128]u64;
125 var buf: [5]A = undefined;112 var buf: [5]A = undefined;
...@@ -137,7 +124,6 @@ test "memset with large array element, comptime known" {...@@ -137,7 +124,6 @@ test "memset with large array element, comptime known" {
137 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;124 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
138 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;125 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
139 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;126 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
141127
142 const A = [128]u64;128 const A = [128]u64;
143 var buf: [5]A = undefined;129 var buf: [5]A = undefined;