authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-12-03 02:18:07-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-12-03 02:18:07-05:00
log610d604f5b4be220ea0f404af871c4365026a6a7
tree56ddbf87ab181c61af498280aa25614bde54493e
parent53dcf78a25b47aa28d271c4c520eafb7c3474704
parente1216077f0edf0efa0b322c5469e2ab572213271
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #13748 from jacobly0/c-unaligned

cbe: use memcpy for underaligned loads and stores

9 files changed, 61 insertions(+), 34 deletions(-)

src/codegen/c.zig+61-19
......@@ -3017,30 +3017,31 @@ fn airArg(f: *Function) CValue {
30173017fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
30183018 const ty_op = f.air.instructions.items(.data)[inst].ty_op;
30193019 const ptr_info = f.air.typeOf(ty_op.operand).ptrInfo().data;
3020 const src_ty = ptr_info.pointee_type;
30203021
3021 const inst_ty = f.air.typeOfIndex(inst);
3022 if (!inst_ty.hasRuntimeBitsIgnoreComptime() or
3022 if (!src_ty.hasRuntimeBitsIgnoreComptime() or
30233023 !ptr_info.@"volatile" and f.liveness.isUnused(inst))
30243024 return CValue.none;
30253025
30263026 const target = f.object.dg.module.getTarget();
3027 const is_array = lowersToArray(inst_ty, target);
3027 const is_aligned = ptr_info.@"align" == 0 or ptr_info.@"align" >= src_ty.abiAlignment(target);
3028 const is_array = lowersToArray(src_ty, target);
3029 const need_memcpy = !is_aligned or is_array;
30283030 const operand = try f.resolveInst(ty_op.operand);
30293031 const writer = f.object.writer();
30303032
3031 // We need to separately initialize arrays with a memcpy so they must be mutable.
3032 const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const);
3033 // We need to initialize arrays and unaligned loads with a memcpy so they must be mutable.
3034 const local = try f.allocLocal(src_ty, if (need_memcpy) .Mut else .Const);
30333035
3034 if (is_array) {
3035 // Insert a memcpy to initialize this array. The source operand is always a pointer
3036 // and thus we only need to know size/type information from the local type/dest.
3036 if (need_memcpy) {
30373037 try writer.writeAll(";\n");
30383038 try writer.writeAll("memcpy(");
3039 if (!is_array) try writer.writeByte('&');
30393040 try f.writeCValue(writer, local, .FunctionArgument);
3040 try writer.writeAll(", ");
3041 try f.writeCValue(writer, operand, .FunctionArgument);
3041 try writer.writeAll(", (const char *)");
3042 try f.writeCValue(writer, operand, .Other);
30423043 try writer.writeAll(", sizeof(");
3043 try f.renderTypecast(writer, inst_ty);
3044 try f.renderTypecast(writer, src_ty);
30443045 try writer.writeAll("))");
30453046 } else if (ptr_info.host_size != 0) {
30463047 var host_pl = Type.Payload.Bits{
......@@ -3063,12 +3064,12 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
30633064
30643065 var field_pl = Type.Payload.Bits{
30653066 .base = .{ .tag = .int_unsigned },
3066 .data = @intCast(u16, inst_ty.bitSize(target)),
3067 .data = @intCast(u16, src_ty.bitSize(target)),
30673068 };
30683069 const field_ty = Type.initPayload(&field_pl.base);
30693070
30703071 try writer.writeAll(" = (");
3071 try f.renderTypecast(writer, inst_ty);
3072 try f.renderTypecast(writer, src_ty);
30723073 try writer.writeAll(")zig_wrap_");
30733074 try f.object.dg.renderTypeForBuiltinFnName(writer, field_ty);
30743075 try writer.writeAll("((");
......@@ -3244,8 +3245,13 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {
32443245 return try airStoreUndefined(f, ptr_info.pointee_type, ptr_val);
32453246
32463247 const target = f.object.dg.module.getTarget();
3248 const is_aligned = ptr_info.@"align" == 0 or
3249 ptr_info.@"align" >= ptr_info.pointee_type.abiAlignment(target);
3250 const is_array = lowersToArray(ptr_info.pointee_type, target);
3251 const need_memcpy = !is_aligned or is_array;
32473252 const writer = f.object.writer();
3248 if (lowersToArray(ptr_info.pointee_type, target)) {
3253
3254 if (need_memcpy) {
32493255 // For this memcpy to safely work we need the rhs to have the same
32503256 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
32513257 assert(src_ty.eql(ptr_info.pointee_type, f.object.dg.module));
......@@ -3262,9 +3268,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue {
32623268 break :blk new_local;
32633269 } else src_val;
32643270
3265 try writer.writeAll("memcpy(");
3271 try writer.writeAll("memcpy((char *)");
32663272 try f.writeCValue(writer, ptr_val, .FunctionArgument);
32673273 try writer.writeAll(", ");
3274 if (!is_array) try writer.writeByte('&');
32683275 try f.writeCValue(writer, array_src, .FunctionArgument);
32693276 try writer.writeAll(", sizeof(");
32703277 try f.renderTypecast(writer, src_ty);
......@@ -4186,11 +4193,46 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
41864193 extra_i += clobber.len / 4 + 1;
41874194 }
41884195 }
4189 const asm_source = std.mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len];
4196 {
4197 const asm_source = std.mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len];
4198
4199 var stack = std.heap.stackFallback(256, f.object.dg.gpa);
4200 const allocator = stack.get();
4201 const fixed_asm_source = try allocator.alloc(u8, asm_source.len);
4202 defer allocator.free(fixed_asm_source);
4203
4204 var src_i: usize = 0;
4205 var dst_i: usize = 0;
4206 while (src_i < asm_source.len) : (src_i += 1) {
4207 fixed_asm_source[dst_i] = asm_source[src_i];
4208 dst_i += 1;
4209 if (asm_source[src_i] != '%' or src_i + 1 >= asm_source.len) continue;
4210 src_i += 1;
4211 if (asm_source[src_i] != '[') {
4212 // This handles %%
4213 fixed_asm_source[dst_i] = asm_source[src_i];
4214 dst_i += 1;
4215 continue;
4216 }
4217 const len = std.mem.indexOfScalar(u8, asm_source[src_i + 1 ..], ']') orelse
4218 return f.fail("CBE: invalid inline asm string '{s}'", .{asm_source});
4219 if (std.mem.indexOfScalar(u8, asm_source[src_i + 1 ..][0..len], ':')) |colon| {
4220 const modifier = asm_source[src_i + 1 + colon + 1 .. src_i + 1 + len];
4221 std.mem.copy(u8, fixed_asm_source[dst_i..], modifier);
4222 dst_i += modifier.len;
4223
4224 const name = asm_source[src_i .. src_i + 1 + colon];
4225 std.mem.copy(u8, fixed_asm_source[dst_i..], name);
4226 dst_i += name.len;
4227
4228 src_i += len;
4229 }
4230 }
41904231
4191 try writer.writeAll("__asm");
4192 if (is_volatile) try writer.writeAll(" volatile");
4193 try writer.print("({s}", .{fmtStringLiteral(asm_source)});
4232 try writer.writeAll("__asm");
4233 if (is_volatile) try writer.writeAll(" volatile");
4234 try writer.print("({s}", .{fmtStringLiteral(fixed_asm_source[0..dst_i])});
4235 }
41944236
41954237 extra_i = constraints_extra_begin;
41964238 var locals_index = locals_begin;
test/behavior/asm.zig-1
......@@ -140,7 +140,6 @@ export fn derp() i32 {
140140test "asm modifiers (AArch64)" {
141141 if (builtin.target.cpu.arch != .aarch64) return error.SkipZigTest;
142142 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
143 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
144143
145144 var x: u32 = 15;
146145 const double = asm ("add %[ret:w], %[in:w], %[in:w]"
test/behavior/basic.zig-2
......@@ -754,7 +754,6 @@ fn maybe(x: bool) anyerror!?u32 {
754754}
755755
756756test "pointer to thread local array" {
757 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
758757 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
759758 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
760759 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
......@@ -770,7 +769,6 @@ threadlocal var buffer: [11]u8 = undefined;
770769
771770test "auto created variables have correct alignment" {
772771 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
773 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
774772
775773 const S = struct {
776774 fn foo(str: [*]const u8) u32 {
test/behavior/bitreverse.zig-1
......@@ -155,7 +155,6 @@ fn vector0() !void {
155155test "bitReverse vectors u0" {
156156 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
157157 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
158 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
159158 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
160159
161160 comptime try vector0();
test/behavior/byteswap.zig-1
......@@ -121,7 +121,6 @@ fn vector0() !void {
121121test "@byteSwap vectors u0" {
122122 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
123123 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
124 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
125124 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
126125
127126 comptime try vector0();
test/behavior/math.zig-1
......@@ -1518,7 +1518,6 @@ fn testRound(comptime T: type, x: T) !void {
15181518}
15191519
15201520test "vector integer addition" {
1521 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
15221521 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
15231522 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15241523 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
test/behavior/ptrcast.zig-1
......@@ -78,7 +78,6 @@ fn testReinterpretBytesAsExternStruct() !void {
7878
7979test "reinterpret bytes of an extern struct (with under-aligned fields) into another" {
8080 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
81 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO: Under-aligned fields are not yet supported in the CBE
8281
8382 try testReinterpretExternStructAsExternStruct();
8483 comptime try testReinterpretExternStructAsExternStruct();
test/behavior/struct.zig-1
......@@ -641,7 +641,6 @@ test "default struct initialization fields" {
641641
642642test "packed array 24bits" {
643643 if (builtin.zig_backend == .stage1) return error.SkipZigTest;
644 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
645644 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
646645 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
647646 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
test/behavior/vector.zig-7
......@@ -75,7 +75,6 @@ test "vector bin compares with mem.eql" {
7575
7676test "vector int operators" {
7777 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
78 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
7978 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8079 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8180 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -139,7 +138,6 @@ test "vector bit operators" {
139138
140139test "implicit cast vector to array" {
141140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
142 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
143141 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
144142 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
145143 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -158,7 +156,6 @@ test "implicit cast vector to array" {
158156
159157test "array to vector" {
160158 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
161 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
162159 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
163160 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
164161 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -177,7 +174,6 @@ test "array to vector" {
177174
178175test "tuple to vector" {
179176 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
180 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
181177 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
182178 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
183179 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -210,7 +206,6 @@ test "tuple to vector" {
210206
211207test "vector casts of sizes not divisible by 8" {
212208 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
213 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
214209 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
215210 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
216211 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -945,7 +940,6 @@ test "multiplication-assignment operator with an array operand" {
945940 return error.SkipZigTest;
946941 }
947942 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
948 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
949943 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
950944 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
951945 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
......@@ -1194,7 +1188,6 @@ test "zero divisor" {
11941188
11951189test "zero multiplicand" {
11961190 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1197 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
11981191 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
11991192 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
12001193 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO