authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-10 15:33:11-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-10 15:46:42-05:00
log59375b3c22ff6694a20847e87a162705dbbc632a
treed3e35fb83c29a239cb9960a0e1e04fa7ac38f981
parent138a35df8f434115be04641b1df29514b0ef1cb8

llvm: workaround SROA misoptimizations in LLVM

Workaround #16392

1 files changed, 35 insertions(+), 25 deletions(-)

src/codegen/llvm.zig+35-25
...@@ -5137,7 +5137,7 @@ pub const FuncGen = struct {...@@ -5137,7 +5137,7 @@ pub const FuncGen = struct {
5137 try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);5137 try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
51385138
5139 const alignment = return_type.abiAlignment(mod).toLlvm();5139 const alignment = return_type.abiAlignment(mod).toLlvm();
5140 const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);5140 const ret_ptr = try self.buildAllocaWorkaround(return_type, alignment);
5141 try llvm_args.append(ret_ptr);5141 try llvm_args.append(ret_ptr);
5142 break :blk ret_ptr;5142 break :blk ret_ptr;
5143 };5143 };
...@@ -5186,7 +5186,7 @@ pub const FuncGen = struct {...@@ -5186,7 +5186,7 @@ pub const FuncGen = struct {
51865186
5187 const alignment = param_ty.abiAlignment(mod).toLlvm();5187 const alignment = param_ty.abiAlignment(mod).toLlvm();
5188 const param_llvm_ty = try o.lowerType(param_ty);5188 const param_llvm_ty = try o.lowerType(param_ty);
5189 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);5189 const arg_ptr = try self.buildAllocaWorkaround(param_ty, alignment);
5190 if (isByRef(param_ty, mod)) {5190 if (isByRef(param_ty, mod)) {
5191 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");5191 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
5192 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);5192 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
...@@ -5209,7 +5209,7 @@ pub const FuncGen = struct {...@@ -5209,7 +5209,7 @@ pub const FuncGen = struct {
5209 // LLVM does not allow bitcasting structs so we must allocate5209 // LLVM does not allow bitcasting structs so we must allocate
5210 // a local, store as one type, and then load as another type.5210 // a local, store as one type, and then load as another type.
5211 const alignment = param_ty.abiAlignment(mod).toLlvm();5211 const alignment = param_ty.abiAlignment(mod).toLlvm();
5212 const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);5212 const int_ptr = try self.buildAllocaWorkaround(param_ty, alignment);
5213 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);5213 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
5214 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");5214 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
5215 try llvm_args.append(loaded);5215 try llvm_args.append(loaded);
...@@ -5552,7 +5552,7 @@ pub const FuncGen = struct {...@@ -5552,7 +5552,7 @@ pub const FuncGen = struct {
5552 const mod = o.module;5552 const mod = o.module;
55535553
5554 const result_alignment = va_list_ty.abiAlignment(mod).toLlvm();5554 const result_alignment = va_list_ty.abiAlignment(mod).toLlvm();
5555 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);5555 const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
55565556
5557 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, "");5557 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, "");
5558 return if (isByRef(va_list_ty, mod))5558 return if (isByRef(va_list_ty, mod))
...@@ -5576,7 +5576,7 @@ pub const FuncGen = struct {...@@ -5576,7 +5576,7 @@ pub const FuncGen = struct {
5576 const llvm_va_list_ty = try o.lowerType(va_list_ty);5576 const llvm_va_list_ty = try o.lowerType(va_list_ty);
55775577
5578 const result_alignment = va_list_ty.abiAlignment(mod).toLlvm();5578 const result_alignment = va_list_ty.abiAlignment(mod).toLlvm();
5579 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);5579 const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
55805580
5581 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, "");5581 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, "");
5582 return if (isByRef(va_list_ty, mod))5582 return if (isByRef(va_list_ty, mod))
...@@ -7407,7 +7407,7 @@ pub const FuncGen = struct {...@@ -7407,7 +7407,7 @@ pub const FuncGen = struct {
7407 self.ret_ptr7407 self.ret_ptr
7408 else brk: {7408 else brk: {
7409 const alignment = optional_ty.abiAlignment(mod).toLlvm();7409 const alignment = optional_ty.abiAlignment(mod).toLlvm();
7410 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);7410 const optional_ptr = try self.buildAllocaWorkaround(optional_ty, alignment);
7411 break :brk optional_ptr;7411 break :brk optional_ptr;
7412 };7412 };
74137413
...@@ -7443,7 +7443,7 @@ pub const FuncGen = struct {...@@ -7443,7 +7443,7 @@ pub const FuncGen = struct {
7443 self.ret_ptr7443 self.ret_ptr
7444 else brk: {7444 else brk: {
7445 const alignment = err_un_ty.abiAlignment(mod).toLlvm();7445 const alignment = err_un_ty.abiAlignment(mod).toLlvm();
7446 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);7446 const result_ptr = try self.buildAllocaWorkaround(err_un_ty, alignment);
7447 break :brk result_ptr;7447 break :brk result_ptr;
7448 };7448 };
74497449
...@@ -7481,7 +7481,7 @@ pub const FuncGen = struct {...@@ -7481,7 +7481,7 @@ pub const FuncGen = struct {
7481 self.ret_ptr7481 self.ret_ptr
7482 else brk: {7482 else brk: {
7483 const alignment = err_un_ty.abiAlignment(mod).toLlvm();7483 const alignment = err_un_ty.abiAlignment(mod).toLlvm();
7484 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);7484 const result_ptr = try self.buildAllocaWorkaround(err_un_ty, alignment);
7485 break :brk result_ptr;7485 break :brk result_ptr;
7486 };7486 };
74877487
...@@ -7965,7 +7965,7 @@ pub const FuncGen = struct {...@@ -7965,7 +7965,7 @@ pub const FuncGen = struct {
79657965
7966 if (isByRef(inst_ty, mod)) {7966 if (isByRef(inst_ty, mod)) {
7967 const result_alignment = inst_ty.abiAlignment(mod).toLlvm();7967 const result_alignment = inst_ty.abiAlignment(mod).toLlvm();
7968 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);7968 const alloca_inst = try self.buildAllocaWorkaround(inst_ty, result_alignment);
7969 {7969 {
7970 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");7970 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
7971 _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);7971 _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
...@@ -8323,7 +8323,7 @@ pub const FuncGen = struct {...@@ -8323,7 +8323,7 @@ pub const FuncGen = struct {
83238323
8324 if (isByRef(dest_ty, mod)) {8324 if (isByRef(dest_ty, mod)) {
8325 const result_alignment = dest_ty.abiAlignment(mod).toLlvm();8325 const result_alignment = dest_ty.abiAlignment(mod).toLlvm();
8326 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);8326 const alloca_inst = try self.buildAllocaWorkaround(dest_ty, result_alignment);
8327 {8327 {
8328 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");8328 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
8329 _ = try self.wip.store(.normal, result, field_ptr, result_alignment);8329 _ = try self.wip.store(.normal, result, field_ptr, result_alignment);
...@@ -8614,7 +8614,7 @@ pub const FuncGen = struct {...@@ -8614,7 +8614,7 @@ pub const FuncGen = struct {
8614 if (!result_is_ref) {8614 if (!result_is_ref) {
8615 return self.dg.todo("implement bitcast vector to non-ref array", .{});8615 return self.dg.todo("implement bitcast vector to non-ref array", .{});
8616 }8616 }
8617 const array_ptr = try self.buildAlloca(llvm_dest_ty, .default);8617 const array_ptr = try self.buildAllocaWorkaround(inst_ty, .default);
8618 const bitcast_ok = elem_ty.bitSize(mod) == elem_ty.abiSize(mod) * 8;8618 const bitcast_ok = elem_ty.bitSize(mod) == elem_ty.abiSize(mod) * 8;
8619 if (bitcast_ok) {8619 if (bitcast_ok) {
8620 const alignment = inst_ty.abiAlignment(mod).toLlvm();8620 const alignment = inst_ty.abiAlignment(mod).toLlvm();
...@@ -8676,7 +8676,7 @@ pub const FuncGen = struct {...@@ -8676,7 +8676,7 @@ pub const FuncGen = struct {
86768676
8677 if (result_is_ref) {8677 if (result_is_ref) {
8678 const alignment = operand_ty.abiAlignment(mod).max(inst_ty.abiAlignment(mod)).toLlvm();8678 const alignment = operand_ty.abiAlignment(mod).max(inst_ty.abiAlignment(mod)).toLlvm();
8679 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);8679 const result_ptr = try self.buildAllocaWorkaround(inst_ty, alignment);
8680 _ = try self.wip.store(.normal, operand, result_ptr, alignment);8680 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
8681 return result_ptr;8681 return result_ptr;
8682 }8682 }
...@@ -8688,7 +8688,7 @@ pub const FuncGen = struct {...@@ -8688,7 +8688,7 @@ pub const FuncGen = struct {
8688 // but LLVM won't let us bitcast struct values or vectors with padding bits.8688 // but LLVM won't let us bitcast struct values or vectors with padding bits.
8689 // Therefore, we store operand to alloca, then load for result.8689 // Therefore, we store operand to alloca, then load for result.
8690 const alignment = operand_ty.abiAlignment(mod).max(inst_ty.abiAlignment(mod)).toLlvm();8690 const alignment = operand_ty.abiAlignment(mod).max(inst_ty.abiAlignment(mod)).toLlvm();
8691 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);8691 const result_ptr = try self.buildAllocaWorkaround(inst_ty, alignment);
8692 _ = try self.wip.store(.normal, operand, result_ptr, alignment);8692 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
8693 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");8693 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
8694 }8694 }
...@@ -8753,9 +8753,9 @@ pub const FuncGen = struct {...@@ -8753,9 +8753,9 @@ pub const FuncGen = struct {
8753 if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime(mod))8753 if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime(mod))
8754 return (try o.lowerPtrToVoid(ptr_ty)).toValue();8754 return (try o.lowerPtrToVoid(ptr_ty)).toValue();
87558755
8756 const pointee_llvm_ty = try o.lowerType(pointee_type);8756 //const pointee_llvm_ty = try o.lowerType(pointee_type);
8757 const alignment = ptr_ty.ptrAlignment(mod).toLlvm();8757 const alignment = ptr_ty.ptrAlignment(mod).toLlvm();
8758 return self.buildAlloca(pointee_llvm_ty, alignment);8758 return self.buildAllocaWorkaround(pointee_type, alignment);
8759 }8759 }
87608760
8761 fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {8761 fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -8766,9 +8766,9 @@ pub const FuncGen = struct {...@@ -8766,9 +8766,9 @@ pub const FuncGen = struct {
8766 if (!ret_ty.isFnOrHasRuntimeBitsIgnoreComptime(mod))8766 if (!ret_ty.isFnOrHasRuntimeBitsIgnoreComptime(mod))
8767 return (try o.lowerPtrToVoid(ptr_ty)).toValue();8767 return (try o.lowerPtrToVoid(ptr_ty)).toValue();
8768 if (self.ret_ptr != .none) return self.ret_ptr;8768 if (self.ret_ptr != .none) return self.ret_ptr;
8769 const ret_llvm_ty = try o.lowerType(ret_ty);8769 //const ret_llvm_ty = try o.lowerType(ret_ty);
8770 const alignment = ptr_ty.ptrAlignment(mod).toLlvm();8770 const alignment = ptr_ty.ptrAlignment(mod).toLlvm();
8771 return self.buildAlloca(ret_llvm_ty, alignment);8771 return self.buildAllocaWorkaround(ret_ty, alignment);
8772 }8772 }
87738773
8774 /// Use this instead of builder.buildAlloca, because this function makes sure to8774 /// Use this instead of builder.buildAlloca, because this function makes sure to
...@@ -8782,6 +8782,16 @@ pub const FuncGen = struct {...@@ -8782,6 +8782,16 @@ pub const FuncGen = struct {
8782 return buildAllocaInner(&self.wip, self.di_scope != null, llvm_ty, alignment, target);8782 return buildAllocaInner(&self.wip, self.di_scope != null, llvm_ty, alignment, target);
8783 }8783 }
87848784
8785 // Workaround for https://github.com/ziglang/zig/issues/16392
8786 fn buildAllocaWorkaround(
8787 self: *FuncGen,
8788 ty: Type,
8789 alignment: Builder.Alignment,
8790 ) Allocator.Error!Builder.Value {
8791 const o = self.dg.object;
8792 return self.buildAlloca(try o.builder.arrayType(ty.abiSize(o.module), .i8), alignment);
8793 }
8794
8785 fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {8795 fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
8786 const o = self.dg.object;8796 const o = self.dg.object;
8787 const mod = o.module;8797 const mod = o.module;
...@@ -9736,7 +9746,7 @@ pub const FuncGen = struct {...@@ -9736,7 +9746,7 @@ pub const FuncGen = struct {
9736 const llvm_result_ty = accum_init.typeOfWip(&self.wip);9746 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
97379747
9738 // Allocate and initialize our mutable variables9748 // Allocate and initialize our mutable variables
9739 const i_ptr = try self.buildAlloca(usize_ty, .default);9749 const i_ptr = try self.buildAllocaWorkaround(Type.usize, .default);
9740 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);9750 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
9741 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);9751 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
9742 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);9752 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
...@@ -9948,7 +9958,7 @@ pub const FuncGen = struct {...@@ -9948,7 +9958,7 @@ pub const FuncGen = struct {
9948 // TODO in debug builds init to undef so that the padding will be 0xaa9958 // TODO in debug builds init to undef so that the padding will be 0xaa
9949 // even if we fully populate the fields.9959 // even if we fully populate the fields.
9950 const alignment = result_ty.abiAlignment(mod).toLlvm();9960 const alignment = result_ty.abiAlignment(mod).toLlvm();
9951 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);9961 const alloca_inst = try self.buildAllocaWorkaround(result_ty, alignment);
99529962
9953 for (elements, 0..) |elem, i| {9963 for (elements, 0..) |elem, i| {
9954 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;9964 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;
...@@ -9985,7 +9995,7 @@ pub const FuncGen = struct {...@@ -9985,7 +9995,7 @@ pub const FuncGen = struct {
9985 const llvm_usize = try o.lowerType(Type.usize);9995 const llvm_usize = try o.lowerType(Type.usize);
9986 const usize_zero = try o.builder.intValue(llvm_usize, 0);9996 const usize_zero = try o.builder.intValue(llvm_usize, 0);
9987 const alignment = result_ty.abiAlignment(mod).toLlvm();9997 const alignment = result_ty.abiAlignment(mod).toLlvm();
9988 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);9998 const alloca_inst = try self.buildAllocaWorkaround(result_ty, alignment);
99899999
9990 const array_info = result_ty.arrayInfo(mod);10000 const array_info = result_ty.arrayInfo(mod);
9991 const elem_ptr_ty = try mod.ptrType(.{10001 const elem_ptr_ty = try mod.ptrType(.{
...@@ -10061,7 +10071,7 @@ pub const FuncGen = struct {...@@ -10061,7 +10071,7 @@ pub const FuncGen = struct {
10061 // We must construct the correct unnamed struct type here, in order to then set10071 // We must construct the correct unnamed struct type here, in order to then set
10062 // the fields appropriately.10072 // the fields appropriately.
10063 const alignment = layout.abi_align.toLlvm();10073 const alignment = layout.abi_align.toLlvm();
10064 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);10074 const result_ptr = try self.buildAllocaWorkaround(union_ty, alignment);
10065 const llvm_payload = try self.resolveInst(extra.init);10075 const llvm_payload = try self.resolveInst(extra.init);
10066 const field_ty = union_obj.field_types.get(ip)[extra.field_index].toType();10076 const field_ty = union_obj.field_types.get(ip)[extra.field_index].toType();
10067 const field_llvm_ty = try o.lowerType(field_ty);10077 const field_llvm_ty = try o.lowerType(field_ty);
...@@ -10340,7 +10350,7 @@ pub const FuncGen = struct {...@@ -10340,7 +10350,7 @@ pub const FuncGen = struct {
1034010350
10341 if (isByRef(optional_ty, mod)) {10351 if (isByRef(optional_ty, mod)) {
10342 const payload_alignment = optional_ty.abiAlignment(mod).toLlvm();10352 const payload_alignment = optional_ty.abiAlignment(mod).toLlvm();
10343 const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);10353 const alloca_inst = try self.buildAllocaWorkaround(optional_ty, payload_alignment);
1034410354
10345 {10355 {
10346 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");10356 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
...@@ -10481,9 +10491,9 @@ pub const FuncGen = struct {...@@ -10481,9 +10491,9 @@ pub const FuncGen = struct {
10481 ) !Builder.Value {10491 ) !Builder.Value {
10482 const o = fg.dg.object;10492 const o = fg.dg.object;
10483 const mod = o.module;10493 const mod = o.module;
10484 const pointee_llvm_ty = try o.lowerType(pointee_type);10494 //const pointee_llvm_ty = try o.lowerType(pointee_type);
10485 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment).max(pointee_type.abiAlignment(mod)).toLlvm();10495 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment).max(pointee_type.abiAlignment(mod)).toLlvm();
10486 const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);10496 const result_ptr = try fg.buildAllocaWorkaround(pointee_type, result_align);
10487 const size_bytes = pointee_type.abiSize(mod);10497 const size_bytes = pointee_type.abiSize(mod);
10488 _ = try fg.wip.callMemCpy(10498 _ = try fg.wip.callMemCpy(
10489 result_ptr,10499 result_ptr,
...@@ -10542,7 +10552,7 @@ pub const FuncGen = struct {...@@ -10542,7 +10552,7 @@ pub const FuncGen = struct {
1054210552
10543 if (isByRef(elem_ty, mod)) {10553 if (isByRef(elem_ty, mod)) {
10544 const result_align = elem_ty.abiAlignment(mod).toLlvm();10554 const result_align = elem_ty.abiAlignment(mod).toLlvm();
10545 const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align);10555 const result_ptr = try self.buildAllocaWorkaround(elem_ty, result_align);
1054610556
10547 const same_size_int = try o.builder.intType(@intCast(elem_bits));10557 const same_size_int = try o.builder.intType(@intCast(elem_bits));
10548 const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");10558 const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");