authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-03-22 20:11:35+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-03-28 16:45:55+00:00
log5941c9da08ae0d8ba4add37f0baeafdcd160dbd4
tree15f1ad30949184585ea9d658485fb5e0bdd2e033
parente3ee37f983ffe655b5e9516c12a1a0d69a967e12
signaturelock-open Commit is signed but in an unrecognized format.

llvm: remove almost all GEPs

LLVM is gradually transitioning from the `getelementptr` instruction to a new `ptradd` instruction. The latter instruction doesn't actually exist yet, but for now, LLVM is considering `getelementptr i8` to be equivalent. LLVM is already internally canonicalizing `getelementptr` usages to this pattern in many cases, and it's far easier for us to emit that, so... let's do so! For runtime indexing this does sometimes require an explicit multiplication to scale an index to a byte offset. The helper function `llvm.FuncGen.ptraddScaled` makes this common pattern more convenient. A particularly nice side effect from this is that after removing some dead code (left over from before we made all `struct`s etc by-ref), it has eliminated the need to maintain that nasty mapping between Zig field indices and LLVM field indices. `FuncGen` no longer cares at all how aggregate types are lowered! Slices are still by-val at least for now, but they never lived in that mapping because their structure is simple and consistent (they always have a pointer at field index 0 and a usize at field index 1, with no explicit padding necessary).

3 files changed, 406 insertions(+), 764 deletions(-)

src/Type.zig+1-1
......@@ -1594,7 +1594,7 @@ pub fn unionTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {
15941594 return zcu.unionTagFieldIndex(union_obj, enum_tag);
15951595}
15961596
1597pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *Zcu) bool {
1597pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *const Zcu) bool {
15981598 assertHasLayout(ty, zcu);
15991599 const ip = &zcu.intern_pool;
16001600 const union_obj = zcu.typeToUnion(ty).?;
src/codegen/llvm.zig+2-59
......@@ -578,24 +578,9 @@ pub const Object = struct {
578578 /// Memoizes a null `?usize` value.
579579 null_opt_usize: Builder.Constant,
580580
581 /// When an LLVM struct type is created, an entry is inserted into this
582 /// table for every zig source field of the struct that has a corresponding
583 /// LLVM struct field. comptime fields are not included. Zero-bit fields are
584 /// mapped to a field at the correct byte, which may be a padding field, or
585 /// are not mapped, in which case they are semantically at the end of the
586 /// struct.
587 /// The value is the LLVM struct field index.
588 /// This is denormalized data.
589 struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint),
590
591581 /// Values for `@llvm.used`.
592582 used: std.ArrayList(Builder.Constant),
593583
594 const ZigStructField = struct {
595 struct_ty: InternPool.Index,
596 field_index: u32,
597 };
598
599584 pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;
600585
601586 const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
......@@ -688,7 +673,6 @@ pub const Object = struct {
688673 .type_map = .empty,
689674 .error_name_table = .none,
690675 .null_opt_usize = .no_init,
691 .struct_field_map = .empty,
692676 .used = .empty,
693677 };
694678 return obj;
......@@ -708,7 +692,6 @@ pub const Object = struct {
708692 self.named_enum_map.deinit(gpa);
709693 self.type_map.deinit(gpa);
710694 self.builder.deinit();
711 self.struct_field_map.deinit(gpa);
712695 self.* = undefined;
713696 }
714697
......@@ -3311,7 +3294,6 @@ pub const Object = struct {
33113294 // Although we can estimate how much capacity to add, these cannot be
33123295 // relied upon because of the recursive calls to lowerType below.
33133296 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
3314 try o.struct_field_map.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
33153297
33163298 comptime assert(struct_layout_version == 2);
33173299 var offset: u64 = 0;
......@@ -3336,23 +3318,8 @@ pub const Object = struct {
33363318 try o.builder.arrayType(padding_len, .i8),
33373319 );
33383320
3339 if (!field_ty.hasRuntimeBits(zcu)) {
3340 // This is a zero-bit field. If there are runtime bits after this field,
3341 // map to the next LLVM field (which we know exists): otherwise, don't
3342 // map the field, indicating it's at the end of the struct.
3343 if (offset != struct_type.size) {
3344 try o.struct_field_map.put(o.gpa, .{
3345 .struct_ty = t.toIntern(),
3346 .field_index = field_index,
3347 }, @intCast(llvm_field_types.items.len));
3348 }
3349 continue;
3350 }
3321 if (!field_ty.hasRuntimeBits(zcu)) continue;
33513322
3352 try o.struct_field_map.put(o.gpa, .{
3353 .struct_ty = t.toIntern(),
3354 .field_index = field_index,
3355 }, @intCast(llvm_field_types.items.len));
33563323 try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty));
33573324
33583325 offset += field_ty.abiSize(zcu);
......@@ -3385,19 +3352,15 @@ pub const Object = struct {
33853352 // Although we can estimate how much capacity to add, these cannot be
33863353 // relied upon because of the recursive calls to lowerType below.
33873354 try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
3388 try o.struct_field_map.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
33893355
33903356 comptime assert(struct_layout_version == 2);
33913357 var offset: u64 = 0;
33923358 var big_align: InternPool.Alignment = .none;
33933359
3394 const struct_size = t.abiSize(zcu);
3395
33963360 for (
33973361 tuple_type.types.get(ip),
33983362 tuple_type.values.get(ip),
3399 0..,
3400 ) |field_ty, field_val, field_index| {
3363 ) |field_ty, field_val| {
34013364 if (field_val != .none) continue;
34023365
34033366 const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
......@@ -3411,21 +3374,8 @@ pub const Object = struct {
34113374 try o.builder.arrayType(padding_len, .i8),
34123375 );
34133376 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3414 // This is a zero-bit field. If there are runtime bits after this field,
3415 // map to the next LLVM field (which we know exists): otherwise, don't
3416 // map the field, indicating it's at the end of the struct.
3417 if (offset != struct_size) {
3418 try o.struct_field_map.put(o.gpa, .{
3419 .struct_ty = t.toIntern(),
3420 .field_index = @intCast(field_index),
3421 }, @intCast(llvm_field_types.items.len));
3422 }
34233377 continue;
34243378 }
3425 try o.struct_field_map.put(o.gpa, .{
3426 .struct_ty = t.toIntern(),
3427 .field_index = @intCast(field_index),
3428 }, @intCast(llvm_field_types.items.len));
34293379 try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty)));
34303380
34313381 offset += Type.fromInterned(field_ty).abiSize(zcu);
......@@ -4338,13 +4288,6 @@ pub const Object = struct {
43384288 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
43394289 }
43404290
4341 pub fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4342 return o.struct_field_map.get(.{
4343 .struct_ty = struct_ty.toIntern(),
4344 .field_index = @intCast(field_index),
4345 });
4346 }
4347
43484291 /// MLUGG TODO: this is super dumb
43494292 pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {
43504293 const name = try o.builder.strtabString(lt_errors_fn_name);
src/codegen/llvm/FuncGen.zig+403-704
......@@ -141,7 +141,7 @@ pub fn deinit(self: *FuncGen) void {
141141 self.switch_dispatch_info.deinit(gpa);
142142}
143143
144fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
144fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) Allocator.Error!Builder.Value {
145145 const gpa = self.gpa;
146146 const gop = try self.func_inst_table.getOrPut(gpa, inst);
147147 if (gop.found_existing) return gop.value_ptr.*;
......@@ -151,7 +151,7 @@ fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
151151 return llvm_val.toValue();
152152}
153153
154fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant {
154fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
155155 const o = self.object;
156156 const pt = self.pt;
157157 const zcu = pt.zcu;
......@@ -177,9 +177,7 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
177177 .poi => if (self.fuzz) |*fuzz| {
178178 const poi_index = fuzz.pcs.items.len;
179179 const base_ptr = fuzz.counters_variable.toValue(&o.builder);
180 const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{
181 try o.builder.intValue(.i32, poi_index),
182 }, "");
180 const ptr = try self.ptraddConst(base_ptr, poi_index);
183181 const one = try o.builder.intValue(.i8, 1);
184182 _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
185183
......@@ -704,8 +702,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif
704702 const llvm_ty = try o.builder.structType(.normal, llvm_types);
705703 try llvm_args.ensureUnusedCapacity(it.types_len);
706704 for (llvm_types, 0..) |field_ty, i| {
707 const alignment =
708 Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
705 const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8));
709706 const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
710707 const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
711708 llvm_args.appendAssumeCapacity(loaded);
......@@ -1153,10 +1150,8 @@ fn cmp(
11531150 }
11541151 // We need to emit instructions to check for equality/inequality
11551152 // of optionals that are not pointers.
1156 const is_by_ref = isByRef(scalar_ty, zcu);
1157 const opt_llvm_ty = try o.lowerType(pt, scalar_ty);
1158 const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal);
1159 const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal);
1153 const lhs_non_null = try self.optCmpNull(.ne, scalar_ty, lhs, .normal);
1154 const rhs_non_null = try self.optCmpNull(.ne, scalar_ty, rhs, .normal);
11601155 const llvm_i2 = try o.builder.intType(2);
11611156 const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
11621157 const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
......@@ -1186,8 +1181,8 @@ fn cmp(
11861181 _ = try self.wip.br(end_block);
11871182
11881183 self.wip.cursor = .{ .block = both_pl_block };
1189 const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);
1190 const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);
1184 const lhs_payload = try self.optPayloadHandle(lhs, scalar_ty, true);
1185 const rhs_payload = try self.optPayloadHandle(rhs, scalar_ty, true);
11911186 const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
11921187 _ = try self.wip.br(end_block);
11931188 const both_pl_block_end = self.wip.cursor.block;
......@@ -1392,12 +1387,10 @@ fn lowerSwitchDispatch(
13921387 try o.lowerType(pt, .usize),
13931388 "",
13941389 );
1395 const target_ptr_ptr = try self.wip.gep(
1396 .inbounds,
1397 .ptr,
1390 const target_ptr_ptr = try self.ptraddScaled(
13981391 jmp_table.table.toValue(),
1399 &.{table_index},
1400 "",
1392 table_index,
1393 Type.usize.abiSize(zcu),
14011394 );
14021395 const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
14031396
......@@ -1580,7 +1573,7 @@ fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
15801573 const body = unwrapped_try.else_body;
15811574 const err_union_ty = self.typeOf(unwrapped_try.error_union);
15821575 const is_unused = self.liveness.isUnused(inst);
1583 return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold);
1576 return lowerTry(self, err_union, body, err_union_ty, false, .none, is_unused, err_cold);
15841577}
15851578
15861579fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
......@@ -1594,7 +1587,7 @@ fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Valu
15941587
15951588 self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
15961589
1597 return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold);
1590 return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), is_unused, err_cold);
15981591}
15991592
16001593fn lowerTry(
......@@ -1604,7 +1597,6 @@ fn lowerTry(
16041597 err_union_ty: Type,
16051598 operand_is_ptr: bool,
16061599 operand_ptr_align: InternPool.Alignment,
1607 can_elide_load: bool,
16081600 is_unused: bool,
16091601 err_cold: bool,
16101602) !Builder.Value {
......@@ -1613,7 +1605,6 @@ fn lowerTry(
16131605 const zcu = pt.zcu;
16141606 const payload_ty = err_union_ty.errorUnionPayload(zcu);
16151607 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1616 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
16171608 const error_type = try o.errorIntType(pt);
16181609
16191610 const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
......@@ -1632,19 +1623,17 @@ fn lowerTry(
16321623 else
16331624 err_union;
16341625 }
1635 const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt);
1636 if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
1637 const err_field_ptr =
1638 try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, "");
1639 break :loaded try fg.wip.load(
1640 if (operand_is_ptr) access_kind else .normal,
1641 error_type,
1642 err_field_ptr,
1643 err_set_align.toLlvm(),
1644 "",
1645 );
1646 }
1647 break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, "");
1626
1627 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1628 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1629 const err_field_ptr = try fg.ptraddConst(err_union, offset);
1630 break :loaded try fg.wip.load(
1631 if (operand_is_ptr) access_kind else .normal,
1632 error_type,
1633 err_field_ptr,
1634 err_set_align.toLlvm(),
1635 "",
1636 );
16481637 };
16491638 const zero = try o.builder.intValue(error_type, 0);
16501639 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
......@@ -1661,21 +1650,15 @@ fn lowerTry(
16611650 }
16621651 if (is_unused) return .none;
16631652 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
1664 const offset = try errUnionPayloadFieldIndex(payload_ty, pt);
1653 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1654 const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));
16651655 if (operand_is_ptr) {
1666 return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
1667 } else if (isByRef(err_union_ty, zcu)) {
1668 const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
1669 if (isByRef(payload_ty, zcu)) {
1670 if (can_elide_load)
1671 return payload_ptr;
1672
1673 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1674 }
1675 const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
1676 return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), "");
1656 return payload_ptr;
1657 } else if (isByRef(payload_ty, zcu)) {
1658 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1659 } else {
1660 return fg.wip.load(.normal, try o.lowerType(pt, payload_ty), payload_ptr, payload_align.toLlvm(), "");
16771661 }
1678 return fg.wip.extractValue(err_union, &.{offset}, "");
16791662}
16801663
16811664fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {
......@@ -1950,12 +1933,7 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
19501933 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
19511934 const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
19521935 const operand = try self.resolveInst(ty_op.operand);
1953 if (!array_ty.hasRuntimeBits(zcu))
1954 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
1955 const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{
1956 try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0),
1957 }, "");
1958 return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, "");
1936 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
19591937}
19601938
19611939fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -2126,20 +2104,14 @@ fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Valu
21262104 return self.wip.extractValue(operand, &.{index}, "");
21272105}
21282106
2129fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value {
2130 const o = self.object;
2131 const pt = self.pt;
2132 const zcu = pt.zcu;
2107fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) !Builder.Value {
2108 const zcu = self.pt.zcu;
21332109 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
21342110 const slice_ptr = try self.resolveInst(ty_op.operand);
2135 const slice_ptr_ty = self.typeOf(ty_op.operand);
2136 const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu));
2137
2138 return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
2111 return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu));
21392112}
21402113
21412114fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2142 const o = self.object;
21432115 const pt = self.pt;
21442116 const zcu = pt.zcu;
21452117 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
......@@ -2149,9 +2121,8 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
21492121 const slice_info = slice_ty.ptrInfo(zcu);
21502122 assert(slice_info.flags.size == .slice);
21512123 const elem_ty: Type = .fromInterned(slice_info.child);
2152 const llvm_elem_ty = try o.lowerType(pt, elem_ty);
21532124 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2154 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
2125 const ptr = try self.ptraddScaled(base_ptr, index, elem_ty.abiSize(zcu));
21552126 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
21562127 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
21572128 self.maybeMarkAllowZeroAccess(slice_info);
......@@ -2163,7 +2134,6 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
21632134}
21642135
21652136fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2166 const o = self.object;
21672137 const pt = self.pt;
21682138 const zcu = pt.zcu;
21692139 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -2172,13 +2142,11 @@ fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
21722142
21732143 const slice = try self.resolveInst(bin_op.lhs);
21742144 const index = try self.resolveInst(bin_op.rhs);
2175 const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu));
21762145 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2177 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
2146 return self.ptraddScaled(base_ptr, index, slice_ty.childType(zcu).abiSize(zcu));
21782147}
21792148
21802149fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2181 const o = self.object;
21822150 const pt = self.pt;
21832151 const zcu = pt.zcu;
21842152
......@@ -2186,16 +2154,12 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
21862154 const array_ty = self.typeOf(bin_op.lhs);
21872155 const array_llvm_val = try self.resolveInst(bin_op.lhs);
21882156 const rhs = try self.resolveInst(bin_op.rhs);
2189 const array_llvm_ty = try o.lowerType(pt, array_ty);
21902157 const elem_ty = array_ty.childType(zcu);
21912158 if (isByRef(array_ty, zcu)) {
2192 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
2193 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
2194 rhs,
2195 }, "");
2159 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
21962160 if (isByRef(elem_ty, zcu)) {
2197 const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
2198 return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
2161 const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
2162 return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
21992163 } else {
22002164 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
22012165 }
......@@ -2206,16 +2170,14 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
22062170}
22072171
22082172fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2209 const o = self.object;
22102173 const pt = self.pt;
22112174 const zcu = pt.zcu;
22122175 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
22132176 const ptr_ty = self.typeOf(bin_op.lhs);
22142177 const elem_ty = ptr_ty.indexableElem(zcu);
2215 const llvm_elem_ty = try o.lowerType(pt, elem_ty);
22162178 const base_ptr = try self.resolveInst(bin_op.lhs);
22172179 const rhs = try self.resolveInst(bin_op.rhs);
2218 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
2180 const ptr = try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
22192181 if (isByRef(elem_ty, zcu)) {
22202182 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
22212183 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
......@@ -2228,7 +2190,6 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
22282190}
22292191
22302192fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2231 const o = self.object;
22322193 const pt = self.pt;
22332194 const zcu = pt.zcu;
22342195 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -2243,8 +2204,7 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
22432204 const elem_ptr = ty_pl.ty.toType();
22442205 if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
22452206
2246 const llvm_elem_ty = try o.lowerType(pt, elem_ty);
2247 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
2207 return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
22482208}
22492209
22502210fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -2276,86 +2236,59 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
22762236 const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
22772237 const field_index = struct_field.field_index;
22782238 const field_ty = struct_ty.fieldType(field_index, zcu);
2279 if (!field_ty.hasRuntimeBits(zcu)) return .none;
2239 assert(field_ty.hasRuntimeBits(zcu));
22802240
22812241 if (!isByRef(struct_ty, zcu)) {
2242 // All auto/extern struct/union types are by-ref, unless they have no runtime bits, in which
2243 // case we shouldn't be seeing this instruction to begin with. Therefore we must be dealing
2244 // with a `packed struct` or `packed union`.
2245 assert(struct_ty.containerLayout(zcu) == .@"packed");
22822246 assert(!isByRef(field_ty, zcu));
2283 switch (struct_ty.zigTypeTag(zcu)) {
2284 .@"struct" => switch (struct_ty.containerLayout(zcu)) {
2285 .@"packed" => {
2286 const struct_type = zcu.typeToStruct(struct_ty).?;
2287 const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index);
2288 const containing_int = struct_llvm_val;
2289 const shift_amt =
2290 try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);
2291 const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
2292 const elem_llvm_ty = try o.lowerType(pt, field_ty);
2293 if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
2294 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2295 const truncated_int =
2296 try self.wip.cast(.trunc, shifted_value, same_size_int, "");
2297 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
2298 }
2299 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
2300 },
2301 else => {
2302 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
2303 return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
2304 },
2305 },
2306 .@"union" => {
2307 assert(struct_ty.containerLayout(zcu) == .@"packed");
2308 const containing_int = struct_llvm_val;
2309 const elem_llvm_ty = try o.lowerType(pt, field_ty);
2310 if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
2311 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2312 const truncated_int =
2313 try self.wip.cast(.trunc, containing_int, same_size_int, "");
2314 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
2315 }
2316 return self.wip.cast(.trunc, containing_int, elem_llvm_ty, "");
2247 const field_int_val: Builder.Value = switch (struct_ty.zigTypeTag(zcu)) {
2248 .@"struct" => field_int_val: {
2249 const llvm_field_int_ty = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2250 const bit_offset = zcu.structPackedFieldBitOffset(
2251 zcu.intern_pool.loadStructType(struct_ty.toIntern()),
2252 field_index,
2253 );
2254 const shift_bits = try o.builder.intValue(struct_llvm_val.typeOfWip(&self.wip), bit_offset);
2255 const shifted = try self.wip.bin(.lshr, struct_llvm_val, shift_bits, "");
2256 break :field_int_val try self.wip.cast(.trunc, shifted, llvm_field_int_ty, "");
23172257 },
2258 .@"union" => struct_llvm_val,
23182259 else => unreachable,
2260 };
2261 switch (field_ty.zigTypeTag(zcu)) {
2262 else => unreachable, // not packable
2263 .void => unreachable, // opv bug in sema
2264 .int, .bool, .@"enum", .@"struct", .@"union" => {
2265 // Represented as integers, so already done
2266 return field_int_val;
2267 },
2268 .float => {
2269 // bitcast int->float
2270 return self.wip.cast(.bitcast, field_int_val, try o.lowerType(pt, field_ty), "");
2271 },
23192272 }
23202273 }
23212274
2322 switch (struct_ty.zigTypeTag(zcu)) {
2323 .@"struct" => {
2324 const layout = struct_ty.containerLayout(zcu);
2325 assert(layout != .@"packed");
2326 const struct_llvm_ty = try o.lowerType(pt, struct_ty);
2327 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
2328 const field_ptr =
2329 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
2330 const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
2331 const field_ptr_ty = try pt.ptrType(.{
2332 .child = field_ty.toIntern(),
2333 .flags = .{ .alignment = explicit_alignment },
2334 });
2335 if (isByRef(field_ty, zcu)) {
2336 const alignment = switch (explicit_alignment) {
2337 .none => field_ty.abiAlignment(zcu),
2338 else => |a| a,
2339 };
2340 return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal);
2341 } else {
2342 return self.load(field_ptr, field_ptr_ty);
2343 }
2344 },
2345 .@"union" => {
2346 const union_llvm_ty = try o.lowerType(pt, struct_ty);
2347 const layout = struct_ty.unionGetLayout(zcu);
2348 const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
2349 const field_ptr =
2350 try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
2351 const payload_alignment = layout.payload_align.toLlvm();
2352 if (isByRef(field_ty, zcu)) {
2353 return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
2354 } else {
2355 return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
2356 }
2357 },
2275 const offset: u64 = switch (struct_ty.zigTypeTag(zcu)) {
2276 .@"struct" => struct_ty.structFieldOffset(field_index, zcu),
2277 .@"union" => struct_ty.unionGetLayout(zcu).payloadOffset(),
23582278 else => unreachable,
2279 };
2280
2281 const struct_ptr_align = struct_ty.abiAlignment(zcu);
2282 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
2283 const field_ptr_align: InternPool.Alignment = switch (offset) {
2284 0 => struct_ptr_align,
2285 else => struct_ptr_align.minStrict(.fromLog2Units(@ctz(offset))),
2286 };
2287
2288 if (isByRef(field_ty, zcu)) {
2289 return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
2290 } else {
2291 return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
23592292 }
23602293}
23612294
......@@ -2964,8 +2897,7 @@ fn airIsNonNull(
29642897 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
29652898 }
29662899
2967 const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu);
2968 return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind);
2900 return self.optCmpNull(cond, optional_ty, operand, access_kind);
29692901}
29702902
29712903fn airIsErr(
......@@ -3006,40 +2938,23 @@ fn airIsErr(
30062938 operand;
30072939 return self.wip.icmp(cond, loaded, zero, "");
30082940 }
2941 assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
30092942
3010 const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt);
3011
3012 const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: {
3013 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3014 const err_alignment = if (operand_is_ptr)
3015 operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
3016 else
3017 .none;
3018 const err_field_ptr =
3019 try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, "");
3020 break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), "");
3021 } else try self.wip.extractValue(operand, &.{err_field_index}, "");
2943 const err_align = if (operand_is_ptr)
2944 operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
2945 else
2946 .none;
2947 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
2948 const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
30222949 return self.wip.icmp(cond, loaded, zero, "");
30232950}
30242951
3025fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3026 const o = self.object;
3027 const pt = self.pt;
3028 const zcu = pt.zcu;
2952fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
30292953 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
30302954 const operand = try self.resolveInst(ty_op.operand);
3031 const optional_ty = self.typeOf(ty_op.operand).childType(zcu);
3032 const payload_ty = optional_ty.optionalChild(zcu);
3033 if (!payload_ty.hasRuntimeBits(zcu)) {
3034 // We have a pointer to a zero-bit value and we need to return
3035 // a pointer to a zero-bit value.
3036 return operand;
3037 }
3038 if (optional_ty.optionalReprIsPayload(zcu)) {
3039 // The payload and the optional are the same value.
3040 return operand;
3041 }
3042 return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, "");
2955 // If `Type.optionalReprIsPayload`, then the address should be the same. Otherwise, optional
2956 // layouts always put the payload at offset 0, so... the address should still be the same.
2957 return operand;
30432958}
30442959
30452960fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -3072,9 +2987,9 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value
30722987 return operand;
30732988 }
30742989
3075 // First set the non-null bit.
3076 const optional_llvm_ty = try o.lowerType(pt, optional_ty);
3077 const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, "");
2990 // First set the non-null bit. It's always immediately after the payload (no padding) because it
2991 // has alignment 1.
2992 const non_null_ptr = try self.ptraddConst(operand, payload_ty.abiSize(zcu));
30782993
30792994 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
30802995
......@@ -3084,11 +2999,10 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value
30842999 // Then return the payload pointer (only if it's used).
30853000 if (self.liveness.isUnused(inst)) return .none;
30863001
3087 return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
3002 return operand; // payload is at offset 0
30883003}
30893004
30903005fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3091 const o = self.object;
30923006 const pt = self.pt;
30933007 const zcu = pt.zcu;
30943008 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
......@@ -3102,8 +3016,7 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
31023016 return operand;
31033017 }
31043018
3105 const opt_llvm_ty = try o.lowerType(pt, optional_ty);
3106 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
3019 return self.optPayloadHandle(operand, optional_ty, false);
31073020}
31083021
31093022fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
......@@ -3120,20 +3033,18 @@ fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool
31203033 if (!payload_ty.hasRuntimeBits(zcu)) {
31213034 return if (operand_is_ptr) operand else .none;
31223035 }
3123 const offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3124 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3036 const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
31253037 if (operand_is_ptr) {
3126 return self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
3127 } else if (isByRef(err_union_ty, zcu)) {
3128 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
3129 const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
3130 if (isByRef(payload_ty, zcu)) {
3131 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
3132 }
3133 const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
3038 return payload_ptr;
3039 }
3040 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3041 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
3042 if (isByRef(payload_ty, zcu)) {
3043 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
3044 } else {
3045 const payload_llvm_ty = try o.lowerType(pt, payload_ty);
31343046 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
31353047 }
3136 return self.wip.extractValue(operand, &.{offset}, "");
31373048}
31383049
31393050fn airErrUnionErr(
......@@ -3169,17 +3080,18 @@ fn airErrUnionErr(
31693080 return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
31703081 }
31713082
3172 const offset = try errUnionErrorFieldIndex(payload_ty, pt);
3083 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
31733084
3174 if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
3175 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3085 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
31763086
3177 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3178 const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
3179 return self.wip.load(access_kind, error_type, err_field_ptr, .default, "");
3180 }
3087 const err_align: InternPool.Alignment = a: {
3088 const err_abi_align = Type.anyerror.abiAlignment(zcu);
3089 if (!operand_is_ptr) break :a err_abi_align;
3090 break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
3091 };
31813092
3182 return self.wip.extractValue(operand, &.{offset}, "");
3093 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3094 return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
31833095}
31843096
31853097fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -3198,27 +3110,18 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value
31983110 const access_kind: Builder.MemoryAccessKind =
31993111 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
32003112
3201 if (!payload_ty.hasRuntimeBits(zcu)) {
3202 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3203 _ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm());
3204 return operand;
3205 }
3206 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3207 {
3208 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3113 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
32093114
3210 const err_int_ty = try pt.errorIntType();
3211 const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
3212 const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3115 {
3116 const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
32133117 // First set the non-error value.
3214 const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, "");
3215 _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment);
3118 const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3119 _ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);
32163120 }
3121
32173122 // Then return the payload pointer (only if it is used).
32183123 if (self.liveness.isUnused(inst)) return .none;
3219
3220 const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3221 return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, "");
3124 return self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
32223125}
32233126
32243127fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {
......@@ -3233,7 +3136,6 @@ fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
32333136}
32343137
32353138fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3236 const o = self.object;
32373139 const pt = self.pt;
32383140 const zcu = pt.zcu;
32393141
......@@ -3241,15 +3143,20 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Val
32413143 const struct_ty = ty_pl.ty.toType();
32423144 const field_index = ty_pl.payload;
32433145
3244 const struct_llvm_ty = try o.lowerType(pt, struct_ty);
3245 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
32463146 assert(self.err_ret_trace != .none);
3247 const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
3248 const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
3147
32493148 const field_ty = struct_ty.fieldType(field_index, zcu);
3149 const field_offset = struct_ty.structFieldOffset(field_index, zcu);
3150 const field_align = switch (field_offset) {
3151 0 => struct_ty.abiAlignment(zcu),
3152 else => struct_ty.abiAlignment(zcu).minStrict(.fromLog2Units(@ctz(field_offset))),
3153 };
3154
3155 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3156
32503157 const field_ptr_ty = try pt.ptrType(.{
32513158 .child = field_ty.toIntern(),
3252 .flags = .{ .alignment = field_alignment },
3159 .flags = .{ .alignment = field_align },
32533160 });
32543161 return self.load(field_ptr, field_ptr_ty);
32553162}
......@@ -3290,25 +3197,23 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.V
32903197 const operand = try self.resolveInst(ty_op.operand);
32913198 const optional_ty = self.typeOfIndex(inst);
32923199 if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3200 assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
32933201 const llvm_optional_ty = try o.lowerType(pt, optional_ty);
3294 if (isByRef(optional_ty, zcu)) {
3295 const directReturn = self.isNextRet(body_tail);
3296 const optional_ptr = if (directReturn)
3297 self.ret_ptr
3298 else brk: {
3299 const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3300 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3301 break :brk optional_ptr;
3302 };
3202 const optional_ptr = if (self.isNextRet(body_tail))
3203 self.ret_ptr
3204 else brk: {
3205 const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3206 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3207 break :brk optional_ptr;
3208 };
33033209
3304 const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, "");
3305 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3306 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3307 const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, "");
3308 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
3309 return optional_ptr;
3310 }
3311 return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, "");
3210 const payload_ptr = optional_ptr; // payload always at offset 0
3211 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3212 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3213 // Non-null bit immediately after payload (no padding because the bit has alignment 1).
3214 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3215 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
3216 return optional_ptr;
33123217}
33133218
33143219fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
......@@ -3321,34 +3226,25 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Bu
33213226 const operand = try self.resolveInst(ty_op.operand);
33223227 const payload_ty = self.typeOf(ty_op.operand);
33233228 assert(payload_ty.hasRuntimeBits(zcu));
3229 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
33243230 const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);
33253231 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
33263232
3327 const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3328 const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3329 if (isByRef(err_un_ty, zcu)) {
3330 const directReturn = self.isNextRet(body_tail);
3331 const result_ptr = if (directReturn)
3332 self.ret_ptr
3333 else brk: {
3334 const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
3335 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3336 break :brk result_ptr;
3337 };
3233 const result_ptr = if (self.isNextRet(body_tail))
3234 self.ret_ptr
3235 else brk: {
3236 const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
3237 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3238 break :brk result_ptr;
3239 };
33383240
3339 const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
3340 const err_int_ty = try pt.errorIntType();
3341 const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm();
3342 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
3343 const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
3344 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3345 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3346 return result_ptr;
3347 }
3348 var fields: [2]Builder.Value = undefined;
3349 fields[payload_offset] = operand;
3350 fields[error_offset] = ok_err_code;
3351 return self.wip.buildAggregate(err_un_llvm_ty, &fields, "");
3241 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3242 const error_alignment = Type.anyerror.abiAlignment(pt.zcu).toLlvm();
3243 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
3244 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3245 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3246 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3247 return result_ptr;
33523248}
33533249
33543250fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
......@@ -3361,35 +3257,26 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builde
33613257 const payload_ty = err_un_ty.errorUnionPayload(zcu);
33623258 const operand = try self.resolveInst(ty_op.operand);
33633259 if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3260 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
33643261 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
33653262
3366 const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3367 const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3368 if (isByRef(err_un_ty, zcu)) {
3369 const directReturn = self.isNextRet(body_tail);
3370 const result_ptr = if (directReturn)
3371 self.ret_ptr
3372 else brk: {
3373 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3374 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3375 break :brk result_ptr;
3376 };
3377
3378 const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
3379 const err_int_ty = try pt.errorIntType();
3380 const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
3381 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
3382 const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
3383 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3384 // TODO store undef to payload_ptr
3385 _ = payload_ptr;
3386 _ = payload_ptr_ty;
3387 return result_ptr;
3388 }
3263 const result_ptr = if (self.isNextRet(body_tail))
3264 self.ret_ptr
3265 else brk: {
3266 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3267 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3268 break :brk result_ptr;
3269 };
33893270
3390 // TODO set payload bytes to undef
3391 const undef = try o.builder.undefValue(err_un_llvm_ty);
3392 return self.wip.insertValue(undef, operand, &.{error_offset}, "");
3271 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3272 const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();
3273 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
3274 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3275 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3276 // TODO store undef to payload_ptr
3277 _ = payload_ptr;
3278 _ = payload_ptr_ty;
3279 return result_ptr;
33933280}
33943281
33953282fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -3804,26 +3691,18 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui
38043691}
38053692
38063693fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3807 const o = self.object;
3808 const pt = self.pt;
3809 const zcu = pt.zcu;
3694 const zcu = self.pt.zcu;
38103695 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
38113696 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3812 const ptr = try self.resolveInst(bin_op.lhs);
3813 const offset = try self.resolveInst(bin_op.rhs);
3697 const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3698 const index = try self.resolveInst(bin_op.rhs);
38143699 const ptr_ty = self.typeOf(bin_op.lhs);
3815 const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
3816 switch (ptr_ty.ptrSize(zcu)) {
3817 // It's a pointer to an array, so according to LLVM we need an extra GEP index.
3818 .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
3819 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset,
3820 }, ""),
3821 .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""),
3822 .slice => {
3823 const base = try self.wip.extractValue(ptr, &.{0}, "");
3824 return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, "");
3825 },
3826 }
3700 const elem_ty = ptr_ty.indexableElem(zcu);
3701 const ptr = switch (ptr_ty.ptrSize(zcu)) {
3702 .one, .many, .c => ptr_or_slice,
3703 .slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
3704 };
3705 return self.ptraddScaled(ptr, index, elem_ty.abiSize(zcu));
38273706}
38283707
38293708fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -3832,22 +3711,18 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
38323711 const zcu = pt.zcu;
38333712 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
38343713 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3835 const ptr = try self.resolveInst(bin_op.lhs);
3836 const offset = try self.resolveInst(bin_op.rhs);
3837 const negative_offset = try self.wip.neg(offset, "");
3714 const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3715 const llvm_usize_ty = try o.lowerType(pt, .usize);
38383716 const ptr_ty = self.typeOf(bin_op.lhs);
3839 const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
3840 switch (ptr_ty.ptrSize(zcu)) {
3841 // It's a pointer to an array, so according to LLVM we need an extra GEP index.
3842 .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
3843 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset,
3844 }, ""),
3845 .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""),
3846 .slice => {
3847 const base = try self.wip.extractValue(ptr, &.{0}, "");
3848 return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, "");
3849 },
3850 }
3717 const elem_ty = ptr_ty.indexableElem(zcu);
3718 const ptr = switch (ptr_ty.ptrSize(zcu)) {
3719 .one, .many, .c => ptr_or_slice,
3720 .slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
3721 };
3722 const scale_val = try o.builder.intValue(llvm_usize_ty, -@as(i65, elem_ty.abiSize(zcu)));
3723 const positive_index = try self.resolveInst(bin_op.rhs);
3724 const negative_offset = try self.wip.bin(.@"mul nsw", positive_index, scale_val, "");
3725 return self.ptradd(ptr, negative_offset);
38513726}
38523727
38533728fn airOverflow(
......@@ -3868,6 +3743,7 @@ fn airOverflow(
38683743 const lhs_ty = self.typeOf(extra.lhs);
38693744 const scalar_ty = lhs_ty.scalarType(zcu);
38703745 const inst_ty = self.typeOfIndex(inst);
3746 assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
38713747
38723748 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
38733749 const llvm_inst_ty = try o.lowerType(pt, inst_ty);
......@@ -3878,28 +3754,22 @@ fn airOverflow(
38783754 const result_val = try self.wip.extractValue(results, &.{0}, "");
38793755 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
38803756
3881 const result_index = o.llvmFieldIndex(inst_ty, 0).?;
3882 const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
3757 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
3758 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
38833759
3884 if (isByRef(inst_ty, zcu)) {
3885 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
3886 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
3887 {
3888 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
3889 _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
3890 }
3891 {
3892 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, "");
3893 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
3894 }
3760 {
3761 // Store to 'result: IntType' field
3762 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));
3763 _ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
3764 }
38953765
3896 return alloca_inst;
3766 {
3767 // Store to 'overflow: u1' field
3768 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));
3769 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
38973770 }
38983771
3899 var fields: [2]Builder.Value = undefined;
3900 fields[result_index] = result_val;
3901 fields[overflow_index] = overflow_bit;
3902 return self.wip.buildAggregate(llvm_inst_ty, &fields, "");
3772 return alloca_inst;
39033773}
39043774
39053775fn buildElementwiseCall(
......@@ -4227,6 +4097,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
42274097 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
42284098
42294099 const dest_ty = self.typeOfIndex(inst);
4100 assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
42304101 const llvm_dest_ty = try o.lowerType(pt, dest_ty);
42314102
42324103 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
......@@ -4239,27 +4110,22 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
42394110
42404111 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
42414112
4242 const result_index = o.llvmFieldIndex(dest_ty, 0).?;
4243 const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
4113 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
4114 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
42444115
4245 if (isByRef(dest_ty, zcu)) {
4246 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
4247 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
4248 {
4249 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
4250 _ = try self.wip.store(.normal, result, field_ptr, result_alignment);
4251 }
4252 {
4253 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, "");
4254 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
4255 }
4256 return alloca_inst;
4116 {
4117 // Store to 'result: IntType' field
4118 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));
4119 _ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
42574120 }
42584121
4259 var fields: [2]Builder.Value = undefined;
4260 fields[result_index] = result;
4261 fields[overflow_index] = overflow_bit;
4262 return self.wip.buildAggregate(llvm_dest_ty, &fields, "");
4122 {
4123 // Store to 'overflow: u1' field
4124 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));
4125 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
4126 }
4127
4128 return alloca_inst;
42634129}
42644130
42654131fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -4678,7 +4544,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
46784544 }
46794545
46804546 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
4681 const elem_ty = operand_ty.childType(zcu);
4547 const elem_ty = inst_ty.childType(zcu);
4548 assert(elem_ty.toIntern() == operand_scalar_ty.toIntern());
46824549 if (!result_is_ref) {
46834550 return self.todo("implement bitcast vector to non-ref array", .{});
46844551 }
......@@ -4690,22 +4557,19 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
46904557 } else {
46914558 // If the ABI size of the element type is not evenly divisible by size in bits;
46924559 // a simple bitcast will not work, and we fall back to extractelement.
4693 const llvm_usize = try o.lowerType(pt, Type.usize);
4694 const usize_zero = try o.builder.intValue(llvm_usize, 0);
4560 const elem_size = elem_ty.abiSize(zcu);
46954561 const vector_len = operand_ty.arrayLen(zcu);
46964562 var i: u64 = 0;
46974563 while (i < vector_len) : (i += 1) {
4698 const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{
4699 usize_zero, try o.builder.intValue(llvm_usize, i),
4700 }, "");
4701 const elem =
4702 try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4703 _ = try self.wip.store(.normal, elem, elem_ptr, .default);
4564 const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
4565 const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4566 _ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);
47044567 }
47054568 }
47064569 return array_ptr;
47074570 } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
47084571 const elem_ty = operand_ty.childType(zcu);
4572 assert(elem_ty.toIntern() == inst_scalar_ty.toIntern());
47094573 const llvm_vector_ty = try o.lowerType(pt, inst_ty);
47104574 if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{});
47114575
......@@ -4718,20 +4582,15 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
47184582 } else {
47194583 // If the ABI size of the element type is not evenly divisible by size in bits;
47204584 // a simple bitcast will not work, and we fall back to extractelement.
4721 const array_llvm_ty = try o.lowerType(pt, operand_ty);
47224585 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
4723 const llvm_usize = try o.lowerType(pt, Type.usize);
4724 const usize_zero = try o.builder.intValue(llvm_usize, 0);
4586 const elem_size = elem_ty.abiSize(zcu);
47254587 const vector_len = operand_ty.arrayLen(zcu);
47264588 var vector = try o.builder.poisonValue(llvm_vector_ty);
47274589 var i: u64 = 0;
47284590 while (i < vector_len) : (i += 1) {
4729 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{
4730 usize_zero, try o.builder.intValue(llvm_usize, i),
4731 }, "");
4732 const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, "");
4733 vector =
4734 try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), "");
4591 const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
4592 const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");
4593 vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
47354594 }
47364595 return vector;
47374596 }
......@@ -5066,10 +4925,23 @@ fn airCmpxchg(
50664925 return self.wip.select(.normal, success_bit, zero, payload, "");
50674926 }
50684927
4928 assert(isByRef(optional_ty, zcu));
4929
50694930 comptime assert(optional_layout_version == 3);
50704931
50714932 const non_null_bit = try self.wip.not(success_bit, "");
5072 return buildOptional(self, optional_ty, payload, non_null_bit);
4933
4934 const payload_align = operand_ty.abiAlignment(zcu).toLlvm();
4935 const alloca_inst = try self.buildAlloca(try o.lowerType(pt, optional_ty), payload_align);
4936
4937 // Payload is always the first field at offset 0, so address is `alloca_inst`
4938 _ = try self.wip.store(.normal, payload, alloca_inst, payload_align);
4939
4940 // Non-null bit is after payload with no padding because it has alignment 1
4941 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
4942 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));
4943
4944 return alloca_inst;
50734945}
50744946
50754947fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -5315,13 +5187,15 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value
53155187 const end_block = try self.wip.block(1, "InlineMemsetEnd");
53165188
53175189 const llvm_usize_ty = try o.lowerType(pt, Type.usize);
5318 const len = switch (ptr_ty.ptrSize(zcu)) {
5319 .slice => try self.wip.extractValue(dest_slice, &.{1}, ""),
5320 .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)),
5190 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
5191 .slice => try self.ptraddScaled(
5192 dest_ptr,
5193 try self.wip.extractValue(dest_slice, &.{1}, ""),
5194 elem_abi_size,
5195 ),
5196 .one => try self.ptraddConst(dest_ptr, ptr_ty.childType(zcu).abiSize(zcu)),
53215197 .many, .c => unreachable,
53225198 };
5323 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
5324 const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, "");
53255199 _ = try self.wip.br(loop_block);
53265200
53275201 self.wip.cursor = .{ .block = loop_block };
......@@ -5343,9 +5217,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value
53435217 self.disable_intrinsics,
53445218 );
53455219 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
5346 const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
5347 try o.builder.intValue(llvm_usize_ty, 1),
5348 }, "");
5220 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
53495221 _ = try self.wip.br(loop_block);
53505222
53515223 self.wip.cursor = .{ .block = end_block };
......@@ -5408,14 +5280,13 @@ fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
54085280}
54095281
54105282fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5411 const o = self.object;
54125283 const pt = self.pt;
54135284 const zcu = pt.zcu;
54145285 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
54155286 const un_ptr_ty = self.typeOf(bin_op.lhs);
54165287 const un_ty = un_ptr_ty.childType(zcu);
54175288 const layout = un_ty.unionGetLayout(zcu);
5418 if (layout.tag_size == 0) return .none;
5289 assert(layout.tag_size != 0);
54195290
54205291 const access_kind: Builder.MemoryAccessKind =
54215292 if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
......@@ -5429,8 +5300,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
54295300 _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
54305301 return .none;
54315302 }
5432 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5433 const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, "");
5303 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
54345304 const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {
54355305 0 => union_ptr_align,
54365306 else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),
......@@ -5446,20 +5316,20 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
54465316 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
54475317 const un_ty = self.typeOf(ty_op.operand);
54485318 const layout = un_ty.unionGetLayout(zcu);
5449 if (layout.tag_size == 0) return .none;
5450 const union_handle = try self.resolveInst(ty_op.operand);
5319 assert(layout.tag_size != 0);
5320 const union_ptr = try self.resolveInst(ty_op.operand);
54515321 if (isByRef(un_ty, zcu)) {
54525322 const llvm_un_ty = try o.lowerType(pt, un_ty);
54535323 if (layout.payload_size == 0)
5454 return self.wip.load(.normal, llvm_un_ty, union_handle, .default, "");
5324 return self.wip.load(.normal, llvm_un_ty, union_ptr, .default, "");
54555325 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5456 const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, "");
5326 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
54575327 const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];
54585328 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
54595329 } else {
5460 if (layout.payload_size == 0) return union_handle;
5330 if (layout.payload_size == 0) return union_ptr;
54615331 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5462 return self.wip.extractValue(union_handle, &.{tag_index}, "");
5332 return self.wip.extractValue(union_ptr, &.{tag_index}, "");
54635333 }
54645334}
54655335
......@@ -5646,19 +5516,18 @@ fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
56465516fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
56475517 const o = self.object;
56485518 const pt = self.pt;
5519 const zcu = pt.zcu;
56495520 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
56505521 const operand = try self.resolveInst(un_op);
56515522 const slice_ty = self.typeOfIndex(inst);
56525523 const slice_llvm_ty = try o.lowerType(pt, slice_ty);
56535524
56545525 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5655 const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
5526 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
56565527
56575528 const error_name_table_ptr = try self.getErrorNameTable();
5658 const error_name_table =
5659 try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
5660 const error_name_ptr =
5661 try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, "");
5529 const error_name_table = try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
5530 const error_name_ptr = try self.ptraddScaled(error_name_table, operand_usize, slice_ty.abiSize(zcu));
56625531 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
56635532}
56645533
......@@ -6074,8 +5943,9 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
60745943 }
60755944 return vector;
60765945 },
6077 .@"struct" => {
6078 if (zcu.typeToPackedStruct(result_ty)) |struct_type| {
5946 .@"struct" => switch (result_ty.containerLayout(zcu)) {
5947 .@"packed" => {
5948 const struct_type = ip.loadStructType(result_ty.toIntern());
60795949 const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
60805950 const big_bits = backing_int_ty.bitSize(zcu);
60815951 const int_ty = try o.builder.intType(@intCast(big_bits));
......@@ -6100,69 +5970,68 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
61005970 running_bits += ty_bit_size;
61015971 }
61025972 return running_int;
6103 }
6104
6105 assert(result_ty.containerLayout(zcu) != .@"packed");
6106
6107 if (isByRef(result_ty, zcu)) {
5973 },
5974 .auto, .@"extern" => {
5975 assert(isByRef(result_ty, zcu));
61085976 // TODO in debug builds init to undef so that the padding will be 0xaa
61095977 // even if we fully populate the fields.
6110 const alignment = result_ty.abiAlignment(zcu).toLlvm();
6111 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
6112
6113 for (elements, 0..) |elem, i| {
6114 if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6115
6116 const llvm_elem = try self.resolveInst(elem);
6117 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
6118 const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
6119
6120 const field_ptr_ty = try pt.ptrType(.{
6121 .child = self.typeOf(elem).toIntern(),
6122 .flags = .{
6123 .alignment = result_ty.explicitFieldAlignment(i, zcu),
6124 },
6125 });
6126 try self.store(field_ptr, field_ptr_ty, llvm_elem, .none);
6127 }
5978 const struct_align = result_ty.abiAlignment(zcu);
5979 const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());
61285980
6129 return alloca_inst;
6130 } else {
6131 var result = try o.builder.poisonValue(llvm_result_ty);
6132 for (elements, 0..) |elem, i| {
6133 if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
5981 for (elements, 0..) |elem, field_index| {
5982 if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
5983 const field_ty = result_ty.fieldType(field_index, zcu);
5984 if (!field_ty.hasRuntimeBits(zcu)) continue;
5985 const offset = result_ty.structFieldOffset(field_index, zcu);
5986 const field_ptr = try self.ptraddConst(alloca_inst, offset);
5987 const field_ptr_align: InternPool.Alignment = switch (offset) {
5988 0 => struct_align,
5989 else => struct_align.minStrict(.fromLog2Units(@ctz(offset))),
5990 };
61345991
6135 const llvm_elem = try self.resolveInst(elem);
6136 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
6137 result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
5992 const llvm_field_val = try self.resolveInst(elem);
5993
5994 if (isByRef(field_ty, zcu)) {
5995 _ = try self.wip.callMemCpy(
5996 field_ptr,
5997 field_ptr_align.toLlvm(),
5998 llvm_field_val,
5999 field_ty.abiAlignment(zcu).toLlvm(),
6000 try o.builder.intValue(try o.lowerType(pt, .usize), field_ty.abiSize(zcu)),
6001 .normal,
6002 self.disable_intrinsics,
6003 );
6004 } else {
6005 _ = try self.wip.store(
6006 .normal,
6007 llvm_field_val,
6008 field_ptr,
6009 field_ptr_align.toLlvm(),
6010 );
6011 }
61386012 }
6139 return result;
6140 }
6013
6014 return alloca_inst;
6015 },
61416016 },
61426017 .array => {
61436018 assert(isByRef(result_ty, zcu));
61446019
6145 const llvm_usize = try o.lowerType(pt, Type.usize);
6146 const usize_zero = try o.builder.intValue(llvm_usize, 0);
61476020 const alignment = result_ty.abiAlignment(zcu).toLlvm();
61486021 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
61496022
61506023 const array_info = result_ty.arrayInfo(zcu);
6151 const elem_ptr_ty = try pt.ptrType(.{
6152 .child = array_info.elem_type.toIntern(),
6153 });
6024 const elem_ptr_ty = try pt.singleConstPtrType(array_info.elem_type);
6025
6026 const elem_size = array_info.elem_type.abiSize(zcu);
61546027
61556028 for (elements, 0..) |elem, i| {
6156 const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
6157 usize_zero, try o.builder.intValue(llvm_usize, i),
6158 }, "");
6029 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
61596030 const llvm_elem = try self.resolveInst(elem);
61606031 try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);
61616032 }
61626033 if (array_info.sentinel) |sent_val| {
6163 const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
6164 usize_zero, try o.builder.intValue(llvm_usize, array_info.len),
6165 }, "");
6034 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
61666035 const llvm_elem = try self.resolveValue(sent_val);
61676036 try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);
61686037 }
......@@ -6188,95 +6057,30 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
61886057
61896058 const layout = Type.getUnionLayout(union_obj, zcu);
61906059
6191 const tag_int_val = blk: {
6192 const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
6193 const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
6194 break :blk tag_val.intFromEnum(zcu);
6195 };
6196 if (layout.payload_size == 0) {
6197 if (layout.tag_size == 0) {
6198 return .none;
6199 }
6200 assert(!isByRef(union_ty, zcu));
6201 var big_int_space: Value.BigIntSpace = undefined;
6202 const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
6203 return try o.builder.bigIntValue(union_llvm_ty, tag_big_int);
6204 }
6060 assert(layout.payload_size != 0); // otherwise the value would be comptime-known
62056061 assert(isByRef(union_ty, zcu));
6206 // The llvm type of the alloca will be the named LLVM union type, and will not
6207 // necessarily match the format that we need, depending on which tag is active.
6208 // We must construct the correct unnamed struct type here, in order to then set
6209 // the fields appropriately.
6062
62106063 const alignment = layout.abi_align.toLlvm();
62116064 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
62126065 const llvm_payload = try self.resolveInst(extra.init);
62136066 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6214 const field_llvm_ty = try o.lowerType(pt, field_ty);
6215 const field_size = field_ty.abiSize(zcu);
6216 const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu);
6217 const llvm_usize = try o.lowerType(pt, Type.usize);
6218 const usize_zero = try o.builder.intValue(llvm_usize, 0);
6219
62206067 assert(field_ty.hasRuntimeBits(zcu));
62216068
6222 const llvm_union_ty = t: {
6223 const payload_ty = p: {
6224 if (field_size == layout.payload_size) {
6225 break :p field_llvm_ty;
6226 }
6227 const padding_len = layout.payload_size - field_size;
6228 break :p try o.builder.structType(.@"packed", &.{
6229 field_llvm_ty, try o.builder.arrayType(padding_len, .i8),
6230 });
6231 };
6232 if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty});
6233 const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
6234 var fields: [3]Builder.Type = undefined;
6235 var fields_len: usize = 2;
6236 if (layout.tag_align.compare(.gte, layout.payload_align)) {
6237 fields = .{ tag_ty, payload_ty, undefined };
6238 } else {
6239 fields = .{ payload_ty, tag_ty, undefined };
6240 }
6241 if (layout.padding != 0) {
6242 fields[fields_len] = try o.builder.arrayType(layout.padding, .i8);
6243 fields_len += 1;
6244 }
6245 break :t try o.builder.structType(.normal, fields[0..fields_len]);
6246 };
6247
6248 // Now we follow the layout as expressed above with GEP instructions to set the
6249 // tag and the payload.
6250 const field_ptr_ty = try pt.ptrType(.{
6251 .child = field_ty.toIntern(),
6252 .flags = .{ .alignment = field_align },
6253 });
6254 if (layout.tag_size == 0) {
6255 const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" };
6256 const len: usize = if (field_size == layout.payload_size) 2 else 3;
6257 const field_ptr =
6258 try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
6259 try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
6260 return result_ptr;
6261 }
6262
62636069 {
6264 const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
6265 const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" };
6266 const len: usize = if (field_size == layout.payload_size) 2 else 3;
6267 const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
6268 try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
6070 const payload_ptr_ty = try pt.ptrType(.{
6071 .child = field_ty.toIntern(),
6072 .flags = .{ .alignment = layout.payload_align },
6073 });
6074 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6075 try self.store(payload_ptr, payload_ptr_ty, llvm_payload, .none);
62696076 }
6270 {
6271 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
6272 const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) };
6273 const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, "");
6274 const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
6275 var big_int_space: Value.BigIntSpace = undefined;
6276 const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
6277 const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int);
6278 const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm();
6279 _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment);
6077
6078 if (layout.tag_size != 0) {
6079 const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
6080 const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
6081 const llvm_tag_val = try o.lowerValue(pt, tag_val.toIntern());
6082 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6083 _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());
62806084 }
62816085
62826086 return result_ptr;
......@@ -6369,7 +6173,6 @@ fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
63696173}
63706174
63716175fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6372 const o = self.object;
63736176 const pt = self.pt;
63746177 const target = pt.zcu.getTarget();
63756178
......@@ -6387,11 +6190,8 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
63876190
63886191 // Load the work_group_* member from the struct as u16.
63896192 // Just treat the dispatch pointer as an array of u16 to keep things simple.
6390 const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{
6391 try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension),
6392 }, "");
6393 const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2);
6394 return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, "");
6193 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6194 return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");
63956195 },
63966196 .nvptx, .nvptx64 => {
63976197 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
......@@ -6435,88 +6235,45 @@ fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {
64356235 return variable_index;
64366236}
64376237
6438/// Assumes the optional is not pointer-like and payload has bits.
6238/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
64396239fn optCmpNull(
64406240 self: *FuncGen,
64416241 cond: Builder.IntegerCondition,
6442 opt_llvm_ty: Builder.Type,
6443 opt_handle: Builder.Value,
6444 is_by_ref: bool,
6242 opt_ty: Type,
6243 opt_ptr: Builder.Value,
64456244 access_kind: Builder.MemoryAccessKind,
64466245) Allocator.Error!Builder.Value {
6447 const o = self.object;
6448 const field = b: {
6449 if (is_by_ref) {
6450 const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, "");
6451 break :b try self.wip.load(access_kind, .i8, field_ptr, .default, "");
6452 }
6453 break :b try self.wip.extractValue(opt_handle, &.{1}, "");
6454 };
6246 const zcu = self.pt.zcu;
6247 assert(isByRef(opt_ty, zcu));
64556248 comptime assert(optional_layout_version == 3);
6456
6457 return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), "");
6249 // Non-null bit is always after the payload, with no padding because it has alignment 1.
6250 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6251 const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");
6252 return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");
64586253}
64596254
6460/// Assumes the optional is not pointer-like and payload has bits.
6255/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
64616256fn optPayloadHandle(
64626257 fg: *FuncGen,
6463 opt_llvm_ty: Builder.Type,
6464 opt_handle: Builder.Value,
6258 opt_ptr: Builder.Value,
64656259 opt_ty: Type,
64666260 can_elide_load: bool,
64676261) !Builder.Value {
64686262 const pt = fg.pt;
64696263 const zcu = pt.zcu;
6264 assert(isByRef(opt_ty, zcu));
64706265 const payload_ty = opt_ty.optionalChild(zcu);
64716266
6472 if (isByRef(opt_ty, zcu)) {
6473 // We have a pointer and we need to return a pointer to the first field.
6474 const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, "");
6475
6476 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
6477 if (isByRef(payload_ty, zcu)) {
6478 if (can_elide_load)
6479 return payload_ptr;
6267 // Payload is first field so always at the same address as the optional itself.
6268 const payload_ptr = opt_ptr;
64806269
6481 return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
6482 }
6483 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment);
6484 }
6485
6486 assert(!isByRef(payload_ty, zcu));
6487 return fg.wip.extractValue(opt_handle, &.{0}, "");
6488}
6489
6490fn buildOptional(
6491 self: *FuncGen,
6492 optional_ty: Type,
6493 payload: Builder.Value,
6494 non_null_bit: Builder.Value,
6495) !Builder.Value {
6496 const o = self.object;
6497 const pt = self.pt;
6498 const zcu = pt.zcu;
6499 const optional_llvm_ty = try o.lowerType(pt, optional_ty);
6500 const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, "");
6501
6502 if (isByRef(optional_ty, zcu)) {
6503 const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm();
6504 const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
6505
6506 {
6507 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
6508 _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment);
6509 }
6510 {
6511 const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1);
6512 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, "");
6513 _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment);
6514 }
6515
6516 return alloca_inst;
6270 const payload_align = payload_ty.abiAlignment(zcu).toLlvm();
6271 if (isByRef(payload_ty, zcu)) {
6272 if (can_elide_load) return payload_ptr;
6273 return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
6274 } else {
6275 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
65176276 }
6518
6519 return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, "");
65206277}
65216278
65226279fn fieldPtr(
......@@ -6525,7 +6282,6 @@ fn fieldPtr(
65256282 aggregate_ptr_ty: Type,
65266283 field_index: u32,
65276284) !Builder.Value {
6528 const o = self.object;
65296285 const pt = self.pt;
65306286 const zcu = pt.zcu;
65316287 const aggregate_ty = aggregate_ptr_ty.childType(zcu);
......@@ -6534,35 +6290,12 @@ fn fieldPtr(
65346290 // bit offset is represented in the pointer *type*.
65356291 return aggregate_ptr;
65366292 }
6537 switch (aggregate_ty.zigTypeTag(zcu)) {
6538 .@"struct" => {
6539 if (!aggregate_ty.hasRuntimeBits(zcu)) {
6540 return aggregate_ptr;
6541 }
6542 const struct_llvm_ty = try o.lowerType(pt, aggregate_ty);
6543 if (o.llvmFieldIndex(aggregate_ty, field_index)) |llvm_field_index| {
6544 return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, "");
6545 } else {
6546 // If we found no index then this means this is a zero sized field at the
6547 // end of the struct. Treat our struct pointer as an array of two and get
6548 // the index to the element at index `1` to get a pointer to the end of
6549 // the struct.
6550 const llvm_index = try o.builder.intValue(
6551 try o.lowerType(pt, Type.usize),
6552 @intFromBool(aggregate_ty.hasRuntimeBits(zcu)),
6553 );
6554 return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, "");
6555 }
6556 },
6557 .@"union" => {
6558 const layout = aggregate_ty.unionGetLayout(zcu);
6559 if (layout.payload_size == 0) return aggregate_ptr;
6560 const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
6561 const union_llvm_ty = try o.lowerType(pt, aggregate_ty);
6562 return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, "");
6563 },
6293 const offset: u64 = switch (aggregate_ty.zigTypeTag(zcu)) {
6294 .@"struct" => aggregate_ty.structFieldOffset(field_index, zcu),
6295 .@"union" => aggregate_ty.unionGetLayout(zcu).payloadOffset(),
65646296 else => unreachable,
6565 }
6297 };
6298 return self.ptraddConst(aggregate_ptr, offset);
65666299}
65676300
65686301/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
......@@ -6828,11 +6561,8 @@ fn valgrindClientRequest(
68286561 break :a array_ptr;
68296562 } else fg.valgrind_client_request_array;
68306563 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6831 const zero = try o.builder.intValue(llvm_usize, 0);
68326564 for (array_elements, 0..) |elem, i| {
6833 const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{
6834 zero, try o.builder.intValue(llvm_usize, i),
6835 }, "");
6565 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
68366566 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
68376567 }
68386568
......@@ -7605,13 +7335,8 @@ pub fn buildAllocaInner(
76057335
76067336/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
76077337/// or as an LLVM value.
7608pub fn isByRef(ty: Type, zcu: *Zcu) bool {
7609 // For tuples and structs, if there are more than this many non-void
7610 // fields, then we make it byref, otherwise byval.
7611 const max_fields_byval = 0;
7612 const ip = &zcu.intern_pool;
7613
7614 switch (ty.zigTypeTag(zcu)) {
7338pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7339 return switch (ty.zigTypeTag(zcu)) {
76157340 .type,
76167341 .comptime_int,
76177342 .comptime_float,
......@@ -7632,62 +7357,24 @@ pub fn isByRef(ty: Type, zcu: *Zcu) bool {
76327357 .@"enum",
76337358 .vector,
76347359 .@"anyframe",
7635 => return false,
7636
7637 .array, .frame => return ty.hasRuntimeBits(zcu),
7638 .@"struct" => {
7639 const struct_type = switch (ip.indexToKey(ty.toIntern())) {
7640 .tuple_type => |tuple| {
7641 var count: usize = 0;
7642 for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
7643 if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
7644
7645 count += 1;
7646 if (count > max_fields_byval) return true;
7647 if (isByRef(Type.fromInterned(field_ty), zcu)) return true;
7648 }
7649 return false;
7650 },
7651 .struct_type => ip.loadStructType(ty.toIntern()),
7652 else => unreachable,
7653 };
7360 => false,
76547361
7655 // Packed structs are represented to LLVM as integers.
7656 if (struct_type.layout == .@"packed") return false;
7657
7658 const field_types = struct_type.field_types.get(ip);
7659 var it = struct_type.iterateRuntimeOrder(ip);
7660 var count: usize = 0;
7661 while (it.next()) |field_index| {
7662 count += 1;
7663 if (count > max_fields_byval) return true;
7664 const field_ty = Type.fromInterned(field_types[field_index]);
7665 if (isByRef(field_ty, zcu)) return true;
7666 }
7667 return false;
7362 .array,
7363 .error_union,
7364 .frame,
7365 => ty.hasRuntimeBits(zcu),
7366
7367 .optional => ty.hasRuntimeBits(zcu) and !ty.optionalReprIsPayload(zcu),
7368
7369 .@"struct" => switch (ty.containerLayout(zcu)) {
7370 .@"packed" => false,
7371 .auto, .@"extern" => ty.hasRuntimeBits(zcu),
76687372 },
76697373 .@"union" => switch (ty.containerLayout(zcu)) {
7670 .@"packed" => return false,
7671 else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
7672 },
7673 .error_union => {
7674 const payload_ty = ty.errorUnionPayload(zcu);
7675 if (!payload_ty.hasRuntimeBits(zcu)) {
7676 return false;
7677 }
7678 return true;
7374 .@"packed" => false,
7375 else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
76797376 },
7680 .optional => {
7681 const payload_ty = ty.optionalChild(zcu);
7682 if (!payload_ty.hasRuntimeBits(zcu)) {
7683 return false;
7684 }
7685 if (ty.optionalReprIsPayload(zcu)) {
7686 return false;
7687 }
7688 return true;
7689 },
7690 }
7377 };
76917378}
76927379
76937380/// If the operand type of an atomic operation is not byte sized we need to
......@@ -7713,16 +7400,27 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
77137400 }
77147401}
77157402
7716fn errUnionPayloadFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 {
7717 const zcu = pt.zcu;
7718 const err_int_ty = try pt.errorIntType();
7719 return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu)));
7403fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7404 if (offset == 0) return ptr;
7405 const llvm_usize_ty = try fg.object.lowerType(fg.pt, .usize);
7406 const offset_val = try fg.object.builder.intValue(llvm_usize_ty, offset);
7407 return fg.ptradd(ptr, offset_val);
77207408}
7721
7722fn errUnionErrorFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 {
7723 const zcu = pt.zcu;
7724 const err_int_ty = try pt.errorIntType();
7725 return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu)));
7409fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
7410 switch (scale) {
7411 0 => return ptr,
7412 1 => return fg.ptradd(ptr, index),
7413 else => {
7414 const o = fg.object;
7415 const llvm_usize_ty = try o.lowerType(fg.pt, .usize);
7416 const scale_val = try o.builder.intValue(llvm_usize_ty, scale);
7417 const offset = try fg.wip.bin(.@"mul nuw", index, scale_val, "");
7418 return fg.ptradd(ptr, offset);
7419 },
7420 }
7421}
7422fn ptradd(fg: *FuncGen, ptr: Builder.Value, offset: Builder.Value) Allocator.Error!Builder.Value {
7423 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset}, "");
77267424}
77277425
77287426fn compilerRtIntBits(bits: u16) ?u16 {
......@@ -8128,6 +7826,7 @@ const Package = @import("../../Package.zig");
81287826const InternPool = @import("../../InternPool.zig");
81297827const Value = @import("../../Value.zig");
81307828const Type = @import("../../Type.zig");
7829const codegen = @import("../../codegen.zig");
81317830
81327831const target_util = @import("../../target.zig");
81337832const libcFloatPrefix = target_util.libcFloatPrefix;