authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-17 18:12:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-18 00:09:30-07:00
logc970dbdfeca60de227bba53b155c8374ae90f919
tree5e5a06f78fdb339d98af490b348759575d9c8b52
parent48e2ba3b3c1e224b59e3c97ed462ac88df4d8a4b

LLVM: cache LLVM struct field indexes

This is an optimization to avoid O(N) field index lookups. It's also nicer in terms of DRY; the only tradeoff is memory usage.

1 files changed, 63 insertions(+), 107 deletions(-)

src/codegen/llvm.zig+63-107
......@@ -825,6 +825,18 @@ pub const Object = struct {
825825 /// Memoizes a null `?usize` value.
826826 null_opt_usize: Builder.Constant,
827827
828 /// When an LLVM struct type is created, an entry is inserted into this
829 /// table for every zig source field of the struct that has a corresponding
830 /// LLVM struct field. comptime fields and 0 bit fields are not included.
831 /// The value is the LLVM struct field index.
832 /// This is denormalized data.
833 struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint),
834
835 const ZigStructField = struct {
836 struct_ty: InternPool.Index,
837 field_index: u32,
838 };
839
828840 pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
829841
830842 /// This is an ArrayHashMap as opposed to a HashMap because in `flushModule` we
......@@ -979,6 +991,7 @@ pub const Object = struct {
979991 .error_name_table = .none,
980992 .extern_collisions = .{},
981993 .null_opt_usize = .no_init,
994 .struct_field_map = .{},
982995 };
983996 }
984997
......@@ -994,6 +1007,7 @@ pub const Object = struct {
9941007 self.type_map.deinit(gpa);
9951008 self.extern_collisions.deinit(gpa);
9961009 self.builder.deinit();
1010 self.struct_field_map.deinit(gpa);
9971011 self.* = undefined;
9981012 }
9991013
......@@ -3274,7 +3288,10 @@ pub const Object = struct {
32743288
32753289 var llvm_field_types = std.ArrayListUnmanaged(Builder.Type){};
32763290 defer llvm_field_types.deinit(o.gpa);
3291 // Although we can estimate how much capacity to add, these cannot be
3292 // relied upon because of the recursive calls to lowerType below.
32773293 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_obj.fields.count());
3294 try o.struct_field_map.ensureUnusedCapacity(o.gpa, @intCast(struct_obj.fields.count()));
32783295
32793296 comptime assert(struct_layout_version == 2);
32803297 var offset: u64 = 0;
......@@ -3296,6 +3313,10 @@ pub const Object = struct {
32963313 o.gpa,
32973314 try o.builder.arrayType(padding_len, .i8),
32983315 );
3316 try o.struct_field_map.put(o.gpa, .{
3317 .struct_ty = t.toIntern(),
3318 .field_index = field_and_index.index,
3319 }, @intCast(llvm_field_types.items.len));
32993320 try llvm_field_types.append(o.gpa, try o.lowerType(field.ty));
33003321
33013322 offset += field.ty.abiSize(mod);
......@@ -3319,7 +3340,10 @@ pub const Object = struct {
33193340 .anon_struct_type => |anon_struct_type| {
33203341 var llvm_field_types: std.ArrayListUnmanaged(Builder.Type) = .{};
33213342 defer llvm_field_types.deinit(o.gpa);
3343 // Although we can estimate how much capacity to add, these cannot be
3344 // relied upon because of the recursive calls to lowerType below.
33223345 try llvm_field_types.ensureUnusedCapacity(o.gpa, anon_struct_type.types.len);
3346 try o.struct_field_map.ensureUnusedCapacity(o.gpa, anon_struct_type.types.len);
33233347
33243348 comptime assert(struct_layout_version == 2);
33253349 var offset: u64 = 0;
......@@ -3328,7 +3352,8 @@ pub const Object = struct {
33283352 for (
33293353 anon_struct_type.types.get(ip),
33303354 anon_struct_type.values.get(ip),
3331 ) |field_ty, field_val| {
3355 0..,
3356 ) |field_ty, field_val, field_index| {
33323357 if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;
33333358
33343359 const field_align = field_ty.toType().abiAlignment(mod);
......@@ -3341,6 +3366,10 @@ pub const Object = struct {
33413366 o.gpa,
33423367 try o.builder.arrayType(padding_len, .i8),
33433368 );
3369 try o.struct_field_map.put(o.gpa, .{
3370 .struct_ty = t.toIntern(),
3371 .field_index = @intCast(field_index),
3372 }, @intCast(llvm_field_types.items.len));
33443373 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty.toType()));
33453374
33463375 offset += field_ty.toType().abiSize(mod);
......@@ -4237,9 +4266,9 @@ pub const Object = struct {
42374266 try o.lowerType(parent_ty),
42384267 parent_ptr,
42394268 null,
4240 if (llvmField(parent_ty, field_index, mod)) |llvm_field| &.{
4269 if (o.llvmFieldIndex(parent_ty, field_index)) |llvm_field_index| &.{
42414270 try o.builder.intConst(.i32, 0),
4242 try o.builder.intConst(.i32, llvm_field.index),
4271 try o.builder.intConst(.i32, llvm_field_index),
42434272 } else &.{
42444273 try o.builder.intConst(.i32, @intFromBool(
42454274 parent_ty.hasRuntimeBitsIgnoreComptime(mod),
......@@ -4396,6 +4425,13 @@ pub const Object = struct {
43964425 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = alignment }, &o.builder);
43974426 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
43984427 }
4428
4429 fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4430 return o.struct_field_map.get(.{
4431 .struct_ty = struct_ty.toIntern(),
4432 .field_index = @intCast(field_index),
4433 });
4434 }
43994435};
44004436
44014437pub const DeclGen = struct {
......@@ -6142,7 +6178,7 @@ pub const FuncGen = struct {
61426178 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
61436179 },
61446180 else => {
6145 const llvm_field_index = llvmField(struct_ty, field_index, mod).?.index;
6181 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
61466182 return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
61476183 },
61486184 },
......@@ -6169,23 +6205,25 @@ pub const FuncGen = struct {
61696205
61706206 switch (struct_ty.zigTypeTag(mod)) {
61716207 .Struct => {
6172 assert(struct_ty.containerLayout(mod) != .Packed);
6173 const llvm_field = llvmField(struct_ty, field_index, mod).?;
6208 const layout = struct_ty.containerLayout(mod);
6209 assert(layout != .Packed);
61746210 const struct_llvm_ty = try o.lowerType(struct_ty);
6211 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
61756212 const field_ptr =
6176 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field.index, "");
6213 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
6214 const alignment = struct_ty.structFieldAlign(field_index, mod);
61776215 const field_ptr_ty = try mod.ptrType(.{
6178 .child = llvm_field.ty.toIntern(),
6216 .child = field_ty.toIntern(),
61796217 .flags = .{
6180 .alignment = InternPool.Alignment.fromNonzeroByteUnits(llvm_field.alignment),
6218 .alignment = InternPool.Alignment.fromNonzeroByteUnits(alignment),
61816219 },
61826220 });
61836221 if (isByRef(field_ty, mod)) {
61846222 if (canElideLoad(self, body_tail))
61856223 return field_ptr;
61866224
6187 assert(llvm_field.alignment != 0);
6188 const field_alignment = Builder.Alignment.fromByteUnits(llvm_field.alignment);
6225 assert(alignment != 0);
6226 const field_alignment = Builder.Alignment.fromByteUnits(alignment);
61896227 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);
61906228 } else {
61916229 return self.load(field_ptr, field_ptr_ty);
......@@ -7080,15 +7118,17 @@ pub const FuncGen = struct {
70807118 const field_index = ty_pl.payload;
70817119
70827120 const mod = o.module;
7083 const llvm_field = llvmField(struct_ty, field_index, mod).?;
70847121 const struct_llvm_ty = try o.lowerType(struct_ty);
7122 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
70857123 assert(self.err_ret_trace != .none);
70867124 const field_ptr =
7087 try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field.index, "");
7125 try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
7126 const field_alignment = struct_ty.structFieldAlign(field_index, mod);
7127 const field_ty = struct_ty.structFieldType(field_index, mod);
70887128 const field_ptr_ty = try mod.ptrType(.{
7089 .child = llvm_field.ty.toIntern(),
7129 .child = field_ty.toIntern(),
70907130 .flags = .{
7091 .alignment = InternPool.Alignment.fromNonzeroByteUnits(llvm_field.alignment),
7131 .alignment = InternPool.Alignment.fromNonzeroByteUnits(field_alignment),
70927132 },
70937133 });
70947134 return self.load(field_ptr, field_ptr_ty);
......@@ -7640,8 +7680,8 @@ pub const FuncGen = struct {
76407680 const result_val = try self.wip.extractValue(results, &.{0}, "");
76417681 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
76427682
7643 const result_index = llvmField(inst_ty, 0, mod).?.index;
7644 const overflow_index = llvmField(inst_ty, 1, mod).?.index;
7683 const result_index = o.llvmFieldIndex(inst_ty, 0).?;
7684 const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
76457685
76467686 if (isByRef(inst_ty, mod)) {
76477687 const result_alignment = Builder.Alignment.fromByteUnits(inst_ty.abiAlignment(mod));
......@@ -7998,8 +8038,8 @@ pub const FuncGen = struct {
79988038
79998039 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
80008040
8001 const result_index = llvmField(dest_ty, 0, mod).?.index;
8002 const overflow_index = llvmField(dest_ty, 1, mod).?.index;
8041 const result_index = o.llvmFieldIndex(dest_ty, 0).?;
8042 const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
80038043
80048044 if (isByRef(dest_ty, mod)) {
80058045 const result_alignment = Builder.Alignment.fromByteUnits(dest_ty.abiAlignment(mod));
......@@ -9616,7 +9656,7 @@ pub const FuncGen = struct {
96169656 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;
96179657
96189658 const llvm_elem = try self.resolveInst(elem);
9619 const llvm_i = llvmField(result_ty, i, mod).?.index;
9659 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
96209660 const field_ptr =
96219661 try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
96229662 const field_ptr_ty = try mod.ptrType(.{
......@@ -9637,7 +9677,7 @@ pub const FuncGen = struct {
96379677 if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue;
96389678
96399679 const llvm_elem = try self.resolveInst(elem);
9640 const llvm_i = llvmField(result_ty, i, mod).?.index;
9680 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
96419681 result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
96429682 }
96439683 return result;
......@@ -10059,8 +10099,8 @@ pub const FuncGen = struct {
1005910099 else => {
1006010100 const struct_llvm_ty = try o.lowerPtrElemTy(struct_ty);
1006110101
10062 if (llvmField(struct_ty, field_index, mod)) |llvm_field| {
10063 return self.wip.gepStruct(struct_llvm_ty, struct_ptr, llvm_field.index, "");
10102 if (o.llvmFieldIndex(struct_ty, field_index)) |llvm_field_index| {
10103 return self.wip.gepStruct(struct_llvm_ty, struct_ptr, llvm_field_index, "");
1006410104 } else {
1006510105 // If we found no index then this means this is a zero sized field at the
1006610106 // end of the struct. Treat our struct pointer as an array of two and get
......@@ -10527,90 +10567,6 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ
1052710567 };
1052810568}
1052910569
10530const LlvmField = struct {
10531 index: c_uint,
10532 ty: Type,
10533 alignment: u32,
10534};
10535
10536/// Take into account 0 bit fields and padding. Returns null if an llvm
10537/// field could not be found.
10538/// This only happens if you want the field index of a zero sized field at
10539/// the end of the struct.
10540fn llvmField(ty: Type, field_index: usize, mod: *Module) ?LlvmField {
10541 // Detects where we inserted extra padding fields so that we can skip
10542 // over them in this function.
10543 comptime assert(struct_layout_version == 2);
10544 var offset: u64 = 0;
10545 var big_align: u32 = 0;
10546
10547 const ip = &mod.intern_pool;
10548 const struct_type = switch (ip.indexToKey(ty.toIntern())) {
10549 .anon_struct_type => |tuple| {
10550 var llvm_field_index: c_uint = 0;
10551 for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| {
10552 if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;
10553
10554 const field_align = field_ty.toType().abiAlignment(mod);
10555 big_align = @max(big_align, field_align);
10556 const prev_offset = offset;
10557 offset = std.mem.alignForward(u64, offset, field_align);
10558
10559 const padding_len = offset - prev_offset;
10560 if (padding_len > 0) {
10561 llvm_field_index += 1;
10562 }
10563
10564 if (field_index <= i) {
10565 return .{
10566 .index = llvm_field_index,
10567 .ty = field_ty.toType(),
10568 .alignment = field_align,
10569 };
10570 }
10571
10572 llvm_field_index += 1;
10573 offset += field_ty.toType().abiSize(mod);
10574 }
10575 return null;
10576 },
10577 .struct_type => |s| s,
10578 else => unreachable,
10579 };
10580 const struct_obj = mod.structPtrUnwrap(struct_type.index).?;
10581 const layout = struct_obj.layout;
10582 assert(layout != .Packed);
10583
10584 var llvm_field_index: c_uint = 0;
10585 var it = struct_obj.runtimeFieldIterator(mod);
10586 while (it.next()) |field_and_index| {
10587 const field = field_and_index.field;
10588 const field_align = field.alignment(mod, layout);
10589 big_align = @max(big_align, field_align);
10590 const prev_offset = offset;
10591 offset = std.mem.alignForward(u64, offset, field_align);
10592
10593 const padding_len = offset - prev_offset;
10594 if (padding_len > 0) {
10595 llvm_field_index += 1;
10596 }
10597
10598 if (field_index == field_and_index.index) {
10599 return .{
10600 .index = llvm_field_index,
10601 .ty = field.ty,
10602 .alignment = field_align,
10603 };
10604 }
10605
10606 llvm_field_index += 1;
10607 offset += field.ty.abiSize(mod);
10608 } else {
10609 // We did not find an llvm field that corresponds to this zig field.
10610 return null;
10611 }
10612}
10613
1061410570fn firstParamSRet(fn_info: InternPool.Key.FuncType, mod: *Module) bool {
1061510571 const return_type = fn_info.return_type.toType();
1061610572 if (!return_type.hasRuntimeBitsIgnoreComptime(mod)) return false;