authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-23 23:06:08-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-23 23:06:08-07:00
loga7088fd9a3edb037f0f51bb402a3c557334634f3
treedba5fd9f32b341d1cb64b9813033e1558c1e8d2b
parent8eff0a0a669dbdacf9cebbc96fdf20536f3073ee

compiler: packed structs cache bit offsets

Instead of linear search every time a packed struct field's bit or byte offset is wanted, they are computed once during resolution of the packed struct's backing int type, and stored in InternPool for O(1) lookup. Closes #17178

9 files changed, 75 insertions(+), 62 deletions(-)

src/InternPool.zig+46-6
......@@ -105,6 +105,25 @@ pub const MapIndex = enum(u32) {
105105 }
106106};
107107
108pub const OptionalInt = enum(u32) {
109 none = std.math.maxInt(u32),
110 _,
111
112 pub fn init(x: u32) @This() {
113 const result: @This() = @enumFromInt(x);
114 assert(result != .none);
115 return result;
116 }
117
118 pub fn initOptional(opt_x: ?u32) @This() {
119 return @This().init(opt_x orelse return .none);
120 }
121
122 pub fn unwrap(this: @This()) ?u32 {
123 return if (this == .none) null else @intFromEnum(this);
124 }
125};
126
108127pub const RuntimeIndex = enum(u32) {
109128 zero = 0,
110129 comptime_field_ptr = std.math.maxInt(u32),
......@@ -377,6 +396,8 @@ pub const Key = union(enum) {
377396 field_aligns: Alignment.Slice,
378397 runtime_order: RuntimeOrder.Slice,
379398 comptime_bits: ComptimeBits,
399 /// In the case of packed structs these are bit offsets; in the case of
400 /// non-packed structs these are byte offsets.
380401 offsets: Offsets,
381402 names_map: OptionalMapIndex,
382403
......@@ -475,6 +496,15 @@ pub const Key = union(enum) {
475496 return s.field_names.get(ip)[i].toOptional();
476497 }
477498
499 /// Asserts it is a packed struct.
500 /// Asserts the layout is resolved.
501 pub fn fieldBitOffset(s: @This(), ip: *InternPool, i: usize) u32 {
502 assert(s.layout == .Packed);
503 assert(s.haveLayout(ip));
504 const result: OptionalInt = @enumFromInt(s.offsets.get(ip)[i]);
505 return result.unwrap().?;
506 }
507
478508 pub fn fieldIsComptime(s: @This(), ip: *const InternPool, i: usize) bool {
479509 return s.comptime_bits.getBit(ip, i);
480510 }
......@@ -593,7 +623,11 @@ pub const Key = union(enum) {
593623
594624 pub fn haveLayout(s: @This(), ip: *InternPool) bool {
595625 return switch (s.layout) {
596 .Packed => s.backingIntType(ip).* != .none,
626 .Packed => {
627 if (s.offsets.len == 0) return true;
628 const first_offset: OptionalInt = @enumFromInt(ip.extra.items[s.offsets.start]);
629 return first_offset != .none;
630 },
597631 .Auto, .Extern => s.flagsPtr(ip).layout_resolved,
598632 };
599633 }
......@@ -2936,7 +2970,8 @@ pub const Tag = enum(u8) {
29362970 /// Trailing:
29372971 /// 0. type: Index for each fields_len
29382972 /// 1. name: NullTerminatedString for each fields_len
2939 /// 2. init: Index for each fields_len // if tag is type_struct_packed_inits
2973 /// 2. bit_offset: OptionalInt for each fields_len // none until layout resolved
2974 /// 3. init: Index for each fields_len // if tag is type_struct_packed_inits
29402975 pub const TypeStructPacked = struct {
29412976 decl: Module.Decl.Index,
29422977 zir_index: Zir.Inst.Index,
......@@ -4194,8 +4229,12 @@ fn extraPackedStructType(ip: *const InternPool, extra_index: u32, inits: bool) K
41944229 .start = type_struct_packed.end + fields_len,
41954230 .len = fields_len,
41964231 },
4232 .offsets = .{
4233 .start = type_struct_packed.end + fields_len * 2,
4234 .len = fields_len,
4235 },
41974236 .field_inits = if (inits) .{
4198 .start = type_struct_packed.end + fields_len + fields_len,
4237 .start = type_struct_packed.end + fields_len * 3,
41994238 .len = fields_len,
42004239 } else .{
42014240 .start = 0,
......@@ -4204,7 +4243,6 @@ fn extraPackedStructType(ip: *const InternPool, extra_index: u32, inits: bool) K
42044243 .field_aligns = .{ .start = 0, .len = 0 },
42054244 .runtime_order = .{ .start = 0, .len = 0 },
42064245 .comptime_bits = .{ .start = 0, .len = 0 },
4207 .offsets = .{ .start = 0, .len = 0 },
42084246 .names_map = type_struct_packed.data.names_map.toOptional(),
42094247 };
42104248}
......@@ -5279,6 +5317,7 @@ pub fn getStructType(
52795317 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.TypeStructPacked).Struct.fields.len +
52805318 ini.fields_len + // types
52815319 ini.fields_len + // names
5320 ini.fields_len + // offsets
52825321 ini.fields_len); // inits
52835322 try ip.items.append(gpa, .{
52845323 .tag = if (ini.any_default_inits) .type_struct_packed_inits else .type_struct_packed,
......@@ -5293,6 +5332,7 @@ pub fn getStructType(
52935332 });
52945333 ip.extra.appendNTimesAssumeCapacity(@intFromEnum(Index.none), ini.fields_len);
52955334 ip.extra.appendNTimesAssumeCapacity(@intFromEnum(OptionalNullTerminatedString.none), ini.fields_len);
5335 ip.extra.appendNTimesAssumeCapacity(@intFromEnum(OptionalInt.none), ini.fields_len);
52965336 if (ini.any_default_inits) {
52975337 ip.extra.appendNTimesAssumeCapacity(@intFromEnum(Index.none), ini.fields_len);
52985338 }
......@@ -7113,12 +7153,12 @@ fn dumpStatsFallible(ip: *const InternPool, arena: Allocator) anyerror!void {
71137153 .type_struct_packed => b: {
71147154 const info = ip.extraData(Tag.TypeStructPacked, data);
71157155 break :b @sizeOf(u32) * (@typeInfo(Tag.TypeStructPacked).Struct.fields.len +
7116 info.fields_len + info.fields_len);
7156 info.fields_len + info.fields_len + info.fields_len);
71177157 },
71187158 .type_struct_packed_inits => b: {
71197159 const info = ip.extraData(Tag.TypeStructPacked, data);
71207160 break :b @sizeOf(u32) * (@typeInfo(Tag.TypeStructPacked).Struct.fields.len +
7121 info.fields_len + info.fields_len + info.fields_len);
7161 info.fields_len + info.fields_len + info.fields_len + info.fields_len);
71227162 },
71237163 .type_tuple_anon => b: {
71247164 const info = ip.extraData(TypeStructAnon, data);
src/Module.zig-23
......@@ -6649,26 +6649,3 @@ pub fn structFieldAlignmentExtern(mod: *Module, field_ty: Type) Alignment {
66496649
66506650 return ty_abi_align;
66516651}
6652
6653/// TODO: avoid linear search by storing these in trailing data of packed struct types
6654/// then packedStructFieldByteOffset can be expressed in terms of bits / 8, fixing
6655/// that one too.
6656/// https://github.com/ziglang/zig/issues/17178
6657pub fn structPackedFieldBitOffset(
6658 mod: *Module,
6659 struct_type: InternPool.Key.StructType,
6660 field_index: u32,
6661) u16 {
6662 const ip = &mod.intern_pool;
6663 assert(struct_type.layout == .Packed);
6664 assert(struct_type.haveLayout(ip));
6665 var bit_sum: u64 = 0;
6666 for (0..struct_type.field_types.len) |i| {
6667 if (i == field_index) {
6668 return @intCast(bit_sum);
6669 }
6670 const field_ty = struct_type.field_types.get(ip)[i].toType();
6671 bit_sum += field_ty.bitSize(mod);
6672 }
6673 unreachable; // index out of bounds
6674}
src/Sema.zig+5-2
......@@ -21342,8 +21342,10 @@ fn reifyStruct(
2134221342 }
2134321343
2134421344 var fields_bit_sum: u64 = 0;
21345 for (struct_type.field_types.get(ip)) |field_ty| {
21346 fields_bit_sum += field_ty.toType().bitSize(mod);
21345 for (0..struct_type.field_types.len) |i| {
21346 struct_type.offsets.get(ip)[i] = @intCast(fields_bit_sum);
21347 const field_ty = struct_type.field_types.get(ip)[i].toType();
21348 fields_bit_sum += field_ty.bitSize(mod);
2134721349 }
2134821350
2134921351 if (backing_int_val.optionalValue(mod)) |backing_int_ty_val| {
......@@ -34772,6 +34774,7 @@ fn semaBackingIntType(mod: *Module, struct_type: InternPool.Key.StructType) Comp
3477234774 var accumulator: u64 = 0;
3477334775 for (0..struct_type.field_types.len) |i| {
3477434776 const field_ty = struct_type.field_types.get(ip)[i].toType();
34777 struct_type.offsets.get(ip)[i] = @intCast(accumulator);
3477534778 accumulator += try field_ty.bitSizeAdvanced(mod, &sema);
3477634779 }
3477734780 break :blk accumulator;
src/arch/wasm/CodeGen.zig+1-1
......@@ -3779,7 +3779,7 @@ fn airStructFieldVal(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
37793779 .Packed => switch (struct_ty.zigTypeTag(mod)) {
37803780 .Struct => result: {
37813781 const packed_struct = mod.typeToPackedStruct(struct_ty).?;
3782 const offset = mod.structPackedFieldBitOffset(packed_struct, field_index);
3782 const offset = packed_struct.fieldBitOffset(ip, field_index);
37833783 const backing_ty = packed_struct.backingIntType(ip).toType();
37843784 const wasm_bits = toWasmBits(backing_ty.intInfo(mod).bits) orelse {
37853785 return func.fail("TODO: airStructFieldVal for packed structs larger than 128 bits", .{});
src/arch/x86_64/CodeGen.zig+4-2
......@@ -5593,6 +5593,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
55935593
55945594fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
55955595 const mod = self.bin_file.options.module.?;
5596 const ip = &mod.intern_pool;
55965597 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
55975598 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
55985599 const result: MCValue = result: {
......@@ -5610,7 +5611,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
56105611 const field_off: u32 = switch (container_ty.containerLayout(mod)) {
56115612 .Auto, .Extern => @intCast(container_ty.structFieldOffset(index, mod) * 8),
56125613 .Packed => if (mod.typeToStruct(container_ty)) |struct_type|
5613 mod.structPackedFieldBitOffset(struct_type, index)
5614 struct_type.fieldBitOffset(ip, index)
56145615 else
56155616 0,
56165617 };
......@@ -11410,6 +11411,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
1141011411
1141111412fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1141211413 const mod = self.bin_file.options.module.?;
11414 const ip = &mod.intern_pool;
1141311415 const result_ty = self.typeOfIndex(inst);
1141411416 const len: usize = @intCast(result_ty.arrayLen(mod));
1141511417 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
......@@ -11440,7 +11442,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1144011442 }
1144111443 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
1144211444 const elem_abi_bits = elem_abi_size * 8;
11443 const elem_off = mod.structPackedFieldBitOffset(struct_type, elem_i);
11445 const elem_off = struct_type.fieldBitOffset(ip, elem_i);
1144411446 const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
1144511447 const elem_bit_off = elem_off % elem_abi_bits;
1144611448 const elem_mcv = try self.resolveInst(elem);
src/codegen.zig+9-8
......@@ -630,7 +630,8 @@ fn lowerParentPtr(
630630 reloc_info: RelocInfo,
631631) CodeGenError!Result {
632632 const mod = bin_file.options.module.?;
633 const ptr = mod.intern_pool.indexToKey(parent_ptr).ptr;
633 const ip = &mod.intern_pool;
634 const ptr = ip.indexToKey(parent_ptr).ptr;
634635 assert(ptr.len == .none);
635636 return switch (ptr.addr) {
636637 .decl, .mut_decl => try lowerDeclRef(
......@@ -656,7 +657,7 @@ fn lowerParentPtr(
656657 code,
657658 debug_output,
658659 reloc_info.offset(@as(u32, @intCast(errUnionPayloadOffset(
659 mod.intern_pool.typeOf(eu_payload).toType(),
660 ip.typeOf(eu_payload).toType(),
660661 mod,
661662 )))),
662663 ),
......@@ -675,17 +676,17 @@ fn lowerParentPtr(
675676 code,
676677 debug_output,
677678 reloc_info.offset(@as(u32, @intCast(elem.index *
678 mod.intern_pool.typeOf(elem.base).toType().elemType2(mod).abiSize(mod)))),
679 ip.typeOf(elem.base).toType().elemType2(mod).abiSize(mod)))),
679680 ),
680681 .field => |field| {
681 const base_type = mod.intern_pool.indexToKey(mod.intern_pool.typeOf(field.base)).ptr_type.child;
682 const base_type = ip.indexToKey(ip.typeOf(field.base)).ptr_type.child;
682683 return lowerParentPtr(
683684 bin_file,
684685 src_loc,
685686 field.base,
686687 code,
687688 debug_output,
688 reloc_info.offset(switch (mod.intern_pool.indexToKey(base_type)) {
689 reloc_info.offset(switch (ip.indexToKey(base_type)) {
689690 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
690691 .One, .Many, .C => unreachable,
691692 .Slice => switch (field.index) {
......@@ -703,9 +704,9 @@ fn lowerParentPtr(
703704 mod,
704705 )),
705706 .Packed => if (mod.typeToStruct(base_type.toType())) |struct_type|
706 math.divExact(u16, mod.structPackedFieldBitOffset(
707 struct_type,
708 @intCast(field.index),
707 math.divExact(u32, struct_type.fieldBitOffset(
708 ip,
709 field.index,
709710 ), 8) catch |err| switch (err) {
710711 error.UnexpectedRemainder => 0,
711712 error.DivisionByZero => unreachable,
src/codegen/c.zig+1-1
......@@ -5429,7 +5429,7 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue {
54295429
54305430 const bit_offset_ty = try mod.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1));
54315431
5432 const bit_offset = mod.structPackedFieldBitOffset(struct_type, extra.field_index);
5432 const bit_offset = struct_type.fieldBitOffset(ip, extra.field_index);
54335433 const bit_offset_val = try mod.intValue(bit_offset_ty, bit_offset);
54345434
54355435 const field_int_signedness = if (inst_ty.isAbiInt(mod))
src/codegen/llvm.zig+2-1
......@@ -6192,6 +6192,7 @@ pub const FuncGen = struct {
61926192 fn airStructFieldVal(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
61936193 const o = self.dg.object;
61946194 const mod = o.module;
6195 const ip = &mod.intern_pool;
61956196 const inst = body_tail[0];
61966197 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
61976198 const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
......@@ -6207,7 +6208,7 @@ pub const FuncGen = struct {
62076208 .Struct => switch (struct_ty.containerLayout(mod)) {
62086209 .Packed => {
62096210 const struct_type = mod.typeToStruct(struct_ty).?;
6210 const bit_offset = mod.structPackedFieldBitOffset(struct_type, field_index);
6211 const bit_offset = struct_type.fieldBitOffset(ip, field_index);
62116212 const containing_int = struct_llvm_val;
62126213 const shift_amt =
62136214 try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);
src/type.zig+7-18
......@@ -3021,27 +3021,16 @@ pub const Type = struct {
30213021 };
30223022 }
30233023
3024 pub fn packedStructFieldByteOffset(ty: Type, field_index: usize, mod: *Module) u32 {
3024 pub fn packedStructFieldBitOffset(ty: Type, field_index: usize, mod: *Module) u32 {
30253025 const ip = &mod.intern_pool;
30263026 const struct_type = ip.indexToKey(ty.toIntern()).struct_type;
30273027 assert(struct_type.layout == .Packed);
3028 comptime assert(Type.packed_struct_layout_version == 2);
3029
3030 var bit_offset: u16 = undefined;
3031 var elem_size_bits: u16 = undefined;
3032 var running_bits: u16 = 0;
3033 for (struct_type.field_types.get(ip), 0..) |field_ty, i| {
3034 if (!field_ty.toType().hasRuntimeBits(mod)) continue;
3035
3036 const field_bits: u16 = @intCast(field_ty.toType().bitSize(mod));
3037 if (i == field_index) {
3038 bit_offset = running_bits;
3039 elem_size_bits = field_bits;
3040 }
3041 running_bits += field_bits;
3042 }
3043 const byte_offset = bit_offset / 8;
3044 return byte_offset;
3028 assert(struct_type.haveLayout(ip));
3029 return struct_type.offsets.get(ip)[field_index];
3030 }
3031
3032 pub fn packedStructFieldByteOffset(ty: Type, field_index: usize, mod: *Module) u32 {
3033 return packedStructFieldBitOffset(ty, field_index, mod) / 8;
30453034 }
30463035
30473036 pub const FieldOffset = struct {