authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-04 15:47:47-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-04 17:56:01-07:00
log74346b0f79ca4bf67d61008030c7cc3565bff3f9
tree69bdcb7800608daed4a8908406623f3bab38f0e4
parent30ec43a6c78d9c8803becbea5a02edb8fae08af6

frontend: TrackedInst stores FileIndex instead of path digest

The purpose of using path digest was to reference a file in a serializable manner. Now that there is a stable index associated with files, it is a superior way to accomplish that goal, since removes one layer of indirection, and makes TrackedInst 8 bytes instead of 20. The saved Zig Compiler State file for "hello world" goes from 1.3M to 1.2M with this change.

5 files changed, 85 insertions(+), 70 deletions(-)

src/Compilation.zig+10-10
......@@ -2649,7 +2649,7 @@ fn reportMultiModuleErrors(zcu: *Zcu) !void {
26492649 .import => |import| try Module.ErrorMsg.init(
26502650 gpa,
26512651 .{
2652 .base_node_inst = try ip.trackZir(gpa, zcu.filePathDigest(import.file), .main_struct_inst),
2652 .base_node_inst = try ip.trackZir(gpa, import.file, .main_struct_inst),
26532653 .offset = .{ .token_abs = import.token },
26542654 },
26552655 "imported from module {s}",
......@@ -2658,7 +2658,7 @@ fn reportMultiModuleErrors(zcu: *Zcu) !void {
26582658 .root => |pkg| try Module.ErrorMsg.init(
26592659 gpa,
26602660 .{
2661 .base_node_inst = try ip.trackZir(gpa, zcu.filePathDigest(file_index), .main_struct_inst),
2661 .base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
26622662 .offset = .entire_file,
26632663 },
26642664 "root of module {s}",
......@@ -2672,7 +2672,7 @@ fn reportMultiModuleErrors(zcu: *Zcu) !void {
26722672 notes[num_notes] = try Module.ErrorMsg.init(
26732673 gpa,
26742674 .{
2675 .base_node_inst = try ip.trackZir(gpa, zcu.filePathDigest(file_index), .main_struct_inst),
2675 .base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
26762676 .offset = .entire_file,
26772677 },
26782678 "{} more references omitted",
......@@ -2684,7 +2684,7 @@ fn reportMultiModuleErrors(zcu: *Zcu) !void {
26842684 const err = try Module.ErrorMsg.create(
26852685 gpa,
26862686 .{
2687 .base_node_inst = try ip.trackZir(gpa, zcu.filePathDigest(file_index), .main_struct_inst),
2687 .base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
26882688 .offset = .entire_file,
26892689 },
26902690 "file exists in multiple modules",
......@@ -2786,7 +2786,7 @@ pub fn saveState(comp: *Compilation) !void {
27862786 .first_dependency_len = @intCast(ip.first_dependency.count()),
27872787 .dep_entries_len = @intCast(ip.dep_entries.items.len),
27882788 .free_dep_entries_len = @intCast(ip.free_dep_entries.items.len),
2789 .files_len = @intCast(zcu.files.entries.len),
2789 .files_len = @intCast(ip.files.entries.len),
27902790 },
27912791 };
27922792 addBuf(&bufs_list, &bufs_len, mem.asBytes(&header));
......@@ -2811,8 +2811,8 @@ pub fn saveState(comp: *Compilation) !void {
28112811 addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.dep_entries.items));
28122812 addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.free_dep_entries.items));
28132813
2814 addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(zcu.files.keys()));
2815 addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(zcu.files.values()));
2814 addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.files.keys()));
2815 addBuf(&bufs_list, &bufs_len, mem.sliceAsBytes(ip.files.values()));
28162816
28172817 // TODO: compilation errors
28182818 // TODO: namespaces
......@@ -4060,7 +4060,7 @@ fn workerAstGenFile(
40604060 defer child_prog_node.end();
40614061
40624062 const zcu = comp.module.?;
4063 zcu.astGenFile(file, path_digest, root_decl) catch |err| switch (err) {
4063 zcu.astGenFile(file, file_index, path_digest, root_decl) catch |err| switch (err) {
40644064 error.AnalysisFail => return,
40654065 else => {
40664066 file.status = .retryable_failure;
......@@ -4477,11 +4477,11 @@ fn reportRetryableAstGenError(
44774477
44784478 const src_loc: Module.LazySrcLoc = switch (src) {
44794479 .root => .{
4480 .base_node_inst = try zcu.intern_pool.trackZir(gpa, zcu.filePathDigest(file_index), .main_struct_inst),
4480 .base_node_inst = try zcu.intern_pool.trackZir(gpa, file_index, .main_struct_inst),
44814481 .offset = .entire_file,
44824482 },
44834483 .import => |info| .{
4484 .base_node_inst = try zcu.intern_pool.trackZir(gpa, zcu.filePathDigest(info.importing_file), .main_struct_inst),
4484 .base_node_inst = try zcu.intern_pool.trackZir(gpa, info.importing_file, .main_struct_inst),
44854485 .offset = .{ .token_abs = info.import_tok },
44864486 },
44874487 };
src/InternPool.zig+21-4
......@@ -92,12 +92,27 @@ dep_entries: std.ArrayListUnmanaged(DepEntry) = .{},
9292/// garbage collection pass.
9393free_dep_entries: std.ArrayListUnmanaged(DepEntry.Index) = .{},
9494
95/// Elements are ordered identically to the `import_table` field of `Zcu`.
96///
97/// Unlike `import_table`, this data is serialized as part of incremental
98/// compilation state.
99///
100/// Key is the hash of the path to this file, used to store
101/// `InternPool.TrackedInst`.
102///
103/// Value is the `Decl` of the struct that represents this `File`.
104files: std.AutoArrayHashMapUnmanaged(Cache.BinDigest, OptionalDeclIndex) = .{},
105
106pub const FileIndex = enum(u32) {
107 _,
108};
109
95110pub const TrackedInst = extern struct {
96 path_digest: Cache.BinDigest,
111 file: FileIndex,
97112 inst: Zir.Inst.Index,
98113 comptime {
99114 // The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`.
100 assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Zir.Inst.Index));
115 assert(@sizeOf(@This()) == @sizeOf(FileIndex) + @sizeOf(Zir.Inst.Index));
101116 }
102117 pub const Index = enum(u32) {
103118 _,
......@@ -126,11 +141,11 @@ pub const TrackedInst = extern struct {
126141pub fn trackZir(
127142 ip: *InternPool,
128143 gpa: Allocator,
129 path_digest: Cache.BinDigest,
144 file: FileIndex,
130145 inst: Zir.Inst.Index,
131146) Allocator.Error!TrackedInst.Index {
132147 const key: TrackedInst = .{
133 .path_digest = path_digest,
148 .file = file,
134149 .inst = inst,
135150 };
136151 const gop = try ip.tracked_insts.getOrPut(gpa, key);
......@@ -4597,6 +4612,8 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
45974612 ip.dep_entries.deinit(gpa);
45984613 ip.free_dep_entries.deinit(gpa);
45994614
4615 ip.files.deinit(gpa);
4616
46004617 ip.* = undefined;
46014618}
46024619
src/Sema.zig+8-9
......@@ -839,8 +839,7 @@ pub const Block = struct {
839839 const zcu = sema.mod;
840840 const ip = &zcu.intern_pool;
841841 const file_index = block.getFileScopeIndex(zcu);
842 const path_digest = zcu.filePathDigest(file_index);
843 return ip.trackZir(gpa, path_digest, inst);
842 return ip.trackZir(gpa, file_index, inst);
844843 }
845844};
846845
......@@ -993,7 +992,7 @@ fn analyzeBodyInner(
993992
994993 try sema.inst_map.ensureSpaceForInstructions(sema.gpa, body);
995994
996 const mod = sema.mod;
995 const zcu = sema.mod;
997996 const map = &sema.inst_map;
998997 const tags = sema.code.instructions.items(.tag);
999998 const datas = sema.code.instructions.items(.data);
......@@ -1013,9 +1012,9 @@ fn analyzeBodyInner(
10131012 // The hashmap lookup in here is a little expensive, and LLVM fails to optimize it away.
10141013 if (build_options.enable_logging) {
10151014 std.log.scoped(.sema_zir).debug("sema ZIR {s} %{d}", .{ sub_file_path: {
1016 const path_digest = block.src_base_inst.resolveFull(&mod.intern_pool).path_digest;
1017 const index = mod.files.getIndex(path_digest).?;
1018 break :sub_file_path mod.import_table.values()[index].sub_file_path;
1015 const file_index = block.src_base_inst.resolveFull(&zcu.intern_pool).file;
1016 const file = zcu.fileByIndex(file_index);
1017 break :sub_file_path file.sub_file_path;
10191018 }, inst });
10201019 }
10211020
......@@ -1776,9 +1775,9 @@ fn analyzeBodyInner(
17761775 const inline_body = sema.code.bodySlice(extra.end, extra.data.body_len);
17771776 const err_union = try sema.resolveInst(extra.data.operand);
17781777 const err_union_ty = sema.typeOf(err_union);
1779 if (err_union_ty.zigTypeTag(mod) != .ErrorUnion) {
1778 if (err_union_ty.zigTypeTag(zcu) != .ErrorUnion) {
17801779 return sema.fail(block, operand_src, "expected error union type, found '{}'", .{
1781 err_union_ty.fmt(mod),
1780 err_union_ty.fmt(zcu),
17821781 });
17831782 }
17841783 const is_non_err = try sema.analyzeIsNonErrComptimeOnly(block, operand_src, err_union);
......@@ -6003,7 +6002,7 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr
60036002
60046003 const path_digest = zcu.filePathDigest(result.file_index);
60056004 const root_decl = zcu.fileRootDecl(result.file_index);
6006 zcu.astGenFile(result.file, path_digest, root_decl) catch |err|
6005 zcu.astGenFile(result.file, result.file_index, path_digest, root_decl) catch |err|
60076006 return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)});
60086007
60096008 try zcu.ensureFileAnalyzed(result.file_index);
src/Type.zig+1-1
......@@ -3455,7 +3455,7 @@ pub fn typeDeclSrcLine(ty: Type, zcu: *const Zcu) ?u32 {
34553455 else => return null,
34563456 };
34573457 const info = tracked.resolveFull(&zcu.intern_pool);
3458 const file = zcu.import_table.values()[zcu.files.getIndex(info.path_digest).?];
3458 const file = zcu.fileByIndex(info.file);
34593459 assert(file.zir_loaded);
34603460 const zir = file.zir;
34613461 const inst = zir.instructions.get(@intFromEnum(info.inst));
src/Zcu.zig+45-46
......@@ -105,19 +105,6 @@ multi_exports: std.AutoArrayHashMapUnmanaged(AnalUnit, extern struct {
105105/// Indexes correspond 1:1 to `files`.
106106import_table: std.StringArrayHashMapUnmanaged(*File) = .{},
107107
108/// Elements are ordered identically to `import_table`.
109///
110/// Unlike `import_table`, this data is serialized as part of incremental
111/// compilation state.
112///
113/// Key is the hash of the path to this file, used to store
114/// `InternPool.TrackedInst`.
115///
116/// Value is the `Decl` of the struct that represents this `File`.
117///
118/// Protected by Compilation's mutex.
119files: std.AutoArrayHashMapUnmanaged(Cache.BinDigest, Decl.OptionalIndex) = .{},
120
121108/// The set of all the files which have been loaded with `@embedFile` in the Module.
122109/// We keep track of this in order to iterate over it and check which files have been
123110/// modified on the file system when an update is requested, as well as to cache
......@@ -572,19 +559,20 @@ pub const Decl = struct {
572559 }
573560
574561 pub fn navSrcLine(decl: Decl, zcu: *Zcu) u32 {
562 const ip = &zcu.intern_pool;
575563 const tracked = decl.zir_decl_index.unwrap() orelse inst: {
576564 // generic instantiation
577565 assert(decl.has_tv);
578566 assert(decl.owns_tv);
579 const generic_owner_func = switch (zcu.intern_pool.indexToKey(decl.val.toIntern())) {
567 const generic_owner_func = switch (ip.indexToKey(decl.val.toIntern())) {
580568 .func => |func| func.generic_owner,
581569 else => return 0, // TODO: this is probably a `variable` or something; figure this out when we finish sorting out `Decl`.
582570 };
583571 const generic_owner_decl = zcu.declPtr(zcu.funcInfo(generic_owner_func).owner_decl);
584572 break :inst generic_owner_decl.zir_decl_index.unwrap().?;
585573 };
586 const info = tracked.resolveFull(&zcu.intern_pool);
587 const file = zcu.import_table.values()[zcu.files.getIndex(info.path_digest).?];
574 const info = tracked.resolveFull(ip);
575 const file = zcu.fileByIndex(info.file);
588576 assert(file.zir_loaded);
589577 const zir = file.zir;
590578 const inst = zir.instructions.get(@intFromEnum(info.inst));
......@@ -969,9 +957,7 @@ pub const File = struct {
969957 }
970958 }
971959
972 pub const Index = enum(u32) {
973 _,
974 };
960 pub const Index = InternPool.FileIndex;
975961};
976962
977963pub const EmbedFile = struct {
......@@ -2351,14 +2337,12 @@ pub const LazySrcLoc = struct {
23512337 };
23522338
23532339 pub fn resolveBaseNode(base_node_inst: InternPool.TrackedInst.Index, zcu: *Zcu) struct { *File, Ast.Node.Index } {
2354 const want_path_digest, const zir_inst = inst: {
2355 const info = base_node_inst.resolveFull(&zcu.intern_pool);
2356 break :inst .{ info.path_digest, info.inst };
2357 };
2358 const file = file: {
2359 const index = zcu.files.getIndex(want_path_digest).?;
2360 break :file zcu.import_table.values()[index];
2340 const ip = &zcu.intern_pool;
2341 const file_index, const zir_inst = inst: {
2342 const info = base_node_inst.resolveFull(ip);
2343 break :inst .{ info.file, info.inst };
23612344 };
2345 const file = zcu.fileByIndex(file_index);
23622346 assert(file.zir_loaded);
23632347
23642348 const zir = file.zir;
......@@ -2429,7 +2413,6 @@ pub fn deinit(zcu: *Zcu) void {
24292413 zcu.destroyFile(file_index);
24302414 }
24312415 zcu.import_table.deinit(gpa);
2432 zcu.files.deinit(gpa);
24332416
24342417 for (zcu.embed_table.keys(), zcu.embed_table.values()) |path, embed_file| {
24352418 gpa.free(path);
......@@ -2596,7 +2579,16 @@ comptime {
25962579 }
25972580}
25982581
2599pub fn astGenFile(zcu: *Zcu, file: *File, path_digest: Cache.BinDigest, opt_root_decl: Zcu.Decl.OptionalIndex) !void {
2582pub fn astGenFile(
2583 zcu: *Zcu,
2584 file: *File,
2585 /// This parameter is provided separately from `file` because it is not
2586 /// safe to access `import_table` without a lock, and this index is needed
2587 /// in the call to `updateZirRefs`.
2588 file_index: File.Index,
2589 path_digest: Cache.BinDigest,
2590 opt_root_decl: Zcu.Decl.OptionalIndex,
2591) !void {
26002592 assert(!file.mod.isBuiltin());
26012593
26022594 const tracy = trace(@src());
......@@ -2850,7 +2842,7 @@ pub fn astGenFile(zcu: *Zcu, file: *File, path_digest: Cache.BinDigest, opt_root
28502842 }
28512843
28522844 if (file.prev_zir) |prev_zir| {
2853 try updateZirRefs(zcu, file, prev_zir.*, path_digest);
2845 try updateZirRefs(zcu, file, file_index, prev_zir.*);
28542846 // No need to keep previous ZIR.
28552847 prev_zir.deinit(gpa);
28562848 gpa.destroy(prev_zir);
......@@ -2939,7 +2931,7 @@ fn loadZirCacheBody(gpa: Allocator, header: Zir.Header, cache_file: std.fs.File)
29392931
29402932/// This is called from the AstGen thread pool, so must acquire
29412933/// the Compilation mutex when acting on shared state.
2942fn updateZirRefs(zcu: *Module, file: *File, old_zir: Zir, path_digest: Cache.BinDigest) !void {
2934fn updateZirRefs(zcu: *Module, file: *File, file_index: File.Index, old_zir: Zir) !void {
29432935 const gpa = zcu.gpa;
29442936 const new_zir = file.zir;
29452937
......@@ -2955,7 +2947,7 @@ fn updateZirRefs(zcu: *Module, file: *File, old_zir: Zir, path_digest: Cache.Bin
29552947 // iterating over this full set for every updated file.
29562948 for (zcu.intern_pool.tracked_insts.keys(), 0..) |*ti, idx_raw| {
29572949 const ti_idx: InternPool.TrackedInst.Index = @enumFromInt(idx_raw);
2958 if (!std.mem.eql(u8, &ti.path_digest, &path_digest)) continue;
2950 if (ti.file != file_index) continue;
29592951 const old_inst = ti.inst;
29602952 ti.inst = inst_map.get(ti.inst) orelse {
29612953 // Tracking failed for this instruction. Invalidate associated `src_hash` deps.
......@@ -3849,7 +3841,7 @@ fn getFileRootStruct(
38493841 const decls = file.zir.bodySlice(extra_index, decls_len);
38503842 extra_index += decls_len;
38513843
3852 const tracked_inst = try ip.trackZir(gpa, zcu.filePathDigest(file_index), .main_struct_inst);
3844 const tracked_inst = try ip.trackZir(gpa, file_index, .main_struct_inst);
38533845 const wip_ty = switch (try ip.getStructType(gpa, .{
38543846 .layout = .auto,
38553847 .fields_len = fields_len,
......@@ -4151,7 +4143,7 @@ fn semaDecl(zcu: *Zcu, decl_index: Decl.Index) !SemaDeclResult {
41514143 // Every Decl (other than file root Decls, which do not have a ZIR index) has a dependency on its own source.
41524144 try sema.declareDependency(.{ .src_hash = try ip.trackZir(
41534145 gpa,
4154 zcu.filePathDigest(decl.getFileScopeIndex(zcu)),
4146 decl.getFileScopeIndex(zcu),
41554147 decl_inst,
41564148 ) });
41574149
......@@ -4391,17 +4383,19 @@ pub fn importPkg(zcu: *Zcu, mod: *Package.Module) !ImportFileResult {
43914383 };
43924384 }
43934385
4394 try zcu.files.ensureUnusedCapacity(gpa, 1);
4386 const ip = &zcu.intern_pool;
4387
4388 try ip.files.ensureUnusedCapacity(gpa, 1);
43954389
43964390 if (mod.builtin_file) |builtin_file| {
43974391 keep_resolved_path = true; // It's now owned by import_table.
43984392 gop.value_ptr.* = builtin_file;
43994393 try builtin_file.addReference(zcu.*, .{ .root = mod });
44004394 const path_digest = computePathDigest(zcu, mod, builtin_file.sub_file_path);
4401 zcu.files.putAssumeCapacityNoClobber(path_digest, .none);
4395 ip.files.putAssumeCapacityNoClobber(path_digest, .none);
44024396 return .{
44034397 .file = builtin_file,
4404 .file_index = @enumFromInt(zcu.files.entries.len - 1),
4398 .file_index = @enumFromInt(ip.files.entries.len - 1),
44054399 .is_new = false,
44064400 .is_pkg = true,
44074401 };
......@@ -4431,10 +4425,10 @@ pub fn importPkg(zcu: *Zcu, mod: *Package.Module) !ImportFileResult {
44314425 const path_digest = computePathDigest(zcu, mod, sub_file_path);
44324426
44334427 try new_file.addReference(zcu.*, .{ .root = mod });
4434 zcu.files.putAssumeCapacityNoClobber(path_digest, .none);
4428 ip.files.putAssumeCapacityNoClobber(path_digest, .none);
44354429 return .{
44364430 .file = new_file,
4437 .file_index = @enumFromInt(zcu.files.entries.len - 1),
4431 .file_index = @enumFromInt(ip.files.entries.len - 1),
44384432 .is_new = true,
44394433 .is_pkg = true,
44404434 };
......@@ -4486,7 +4480,9 @@ pub fn importFile(
44864480 .is_pkg = false,
44874481 };
44884482
4489 try zcu.files.ensureUnusedCapacity(gpa, 1);
4483 const ip = &zcu.intern_pool;
4484
4485 try ip.files.ensureUnusedCapacity(gpa, 1);
44904486
44914487 const new_file = try gpa.create(File);
44924488 errdefer gpa.destroy(new_file);
......@@ -4528,10 +4524,10 @@ pub fn importFile(
45284524 };
45294525
45304526 const path_digest = computePathDigest(zcu, mod, sub_file_path);
4531 zcu.files.putAssumeCapacityNoClobber(path_digest, .none);
4527 ip.files.putAssumeCapacityNoClobber(path_digest, .none);
45324528 return .{
45334529 .file = new_file,
4534 .file_index = @enumFromInt(zcu.files.entries.len - 1),
4530 .file_index = @enumFromInt(ip.files.entries.len - 1),
45354531 .is_new = true,
45364532 .is_pkg = false,
45374533 };
......@@ -4892,7 +4888,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_inst: Zir.Inst.Index) Allocator.Error!void
48924888 }
48934889
48944890 const parent_file_scope_index = iter.parent_decl.getFileScopeIndex(zcu);
4895 const tracked_inst = try ip.trackZir(gpa, zcu.filePathDigest(parent_file_scope_index), decl_inst);
4891 const tracked_inst = try ip.trackZir(gpa, parent_file_scope_index, decl_inst);
48964892
48974893 // We create a Decl for it regardless of analysis status.
48984894
......@@ -5743,7 +5739,7 @@ fn reportRetryableFileError(
57435739 const err_msg = try ErrorMsg.create(
57445740 gpa,
57455741 .{
5746 .base_node_inst = try ip.trackZir(gpa, zcu.filePathDigest(file_index), .main_struct_inst),
5742 .base_node_inst = try ip.trackZir(gpa, file_index, .main_struct_inst),
57475743 .offset = .entire_file,
57485744 },
57495745 format,
......@@ -6601,13 +6597,16 @@ pub fn fileByIndex(zcu: *const Zcu, i: File.Index) *File {
66016597
66026598/// Returns the `Decl` of the struct that represents this `File`.
66036599pub fn fileRootDecl(zcu: *const Zcu, i: File.Index) Decl.OptionalIndex {
6604 return zcu.files.values()[@intFromEnum(i)];
6600 const ip = &zcu.intern_pool;
6601 return ip.files.values()[@intFromEnum(i)];
66056602}
66066603
66076604pub fn setFileRootDecl(zcu: *Zcu, i: File.Index, root_decl: Decl.OptionalIndex) void {
6608 zcu.files.values()[@intFromEnum(i)] = root_decl;
6605 const ip = &zcu.intern_pool;
6606 ip.files.values()[@intFromEnum(i)] = root_decl;
66096607}
66106608
66116609pub fn filePathDigest(zcu: *const Zcu, i: File.Index) Cache.BinDigest {
6612 return zcu.files.keys()[@intFromEnum(i)];
6610 const ip = &zcu.intern_pool;
6611 return ip.files.keys()[@intFromEnum(i)];
66136612}