authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-11 18:28:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:15:11-07:00
log5a34e6c3e608e1f526bababd3a2a146f6216d045
tree154864ee4b7841598494d5c4a5449b38de09ed3c
parenta3c20dffaed77727494d34f7b4b03c0d10771270

frontend: add file system inputs for incremental cache mode

These are also used for whole cache mode in the case that any compile errors are emitted.

4 files changed, 64 insertions(+), 8 deletions(-)

lib/std/Build/Cache.zig+1
......@@ -1010,6 +1010,7 @@ pub const Manifest = struct {
10101010
10111011 pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayListUnmanaged(u8)) Allocator.Error!void {
10121012 assert(@typeInfo(std.zig.Server.Message.PathPrefix).Enum.fields.len == man.cache.prefixes_len);
1013 buf.clearRetainingCapacity();
10131014 const gpa = man.cache.gpa;
10141015 const files = man.files.keys();
10151016 if (files.len > 0) {
src/Compilation.zig+61-6
......@@ -2051,6 +2051,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
20512051 );
20522052 };
20532053 if (is_hit) {
2054 // In this case the cache hit contains the full set of file system inputs. Nice!
20542055 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
20552056
20562057 comp.last_update_was_cache_hit = true;
......@@ -2112,12 +2113,24 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21122113 .incremental => {},
21132114 }
21142115
2116 // From this point we add a preliminary set of file system inputs that
2117 // affects both incremental and whole cache mode. For incremental cache
2118 // mode, the long-lived compiler state will track additional file system
2119 // inputs discovered after this point. For whole cache mode, we rely on
2120 // these inputs to make it past AstGen, and once there, we can rely on
2121 // learning file system inputs from the Cache object.
2122
21152123 // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
21162124 // Add a Job for each C object.
21172125 try comp.c_object_work_queue.ensureUnusedCapacity(comp.c_object_table.count());
21182126 for (comp.c_object_table.keys()) |key| {
21192127 comp.c_object_work_queue.writeItemAssumeCapacity(key);
21202128 }
2129 if (comp.file_system_inputs) |fsi| {
2130 for (comp.c_object_table.keys()) |c_object| {
2131 try comp.appendFileSystemInput(fsi, c_object.src.owner.root, c_object.src.src_path);
2132 }
2133 }
21212134
21222135 // For compiling Win32 resources, we rely on the cache hash system to avoid duplicating work.
21232136 // Add a Job for each Win32 resource file.
......@@ -2126,6 +2139,12 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21262139 for (comp.win32_resource_table.keys()) |key| {
21272140 comp.win32_resource_work_queue.writeItemAssumeCapacity(key);
21282141 }
2142 if (comp.file_system_inputs) |fsi| {
2143 for (comp.win32_resource_table.keys()) |win32_resource| switch (win32_resource.src) {
2144 .rc => |f| try comp.appendFileSystemInput(fsi, f.owner.root, f.src_path),
2145 .manifest => continue,
2146 };
2147 }
21292148 }
21302149
21312150 if (comp.module) |zcu| {
......@@ -2160,12 +2179,24 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21602179 if (zcu.fileByIndex(file_index).mod.isBuiltin()) continue;
21612180 comp.astgen_work_queue.writeItemAssumeCapacity(file_index);
21622181 }
2182 if (comp.file_system_inputs) |fsi| {
2183 for (zcu.import_table.values()) |file| {
2184 try comp.appendFileSystemInput(fsi, file.mod.root, file.sub_file_path);
2185 }
2186 }
21632187
21642188 // Put a work item in for checking if any files used with `@embedFile` changed.
21652189 try comp.embed_file_work_queue.ensureUnusedCapacity(zcu.embed_table.count());
21662190 for (zcu.embed_table.values()) |embed_file| {
21672191 comp.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
21682192 }
2193 if (comp.file_system_inputs) |fsi| {
2194 const ip = &zcu.intern_pool;
2195 for (zcu.embed_table.values()) |embed_file| {
2196 const sub_file_path = embed_file.sub_file_path.toSlice(ip);
2197 try comp.appendFileSystemInput(fsi, embed_file.owner.root, sub_file_path);
2198 }
2199 }
21692200
21702201 try comp.work_queue.writeItem(.{ .analyze_mod = std_mod });
21712202 if (comp.config.is_test) {
......@@ -2179,11 +2210,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21792210
21802211 try comp.performAllTheWork(main_progress_node);
21812212
2182 switch (comp.cache_use) {
2183 .whole => if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf),
2184 .incremental => {},
2185 }
2186
21872213 if (comp.module) |zcu| {
21882214 const pt: Zcu.PerThread = .{ .zcu = zcu, .tid = .main };
21892215
......@@ -2224,6 +2250,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
22242250
22252251 switch (comp.cache_use) {
22262252 .whole => |whole| {
2253 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2254
22272255 const digest = man.final();
22282256
22292257 // Rename the temporary directory into place.
......@@ -2311,6 +2339,30 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
23112339 }
23122340}
23132341
2342fn appendFileSystemInput(
2343 comp: *Compilation,
2344 file_system_inputs: *std.ArrayListUnmanaged(u8),
2345 root: Cache.Path,
2346 sub_file_path: []const u8,
2347) Allocator.Error!void {
2348 const gpa = comp.gpa;
2349 const prefixes = comp.cache_parent.prefixes();
2350 try file_system_inputs.ensureUnusedCapacity(gpa, root.sub_path.len + sub_file_path.len + 3);
2351 if (file_system_inputs.items.len > 0) file_system_inputs.appendAssumeCapacity(0);
2352 for (prefixes, 1..) |prefix_directory, i| {
2353 if (prefix_directory.eql(root.root_dir)) {
2354 file_system_inputs.appendAssumeCapacity(@intCast(i));
2355 if (root.sub_path.len > 0) {
2356 file_system_inputs.appendSliceAssumeCapacity(root.sub_path);
2357 file_system_inputs.appendAssumeCapacity(std.fs.path.sep);
2358 }
2359 file_system_inputs.appendSliceAssumeCapacity(sub_file_path);
2360 return;
2361 }
2362 }
2363 std.debug.panic("missing prefix directory: {}, {s}", .{ root, sub_file_path });
2364}
2365
23142366fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: std.Progress.Node) !void {
23152367 if (comp.bin_file) |lf| {
23162368 // This is needed before reading the error flags.
......@@ -4218,6 +4270,9 @@ fn workerAstGenFile(
42184270 .token = item.data.token,
42194271 } }) catch continue;
42204272 }
4273 if (res.is_new) if (comp.file_system_inputs) |fsi| {
4274 comp.appendFileSystemInput(fsi, res.file.mod.root, res.file.sub_file_path) catch continue;
4275 };
42214276 const imported_path_digest = pt.zcu.filePathDigest(res.file_index);
42224277 const imported_root_decl = pt.zcu.fileRootDecl(res.file_index);
42234278 break :blk .{ res, imported_path_digest, imported_root_decl };
......@@ -4588,7 +4643,7 @@ fn reportRetryableEmbedFileError(
45884643 const gpa = mod.gpa;
45894644 const src_loc = embed_file.src_loc;
45904645 const ip = &mod.intern_pool;
4591 const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
4646 const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{
45924647 embed_file.owner.root,
45934648 embed_file.sub_file_path.toSlice(ip),
45944649 @errorName(err),
src/Zcu.zig+1-1
......@@ -728,7 +728,7 @@ pub const File = struct {
728728 source_loaded: bool,
729729 tree_loaded: bool,
730730 zir_loaded: bool,
731 /// Relative to the owning package's root_src_dir.
731 /// Relative to the owning package's root source directory.
732732 /// Memory is stored in gpa, owned by File.
733733 sub_file_path: []const u8,
734734 /// Whether this is populated depends on `source_loaded`.
src/Zcu/PerThread.zig+1-1
......@@ -2666,7 +2666,7 @@ pub fn reportRetryableAstGenError(
26662666 },
26672667 };
26682668
2669 const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
2669 const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{
26702670 file.mod.root, file.sub_file_path, @errorName(err),
26712671 });
26722672 errdefer err_msg.destroy(gpa);