authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-10-17 18:57:54-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-10-17 18:59:11-07:00
loge5dac0a0b391f227605e496a09f32b453ac3280d
treedf9fc28a45b801727a28ef56a8abc08deff43bc9
parentad17108bddc3bc198190407ab5b00820b2c17cd5

stage2: implement `@embedFile`

3 files changed, 275 insertions(+), 8 deletions(-)

src/Compilation.zig+100
......@@ -55,6 +55,10 @@ c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
5555/// since the last compilation, as well as scan for `@import` and queue up
5656/// additional jobs corresponding to those new files.
5757astgen_work_queue: std.fifo.LinearFifo(*Module.File, .Dynamic),
58/// These jobs are to inspect the file system stat() and if the embedded file has changed
59/// on disk, mark the corresponding Decl outdated and queue up an `analyze_decl`
60/// task for it.
61embed_file_work_queue: std.fifo.LinearFifo(*Module.EmbedFile, .Dynamic),
5862
5963/// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
6064/// This data is accessed by multiple threads and is protected by `mutex`.
......@@ -181,6 +185,10 @@ const Job = union(enum) {
181185 /// It may have already be analyzed, or it may have been determined
182186 /// to be outdated; in this case perform semantic analysis again.
183187 analyze_decl: *Module.Decl,
188 /// The file that was loaded with `@embedFile` has changed on disk
189 /// and has been re-loaded into memory. All Decls that depend on it
190 /// need to be re-analyzed.
191 update_embed_file: *Module.EmbedFile,
184192 /// The source file containing the Decl has been updated, and so the
185193 /// Decl may need its line number information updated in the debug info.
186194 update_line_number: *Module.Decl,
......@@ -1447,6 +1455,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
14471455 .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
14481456 .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
14491457 .astgen_work_queue = std.fifo.LinearFifo(*Module.File, .Dynamic).init(gpa),
1458 .embed_file_work_queue = std.fifo.LinearFifo(*Module.EmbedFile, .Dynamic).init(gpa),
14501459 .keep_source_files_loaded = options.keep_source_files_loaded,
14511460 .use_clang = use_clang,
14521461 .clang_argv = options.clang_argv,
......@@ -1632,6 +1641,7 @@ pub fn destroy(self: *Compilation) void {
16321641 self.work_queue.deinit();
16331642 self.c_object_work_queue.deinit();
16341643 self.astgen_work_queue.deinit();
1644 self.embed_file_work_queue.deinit();
16351645
16361646 {
16371647 var it = self.crt_files.iterator();
......@@ -1747,6 +1757,16 @@ pub fn update(self: *Compilation) !void {
17471757 }
17481758
17491759 if (!use_stage1) {
1760 // Put a work item in for checking if any files used with `@embedFile` changed.
1761 {
1762 try self.embed_file_work_queue.ensureUnusedCapacity(module.embed_table.count());
1763 var it = module.embed_table.iterator();
1764 while (it.next()) |entry| {
1765 const embed_file = entry.value_ptr.*;
1766 self.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
1767 }
1768 }
1769
17501770 try self.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
17511771 if (self.bin_file.options.is_test) {
17521772 try self.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
......@@ -1870,6 +1890,7 @@ pub fn totalErrorCount(self: *Compilation) usize {
18701890
18711891 if (self.bin_file.options.module) |module| {
18721892 total += module.failed_exports.count();
1893 total += module.failed_embed_files.count();
18731894
18741895 {
18751896 var it = module.failed_files.iterator();
......@@ -1966,6 +1987,13 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
19661987 }
19671988 }
19681989 }
1990 {
1991 var it = module.failed_embed_files.iterator();
1992 while (it.next()) |entry| {
1993 const msg = entry.value_ptr.*;
1994 try AllErrors.add(module, &arena, &errors, msg.*);
1995 }
1996 }
19691997 {
19701998 var it = module.failed_decls.iterator();
19711999 while (it.next()) |entry| {
......@@ -2065,6 +2093,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
20652093 var c_obj_prog_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
20662094 defer c_obj_prog_node.end();
20672095
2096 var embed_file_prog_node = main_progress_node.start("Detect @embedFile updates", self.embed_file_work_queue.count);
2097 defer embed_file_prog_node.end();
2098
20682099 self.work_queue_wait_group.reset();
20692100 defer self.work_queue_wait_group.wait();
20702101
......@@ -2079,6 +2110,13 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
20792110 });
20802111 }
20812112
2113 while (self.embed_file_work_queue.readItem()) |embed_file| {
2114 self.astgen_wait_group.start();
2115 try self.thread_pool.spawn(workerCheckEmbedFile, .{
2116 self, embed_file, &embed_file_prog_node, &self.astgen_wait_group,
2117 });
2118 }
2119
20822120 while (self.c_object_work_queue.readItem()) |c_object| {
20832121 self.work_queue_wait_group.start();
20842122 try self.thread_pool.spawn(workerUpdateCObject, .{
......@@ -2260,6 +2298,15 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
22602298 error.AnalysisFail => continue,
22612299 };
22622300 },
2301 .update_embed_file => |embed_file| {
2302 if (build_options.omit_stage2)
2303 @panic("sadly stage2 is omitted from this build to save memory on the CI server");
2304 const module = self.bin_file.options.module.?;
2305 module.updateEmbedFile(embed_file) catch |err| switch (err) {
2306 error.OutOfMemory => return error.OutOfMemory,
2307 error.AnalysisFail => continue,
2308 };
2309 },
22632310 .update_line_number => |decl| {
22642311 if (build_options.omit_stage2)
22652312 @panic("sadly stage2 is omitted from this build to save memory on the CI server");
......@@ -2542,6 +2589,29 @@ fn workerAstGenFile(
25422589 }
25432590}
25442591
2592fn workerCheckEmbedFile(
2593 comp: *Compilation,
2594 embed_file: *Module.EmbedFile,
2595 prog_node: *std.Progress.Node,
2596 wg: *WaitGroup,
2597) void {
2598 defer wg.finish();
2599
2600 var child_prog_node = prog_node.start(embed_file.sub_file_path, 0);
2601 child_prog_node.activate();
2602 defer child_prog_node.end();
2603
2604 const mod = comp.bin_file.options.module.?;
2605 mod.detectEmbedFileUpdate(embed_file) catch |err| {
2606 comp.reportRetryableEmbedFileError(embed_file, err) catch |oom| switch (oom) {
2607 // Swallowing this error is OK because it's implied to be OOM when
2608 // there is a missing `failed_embed_files` error message.
2609 error.OutOfMemory => {},
2610 };
2611 return;
2612 };
2613}
2614
25452615pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {
25462616 var man = comp.cache_parent.obtain();
25472617
......@@ -2790,6 +2860,36 @@ fn reportRetryableAstGenError(
27902860 }
27912861}
27922862
2863fn reportRetryableEmbedFileError(
2864 comp: *Compilation,
2865 embed_file: *Module.EmbedFile,
2866 err: anyerror,
2867) error{OutOfMemory}!void {
2868 const mod = comp.bin_file.options.module.?;
2869 const gpa = mod.gpa;
2870
2871 const src_loc: Module.SrcLoc = embed_file.owner_decl.srcLoc();
2872
2873 const err_msg = if (embed_file.pkg.root_src_directory.path) |dir_path|
2874 try Module.ErrorMsg.create(
2875 gpa,
2876 src_loc,
2877 "unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}",
2878 .{ dir_path, embed_file.sub_file_path, @errorName(err) },
2879 )
2880 else
2881 try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{
2882 embed_file.sub_file_path, @errorName(err),
2883 });
2884 errdefer err_msg.destroy(gpa);
2885
2886 {
2887 const lock = comp.mutex.acquire();
2888 defer lock.release();
2889 try mod.failed_embed_files.putNoClobber(gpa, embed_file, err_msg);
2890 }
2891}
2892
27932893fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.Progress.Node) !void {
27942894 if (!build_options.have_llvm) {
27952895 return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
src/Module.zig+136-2
......@@ -55,11 +55,17 @@ decl_exports: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{},
5555/// is performing the export of another Decl.
5656/// This table owns the Export memory.
5757export_owners: std.AutoArrayHashMapUnmanaged(*Decl, []*Export) = .{},
58/// The set of all the files in the Module. We keep track of this in order to iterate
59/// over it and check which source files have been modified on the file system when
58/// The set of all the Zig source files in the Module. We keep track of this in order
59/// to iterate over it and check which source files have been modified on the file system when
6060/// an update is requested, as well as to cache `@import` results.
6161/// Keys are fully resolved file paths. This table owns the keys and values.
6262import_table: std.StringArrayHashMapUnmanaged(*File) = .{},
63/// The set of all the files which have been loaded with `@embedFile` in the Module.
64/// We keep track of this in order to iterate over it and check which files have been
65/// modified on the file system when an update is requested, as well as to cache
66/// `@embedFile` results.
67/// Keys are fully resolved file paths. This table owns the keys and values.
68embed_table: std.StringHashMapUnmanaged(*EmbedFile) = .{},
6369
6470/// The set of all the generic function instantiations. This is used so that when a generic
6571/// function is called twice with the same comptime parameter arguments, both calls dispatch
......@@ -87,6 +93,8 @@ compile_log_decls: std.AutoArrayHashMapUnmanaged(*Decl, i32) = .{},
8793/// Using a map here for consistency with the other fields here.
8894/// The ErrorMsg memory is owned by the `File`, using Module's general purpose allocator.
8995failed_files: std.AutoArrayHashMapUnmanaged(*File, ?*ErrorMsg) = .{},
96/// The ErrorMsg memory is owned by the `EmbedFile`, using Module's general purpose allocator.
97failed_embed_files: std.AutoArrayHashMapUnmanaged(*EmbedFile, *ErrorMsg) = .{},
9098/// Using a map here for consistency with the other fields here.
9199/// The ErrorMsg memory is owned by the `Export`, using Module's general purpose allocator.
92100failed_exports: std.AutoArrayHashMapUnmanaged(*Export, *ErrorMsg) = .{},
......@@ -1534,6 +1542,23 @@ pub const File = struct {
15341542 }
15351543};
15361544
1545/// Represents the contents of a file loaded with `@embedFile`.
1546pub const EmbedFile = struct {
1547 /// Relative to the owning package's root_src_dir.
1548 /// Memory is stored in gpa, owned by EmbedFile.
1549 sub_file_path: []const u8,
1550 bytes: [:0]const u8,
1551 stat_size: u64,
1552 stat_inode: std.fs.File.INode,
1553 stat_mtime: i128,
1554 /// Package that this file is a part of, managed externally.
1555 pkg: *Package,
1556 /// The Decl that was created from the `@embedFile` to own this resource.
1557 /// This is how zig knows what other Decl objects to invalidate if the file
1558 /// changes on disk.
1559 owner_decl: *Decl,
1560};
1561
15371562/// This struct holds data necessary to construct API-facing `AllErrors.Message`.
15381563/// Its memory is managed with the general purpose allocator so that they
15391564/// can be created and destroyed in response to incremental updates.
......@@ -2364,6 +2389,11 @@ pub fn deinit(mod: *Module) void {
23642389 }
23652390 mod.failed_files.deinit(gpa);
23662391
2392 for (mod.failed_embed_files.values()) |msg| {
2393 msg.destroy(gpa);
2394 }
2395 mod.failed_embed_files.deinit(gpa);
2396
23672397 for (mod.failed_exports.values()) |value| {
23682398 value.destroy(gpa);
23692399 }
......@@ -3060,6 +3090,32 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) SemaError!void {
30603090 }
30613091}
30623092
3093pub fn updateEmbedFile(mod: *Module, embed_file: *EmbedFile) SemaError!void {
3094 const tracy = trace(@src());
3095 defer tracy.end();
3096
3097 // TODO we can potentially relax this if we store some more information along
3098 // with decl dependency edges
3099 for (embed_file.owner_decl.dependants.keys()) |dep| {
3100 switch (dep.analysis) {
3101 .unreferenced => unreachable,
3102 .in_progress => continue, // already doing analysis, ok
3103 .outdated => continue, // already queued for update
3104
3105 .file_failure,
3106 .dependency_failure,
3107 .sema_failure,
3108 .sema_failure_retryable,
3109 .codegen_failure,
3110 .codegen_failure_retryable,
3111 .complete,
3112 => if (dep.generation != mod.generation) {
3113 try mod.markOutdatedDecl(dep);
3114 },
3115 }
3116 }
3117}
3118
30633119pub fn semaPkg(mod: *Module, pkg: *Package) !void {
30643120 const file = (try mod.importPkg(pkg)).file;
30653121 return mod.semaFile(file);
......@@ -3551,6 +3607,84 @@ pub fn importFile(
35513607 };
35523608}
35533609
3610pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*EmbedFile {
3611 const gpa = mod.gpa;
3612
3613 // The resolved path is used as the key in the table, to detect if
3614 // a file refers to the same as another, despite different relative paths.
3615 const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse ".";
3616 const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{
3617 cur_pkg_dir_path, cur_file.sub_file_path, "..", rel_file_path,
3618 });
3619 var keep_resolved_path = false;
3620 defer if (!keep_resolved_path) gpa.free(resolved_path);
3621
3622 const gop = try mod.embed_table.getOrPut(gpa, resolved_path);
3623 if (gop.found_existing) return gop.value_ptr.*;
3624 keep_resolved_path = true; // It's now owned by embed_table.
3625
3626 const new_file = try gpa.create(EmbedFile);
3627 errdefer gpa.destroy(new_file);
3628
3629 const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path});
3630 defer gpa.free(resolved_root_path);
3631
3632 if (!mem.startsWith(u8, resolved_path, resolved_root_path)) {
3633 return error.ImportOutsidePkgPath;
3634 }
3635 // +1 for the directory separator here.
3636 const sub_file_path = try gpa.dupe(u8, resolved_path[resolved_root_path.len + 1 ..]);
3637 errdefer gpa.free(sub_file_path);
3638
3639 var file = try cur_file.pkg.root_src_directory.handle.openFile(sub_file_path, .{});
3640 defer file.close();
3641
3642 const stat = try file.stat();
3643 const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), stat.size, 1, 0);
3644
3645 log.debug("new embedFile. resolved_root_path={s}, resolved_path={s}, sub_file_path={s}, rel_file_path={s}", .{
3646 resolved_root_path, resolved_path, sub_file_path, rel_file_path,
3647 });
3648
3649 gop.value_ptr.* = new_file;
3650 new_file.* = .{
3651 .sub_file_path = sub_file_path,
3652 .bytes = bytes,
3653 .stat_size = stat.size,
3654 .stat_inode = stat.inode,
3655 .stat_mtime = stat.mtime,
3656 .pkg = cur_file.pkg,
3657 .owner_decl = undefined, // Set by Sema immediately after this function returns.
3658 };
3659 return new_file;
3660}
3661
3662pub fn detectEmbedFileUpdate(mod: *Module, embed_file: *EmbedFile) !void {
3663 var file = try embed_file.pkg.root_src_directory.handle.openFile(embed_file.sub_file_path, .{});
3664 defer file.close();
3665
3666 const stat = try file.stat();
3667
3668 const unchanged_metadata =
3669 stat.size == embed_file.stat_size and
3670 stat.mtime == embed_file.stat_mtime and
3671 stat.inode == embed_file.stat_inode;
3672
3673 if (unchanged_metadata) return;
3674
3675 const gpa = mod.gpa;
3676 const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), stat.size, 1, 0);
3677 gpa.free(embed_file.bytes);
3678 embed_file.bytes = bytes;
3679 embed_file.stat_size = stat.size;
3680 embed_file.stat_mtime = stat.mtime;
3681 embed_file.stat_inode = stat.inode;
3682
3683 const lock = mod.comp.mutex.acquire();
3684 defer lock.release();
3685 try mod.comp.work_queue.writeItem(.{ .update_embed_file = embed_file });
3686}
3687
35543688pub fn scanNamespace(
35553689 mod: *Module,
35563690 namespace: *Namespace,
src/Sema.zig+39-6
......@@ -6467,6 +6467,45 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
64676467 return sema.addConstant(file_root_decl.ty, file_root_decl.val);
64686468}
64696469
6470fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
6471 const tracy = trace(@src());
6472 defer tracy.end();
6473
6474 const mod = sema.mod;
6475 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
6476 const src = inst_data.src();
6477 const name = try sema.resolveConstString(block, src, inst_data.operand);
6478
6479 const embed_file = mod.embedFile(block.getFileScope(), name) catch |err| switch (err) {
6480 error.ImportOutsidePkgPath => {
6481 return sema.fail(block, src, "embed of file outside package path: '{s}'", .{name});
6482 },
6483 else => {
6484 // TODO: these errors are file system errors; make sure an update() will
6485 // retry this and not cache the file system error, which may be transient.
6486 return sema.fail(block, src, "unable to open '{s}': {s}", .{ name, @errorName(err) });
6487 },
6488 };
6489
6490 var anon_decl = try block.startAnonDecl();
6491 defer anon_decl.deinit();
6492
6493 const bytes_including_null = embed_file.bytes[0 .. embed_file.bytes.len + 1];
6494
6495 // TODO instead of using `Value.Tag.bytes`, create a new value tag for pointing at
6496 // a `*Module.EmbedFile`. The purpose of this would be:
6497 // - If only the length is read and the bytes are not inspected by comptime code,
6498 // there can be an optimization where the codegen backend does a copy_file_range
6499 // into the final binary, and never loads the data into memory.
6500 // - When a Decl is destroyed, it can free the `*Module.EmbedFile`.
6501 embed_file.owner_decl = try anon_decl.finish(
6502 try Type.Tag.array_u8_sentinel_0.create(anon_decl.arena(), embed_file.bytes.len),
6503 try Value.Tag.bytes.create(anon_decl.arena(), bytes_including_null),
6504 );
6505
6506 return sema.analyzeDeclRef(embed_file.owner_decl);
6507}
6508
64706509fn zirRetErrValueCode(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
64716510 _ = block;
64726511 _ = inst;
......@@ -9020,12 +9059,6 @@ fn zirBoolToInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
90209059 return block.addUnOp(.bool_to_int, operand);
90219060}
90229061
9023fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
9024 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
9025 const src = inst_data.src();
9026 return sema.fail(block, src, "TODO: Sema.zirEmbedFile", .{});
9027}
9028
90299062fn zirErrorName(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
90309063 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
90319064 const src = inst_data.src();