authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 18:21:49-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 18:21:49-07:00
log61c54b2d49bf087465e2c9a72a190e6ee5861027
tree41dbae68dffbbba689835f94b03afb36bbd76f1f
parentb8861a94ddd6d58c69b1c6d2874df27cc3f6f347

Cache integration for stage1 zig code compilation

6 files changed, 178 insertions(+), 111 deletions(-)

BRANCH_TODO-3
...@@ -1,6 +1,3 @@...@@ -1,6 +1,3 @@
1 * Cache integration for stage1 zig code compilation
2 * build & link against compiler-rt
3 * build & link against freestanding libc
4 * resolve builtin.zig not working on windows & macos; try os_path_real1 * resolve builtin.zig not working on windows & macos; try os_path_real
5 * build & link against libcxx and libcxxabi2 * build & link against libcxx and libcxxabi
6 * `zig test`3 * `zig test`
src-self-hosted/Cache.zig+1-1
...@@ -410,7 +410,7 @@ pub const CacheHash = struct {...@@ -410,7 +410,7 @@ pub const CacheHash = struct {
410 /// calculated. This is useful for processes that don't know the all the files that410 /// calculated. This is useful for processes that don't know the all the files that
411 /// are depended on ahead of time. For example, a source file that can import other files411 /// are depended on ahead of time. For example, a source file that can import other files
412 /// will need to be recompiled if the imported file is changed.412 /// will need to be recompiled if the imported file is changed.
413 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {413 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]const u8 {
414 assert(self.manifest_file != null);414 assert(self.manifest_file != null);
415415
416 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});416 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
src-self-hosted/Compilation.zig+173-103
...@@ -27,7 +27,8 @@ gpa: *Allocator,...@@ -27,7 +27,8 @@ gpa: *Allocator,
27arena_state: std.heap.ArenaAllocator.State,27arena_state: std.heap.ArenaAllocator.State,
28bin_file: *link.File,28bin_file: *link.File,
29c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},29c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
30stage1_module: ?*stage1.Module,30stage1_lock: ?Cache.Lock = null,
31stage1_cache_hash: *Cache.CacheHash = undefined,
3132
32link_error_flags: link.File.ErrorFlags = .{},33link_error_flags: link.File.ErrorFlags = .{},
3334
...@@ -54,6 +55,7 @@ verbose_cimport: bool,...@@ -54,6 +55,7 @@ verbose_cimport: bool,
54verbose_llvm_cpu_features: bool,55verbose_llvm_cpu_features: bool,
55disable_c_depfile: bool,56disable_c_depfile: bool,
56is_test: bool,57is_test: bool,
58time_report: bool,
5759
58c_source_files: []const CSourceFile,60c_source_files: []const CSourceFile,
59clang_argv: []const []const u8,61clang_argv: []const []const u8,
...@@ -91,6 +93,10 @@ crt_files: std.StringHashMapUnmanaged(CRTFile) = .{},...@@ -91,6 +93,10 @@ crt_files: std.StringHashMapUnmanaged(CRTFile) = .{},
91/// Keeping track of this possibly open resource so we can close it later.93/// Keeping track of this possibly open resource so we can close it later.
92owned_link_dir: ?std.fs.Dir,94owned_link_dir: ?std.fs.Dir,
9395
96/// This is for stage1 and should be deleted upon completion of self-hosting.
97/// Don't use this for anything other than stage1 compatibility.
98color: @import("main.zig").Color = .Auto,
99
94pub const InnerError = Module.InnerError;100pub const InnerError = Module.InnerError;
95101
96pub const CRTFile = struct {102pub const CRTFile = struct {
...@@ -643,100 +649,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -643,100 +649,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
643 .ReleaseFast, .ReleaseSmall => false,649 .ReleaseFast, .ReleaseSmall => false,
644 };650 };
645651
646 const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: {
647 // Here we use the legacy stage1 C++ compiler to compile Zig code.
648 const stage2_target = try arena.create(stage1.Stage2Target);
649 stage2_target.* = .{
650 .arch = @enumToInt(options.target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch
651 .os = @enumToInt(options.target.os.tag),
652 .abi = @enumToInt(options.target.abi),
653 .is_native_os = options.is_native_os,
654 .is_native_cpu = false, // Only true when bootstrapping the compiler.
655 .llvm_cpu_name = if (options.target.cpu.model.llvm_name) |s| s.ptr else null,
656 .llvm_cpu_features = llvm_cpu_features.?,
657 };
658 const progress = try arena.create(std.Progress);
659 const main_progress_node = try progress.start("", 100);
660 if (options.color == .Off) progress.terminal = null;
661
662 const mod = module.?;
663 const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{
664 mod.root_pkg.root_src_path,
665 });
666 const zig_lib_dir = options.zig_lib_directory.path.?;
667 const builtin_sub = &[_][]const u8{"builtin.zig"};
668 const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub);
669
670 const stage1_module = stage1.create(
671 @enumToInt(options.optimize_mode),
672 undefined,
673 0, // TODO --main-pkg-path
674 main_zig_file.ptr,
675 main_zig_file.len,
676 zig_lib_dir.ptr,
677 zig_lib_dir.len,
678 stage2_target,
679 options.is_test,
680 ) orelse return error.OutOfMemory;
681
682 const output_dir = bin_directory.path orelse ".";
683
684 const stage1_pkg = try arena.create(stage1.Pkg);
685 stage1_pkg.* = .{
686 .name_ptr = undefined,
687 .name_len = 0,
688 .path_ptr = undefined,
689 .path_len = 0,
690 .children_ptr = undefined,
691 .children_len = 0,
692 .parent = null,
693 };
694
695 stage1_module.* = .{
696 .root_name_ptr = root_name.ptr,
697 .root_name_len = root_name.len,
698 .output_dir_ptr = output_dir.ptr,
699 .output_dir_len = output_dir.len,
700 .builtin_zig_path_ptr = builtin_zig_path.ptr,
701 .builtin_zig_path_len = builtin_zig_path.len,
702 .test_filter_ptr = "",
703 .test_filter_len = 0,
704 .test_name_prefix_ptr = "",
705 .test_name_prefix_len = 0,
706 .userdata = @ptrToInt(comp),
707 .root_pkg = stage1_pkg,
708 .code_model = @enumToInt(options.machine_code_model),
709 .subsystem = stage1.TargetSubsystem.Auto,
710 .err_color = @enumToInt(options.color),
711 .pic = pic,
712 .link_libc = options.link_libc,
713 .link_libcpp = options.link_libcpp,
714 .strip = options.strip,
715 .is_single_threaded = single_threaded,
716 .dll_export_fns = dll_export_fns,
717 .link_mode_dynamic = link_mode == .Dynamic,
718 .valgrind_enabled = valgrind,
719 .function_sections = function_sections,
720 .enable_stack_probing = stack_check,
721 .enable_time_report = options.time_report,
722 .enable_stack_report = false,
723 .dump_analysis = false,
724 .enable_doc_generation = false,
725 .emit_bin = true,
726 .emit_asm = false,
727 .emit_llvm_ir = false,
728 .test_is_evented = false,
729 .verbose_tokenize = options.verbose_tokenize,
730 .verbose_ast = options.verbose_ast,
731 .verbose_ir = options.verbose_ir,
732 .verbose_llvm_ir = options.verbose_llvm_ir,
733 .verbose_cimport = options.verbose_cimport,
734 .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features,
735 .main_progress_node = main_progress_node,
736 };
737 break :blk stage1_module;
738 } else null;
739
740 const bin_file = try link.File.openPath(gpa, .{652 const bin_file = try link.File.openPath(gpa, .{
741 .directory = bin_directory,653 .directory = bin_directory,
742 .sub_path = emit_bin.basename,654 .sub_path = emit_bin.basename,
...@@ -793,7 +705,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -793,7 +705,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
793 .zig_lib_directory = options.zig_lib_directory,705 .zig_lib_directory = options.zig_lib_directory,
794 .zig_cache_directory = options.zig_cache_directory,706 .zig_cache_directory = options.zig_cache_directory,
795 .bin_file = bin_file,707 .bin_file = bin_file,
796 .stage1_module = stage1_module,
797 .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),708 .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
798 .keep_source_files_loaded = options.keep_source_files_loaded,709 .keep_source_files_loaded = options.keep_source_files_loaded,
799 .use_clang = use_clang,710 .use_clang = use_clang,
...@@ -815,6 +726,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -815,6 +726,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
815 .disable_c_depfile = options.disable_c_depfile,726 .disable_c_depfile = options.disable_c_depfile,
816 .owned_link_dir = owned_link_dir,727 .owned_link_dir = owned_link_dir,
817 .is_test = options.is_test,728 .is_test = options.is_test,
729 .color = options.color,
730 .time_report = options.time_report,
818 };731 };
819 break :comp comp;732 break :comp comp;
820 };733 };
...@@ -845,7 +758,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -845,7 +758,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
845 if (comp.wantBuildLibUnwindFromSource()) {758 if (comp.wantBuildLibUnwindFromSource()) {
846 try comp.work_queue.writeItem(.{ .libunwind = {} });759 try comp.work_queue.writeItem(.{ .libunwind = {} });
847 }760 }
848 if (comp.stage1_module) |module| {761 if (build_options.is_stage1 and comp.bin_file.options.use_llvm) {
849 try comp.work_queue.writeItem(.{ .stage1_module = {} });762 try comp.work_queue.writeItem(.{ .stage1_module = {} });
850 }763 }
851 if (is_exe_or_dyn_lib) {764 if (is_exe_or_dyn_lib) {
...@@ -858,15 +771,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -858,15 +771,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
858 return comp;771 return comp;
859}772}
860773
774fn releaseStage1Lock(comp: *Compilation) void {
775 if (comp.stage1_lock) |*lock| {
776 lock.release();
777 comp.stage1_lock = null;
778 }
779}
780
861pub fn destroy(self: *Compilation) void {781pub fn destroy(self: *Compilation) void {
862 const optional_module = self.bin_file.options.module;782 const optional_module = self.bin_file.options.module;
863 self.bin_file.destroy();783 self.bin_file.destroy();
864 if (optional_module) |module| module.deinit();784 if (optional_module) |module| module.deinit();
865785
866 if (self.stage1_module) |module| {786 self.releaseStage1Lock();
867 module.main_progress_node.?.end();
868 module.destroy();
869 }
870787
871 const gpa = self.gpa;788 const gpa = self.gpa;
872 self.work_queue.deinit();789 self.work_queue.deinit();
...@@ -1200,8 +1117,9 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {...@@ -1200,8 +1117,9 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
1200 };1117 };
1201 },1118 },
1202 .stage1_module => {1119 .stage1_module => {
1203 // This Job is only queued up if there is a zig module.1120 self.updateStage1Module() catch |err| {
1204 self.stage1_module.?.build_object();1121 fatal("unable to build stage1 zig object: {}", .{@errorName(err)});
1122 };
1205 },1123 },
1206 };1124 };
1207}1125}
...@@ -2174,3 +2092,155 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil...@@ -2174,3 +2092,155 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil
2174 .lock = sub_compilation.bin_file.toOwnedLock(),2092 .lock = sub_compilation.bin_file.toOwnedLock(),
2175 };2093 };
2176}2094}
2095
2096fn updateStage1Module(comp: *Compilation) !void {
2097 const tracy = trace(@src());
2098 defer tracy.end();
2099
2100 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
2101 defer arena_allocator.deinit();
2102 const arena = &arena_allocator.allocator;
2103
2104 // Here we use the legacy stage1 C++ compiler to compile Zig code.
2105 const mod = comp.bin_file.options.module.?;
2106 const directory = mod.zig_cache_artifact_directory; // Just an alias to make it shorter to type.
2107 const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{
2108 mod.root_pkg.root_src_path,
2109 });
2110 const zig_lib_dir = comp.zig_lib_directory.path.?;
2111 const builtin_zig_path = try directory.join(arena, &[_][]const u8{"builtin.zig"});
2112 const target = comp.getTarget();
2113 const id_symlink_basename = "stage1.id";
2114
2115 // We are about to obtain this lock, so here we give other processes a chance first.
2116 comp.releaseStage1Lock();
2117
2118 // Unlike with the self-hosted Zig module, stage1 does not support incremental compilation,
2119 // so we input all the zig source files into the cache hash system. We're going to keep
2120 // the artifact directory the same, however, so we take the same strategy as linking
2121 // does where we have a file which specifies the hash of the output directory so that we can
2122 // skip the expensive compilation step if the hash matches.
2123 var ch = comp.cache_parent.obtain();
2124 defer ch.deinit();
2125
2126 _ = try ch.addFile(main_zig_file, null);
2127 ch.hash.add(comp.bin_file.options.valgrind);
2128 ch.hash.add(comp.bin_file.options.single_threaded);
2129 ch.hash.add(target.os.getVersionRange());
2130 ch.hash.add(comp.bin_file.options.dll_export_fns);
2131 ch.hash.add(comp.bin_file.options.function_sections);
2132
2133 if (try ch.hit()) {
2134 const digest = ch.final();
2135
2136 var prev_digest_buf: [digest.len]u8 = undefined;
2137 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
2138 // Handle this as a cache miss.
2139 break :blk prev_digest_buf[0..0];
2140 };
2141 if (mem.eql(u8, prev_digest, &digest)) {
2142 comp.stage1_lock = ch.toOwnedLock();
2143 return;
2144 }
2145 }
2146
2147 const stage2_target = try arena.create(stage1.Stage2Target);
2148 stage2_target.* = .{
2149 .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch
2150 .os = @enumToInt(target.os.tag),
2151 .abi = @enumToInt(target.abi),
2152 .is_native_os = comp.bin_file.options.is_native_os,
2153 .is_native_cpu = false, // Only true when bootstrapping the compiler.
2154 .llvm_cpu_name = if (target.cpu.model.llvm_name) |s| s.ptr else null,
2155 .llvm_cpu_features = comp.bin_file.options.llvm_cpu_features.?,
2156 };
2157 var progress: std.Progress = .{};
2158 var main_progress_node = try progress.start("", 100);
2159 defer main_progress_node.end();
2160 if (comp.color == .Off) progress.terminal = null;
2161
2162 comp.stage1_cache_hash = &ch;
2163
2164 const stage1_module = stage1.create(
2165 @enumToInt(comp.bin_file.options.optimize_mode),
2166 undefined,
2167 0, // TODO --main-pkg-path
2168 main_zig_file.ptr,
2169 main_zig_file.len,
2170 zig_lib_dir.ptr,
2171 zig_lib_dir.len,
2172 stage2_target,
2173 comp.is_test,
2174 ) orelse return error.OutOfMemory;
2175
2176 const stage1_pkg = try arena.create(stage1.Pkg);
2177 stage1_pkg.* = .{
2178 .name_ptr = undefined,
2179 .name_len = 0,
2180 .path_ptr = undefined,
2181 .path_len = 0,
2182 .children_ptr = undefined,
2183 .children_len = 0,
2184 .parent = null,
2185 };
2186 const output_dir = comp.bin_file.options.directory.path orelse ".";
2187 stage1_module.* = .{
2188 .root_name_ptr = comp.bin_file.options.root_name.ptr,
2189 .root_name_len = comp.bin_file.options.root_name.len,
2190 .output_dir_ptr = output_dir.ptr,
2191 .output_dir_len = output_dir.len,
2192 .builtin_zig_path_ptr = builtin_zig_path.ptr,
2193 .builtin_zig_path_len = builtin_zig_path.len,
2194 .test_filter_ptr = "",
2195 .test_filter_len = 0,
2196 .test_name_prefix_ptr = "",
2197 .test_name_prefix_len = 0,
2198 .userdata = @ptrToInt(comp),
2199 .root_pkg = stage1_pkg,
2200 .code_model = @enumToInt(comp.bin_file.options.machine_code_model),
2201 .subsystem = stage1.TargetSubsystem.Auto,
2202 .err_color = @enumToInt(comp.color),
2203 .pic = comp.bin_file.options.pic,
2204 .link_libc = comp.bin_file.options.link_libc,
2205 .link_libcpp = comp.bin_file.options.link_libcpp,
2206 .strip = comp.bin_file.options.strip,
2207 .is_single_threaded = comp.bin_file.options.single_threaded,
2208 .dll_export_fns = comp.bin_file.options.dll_export_fns,
2209 .link_mode_dynamic = comp.bin_file.options.link_mode == .Dynamic,
2210 .valgrind_enabled = comp.bin_file.options.valgrind,
2211 .function_sections = comp.bin_file.options.function_sections,
2212 .enable_stack_probing = comp.bin_file.options.stack_check,
2213 .enable_time_report = comp.time_report,
2214 .enable_stack_report = false,
2215 .dump_analysis = false,
2216 .enable_doc_generation = false,
2217 .emit_bin = true,
2218 .emit_asm = false,
2219 .emit_llvm_ir = false,
2220 .test_is_evented = false,
2221 .verbose_tokenize = comp.verbose_tokenize,
2222 .verbose_ast = comp.verbose_ast,
2223 .verbose_ir = comp.verbose_ir,
2224 .verbose_llvm_ir = comp.verbose_llvm_ir,
2225 .verbose_cimport = comp.verbose_cimport,
2226 .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
2227 .main_progress_node = main_progress_node,
2228 };
2229 stage1_module.build_object();
2230 stage1_module.destroy();
2231
2232 const digest = ch.final();
2233
2234 // Update the dangling symlink with the digest. If it fails we can continue; it only
2235 // means that the next invocation will have an unnecessary cache miss.
2236 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
2237 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
2238 };
2239 // Again failure here only means an unnecessary cache miss.
2240 ch.writeManifest() catch |err| {
2241 std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
2242 };
2243 // We hang on to this lock so that the output file path can be used without
2244 // other processes clobbering it.
2245 comp.stage1_lock = ch.toOwnedLock();
2246}
src-self-hosted/link/Elf.zig+1-1
...@@ -1590,7 +1590,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1590,7 +1590,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1590 const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0);1590 const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0);
1591 if (!ok) return error.LLDReportedFailure;1591 if (!ok) return error.LLDReportedFailure;
15921592
1593 // Update the dangling symlink "id.txt" with the digest. If it fails we can continue; it only1593 // Update the dangling symlink with the digest. If it fails we can continue; it only
1594 // means that the next invocation will have an unnecessary cache miss.1594 // means that the next invocation will have an unnecessary cache miss.
1595 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {1595 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
1596 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});1596 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
src-self-hosted/main.zig+1-1
...@@ -1239,7 +1239,7 @@ pub fn buildOutputType(...@@ -1239,7 +1239,7 @@ pub fn buildOutputType(
1239 fatal("TODO: implement `zig cc` when using it as a preprocessor", .{});1239 fatal("TODO: implement `zig cc` when using it as a preprocessor", .{});
1240 }1240 }
12411241
1242 if (build_options.is_stage1 and comp.stage1_module != null and watch) {1242 if (build_options.is_stage1 and comp.stage1_lock != null and watch) {
1243 std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{});1243 std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{});
1244 }1244 }
12451245
src-self-hosted/stage1.zig+2-2
...@@ -343,9 +343,9 @@ export fn stage2_fetch_file(...@@ -343,9 +343,9 @@ export fn stage2_fetch_file(
343 result_len: *usize,343 result_len: *usize,
344) ?[*]const u8 {344) ?[*]const u8 {
345 const comp = @intToPtr(*Compilation, stage1.userdata);345 const comp = @intToPtr(*Compilation, stage1.userdata);
346 // TODO integrate this with cache hash
347 const file_path = path_ptr[0..path_len];346 const file_path = path_ptr[0..path_len];
348 const contents = std.fs.cwd().readFileAlloc(comp.gpa, file_path, std.math.maxInt(u32)) catch return null;347 const max_file_size = std.math.maxInt(u32);
348 const contents = comp.stage1_cache_hash.addFilePostFetch(file_path, max_file_size) catch return null;
349 result_len.* = contents.len;349 result_len.* = contents.len;
350 return contents.ptr;350 return contents.ptr;
351}351}