authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 16:33:29-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 16:33:29-07:00
logb8861a94ddd6d58c69b1c6d2874df27cc3f6f347
tree317e62640380e451cdbd8c92dd953f8099a4f3f5
parentbd1465a3fedf2a377a9e350400107f32595097a0

stage2: building compiler_rt and libc static archive with stage1

* add CLI support for verbose debugging options * implement building compiler_rt and libc static archive using stage1 C++ backend * add function_sections and link_libcpp to root cache hash. * cleanups to use the new Directory.join method. * Package supports being initialized directly and cleaned up create() method. * fix classifyFileExt incorrectly saying .zir is .zig. Thanks @Rocknest! * unify updateSubCompilation implementations

8 files changed, 295 insertions(+), 165 deletions(-)

BRANCH_TODO+14-14
...@@ -1,6 +1,8 @@...@@ -1,6 +1,8 @@
1 * Cache integration for stage1 zig code compilation
1 * build & link against compiler-rt2 * build & link against compiler-rt
2 * build & link against freestanding libc3 * build & link against freestanding libc
3 * Cache integration for stage1 zig code compilation4 * resolve builtin.zig not working on windows & macos; try os_path_real
5 * build & link against libcxx and libcxxabi
4 * `zig test`6 * `zig test`
5 * `zig build`7 * `zig build`
6 * `-ftime-report`8 * `-ftime-report`
...@@ -16,32 +18,33 @@...@@ -16,32 +18,33 @@
16 * make sure zig cc works18 * make sure zig cc works
17 - using it as a preprocessor (-E)19 - using it as a preprocessor (-E)
18 - try building some software20 - try building some software
21 * implement proper parsing of LLD stderr/stdout and exposing compile errors
22 * implement proper parsing of clang stderr/stdout and exposing compile errors
19 * support rpaths in ELF linker code23 * support rpaths in ELF linker code
20 * repair @cImport24 * repair @cImport
21 * add CLI support for a way to pass extra flags to c source files25 * add CLI support for a way to pass extra flags to c source files
22 * capture lld stdout/stderr better
23 * musl26 * musl
24 * mingw-w6427 * mingw-w64
25 * use global zig-cache dir for crt files28 * use global zig-cache dir for crt files
26 * MachO LLD linking29 * MachO LLD linking
27 * COFF LLD linking30 * COFF LLD linking
28 * WASM LLD linking31 * WASM LLD linking
29 * implement proper parsing of LLD stderr/stdout and exposing compile errors
30 * implement proper parsing of clang stderr/stdout and exposing compile errors
31 * implement proper compile errors for failing to build glibc crt files and shared libs
32 * improve the stage2 tests to support testing with LLVM extensions enabled
33 * multi-thread building C objects
34 * support cross compiling stage2 with `zig build`32 * support cross compiling stage2 with `zig build`
35 * implement emit-h in stage2
36 * implement -fno-emit-bin
37 * audit the base cache hash
38 * --main-pkg-path33 * --main-pkg-path
39 * audit the CLI options for stage234 * audit the CLI options for stage2
40 * `zig init-lib`35 * `zig init-lib`
41 * `zig init-exe`36 * `zig init-exe`
42 * `zig run`37 * `zig run`
43 * restore error messages for stage2_add_link_lib38 * restore error messages for stage2_add_link_lib
39 * audit the base cache hash
4440
41 * implement proper compile errors for failing to build glibc crt files and shared libs
42 * implement -fno-emit-bin
43 * improve the stage2 tests to support testing with LLVM extensions enabled
44 * rename src/ to src/stage1/
45 * rename src-self-hosted/ to src/
46 * implement emit-h in stage2
47 * multi-thread building C objects
45 * implement serialization/deserialization of incremental compilation metadata48 * implement serialization/deserialization of incremental compilation metadata
46 * incremental compilation - implement detection of which source files changed49 * incremental compilation - implement detection of which source files changed
47 * improve the cache hash logic for c objects with respect to extra flags and file parameters50 * improve the cache hash logic for c objects with respect to extra flags and file parameters
...@@ -59,16 +62,13 @@...@@ -59,16 +62,13 @@
59 * integrate target features into building assembly code62 * integrate target features into building assembly code
60 * libc_installation.zig: make it look for msvc only if msvc abi is chosen63 * libc_installation.zig: make it look for msvc only if msvc abi is chosen
61 * switch the default C ABI for windows to be mingw-w6464 * switch the default C ABI for windows to be mingw-w64
62 * port windows_sdk.cpp to zig
63 * change glibc log errors to normal exposed compile errors65 * change glibc log errors to normal exposed compile errors
64 * update Package to use Compilation.Directory in create()66 * update Package to use Compilation.Directory in create()
65 - skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)67 - skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
66 (maybe make it an explicit option and have main.zig disable it)68 (maybe make it an explicit option and have main.zig disable it)
67 - make it possible for Package to not openDir and reference already existing resources.69 - make it possible for Package to not openDir and reference already existing resources.
68 * rename src/ to src/stage1/
69 * rename src-self-hosted/ to src/
70 * improve Directory.join to only use 1 allocation in a clean way.70 * improve Directory.join to only use 1 allocation in a clean way.
71 * tracy builds with lc++71 * tracy builds with lc++
72 * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax72 * some kind of "zig identifier escape" function rather than unconditionally using @"" syntax
73 in builtin.zig73 in builtin.zig
74 * rename Mode to OptimizeMode74 * rename std.builtin.Mode to std.builtin.OptimizeMode
src-self-hosted/Compilation.zig+217-73
...@@ -46,6 +46,12 @@ sanitize_c: bool,...@@ -46,6 +46,12 @@ sanitize_c: bool,
46clang_passthrough_mode: bool,46clang_passthrough_mode: bool,
47/// Whether to print clang argvs to stdout.47/// Whether to print clang argvs to stdout.
48verbose_cc: bool,48verbose_cc: bool,
49verbose_tokenize: bool,
50verbose_ast: bool,
51verbose_ir: bool,
52verbose_llvm_ir: bool,
53verbose_cimport: bool,
54verbose_llvm_cpu_features: bool,
49disable_c_depfile: bool,55disable_c_depfile: bool,
50is_test: bool,56is_test: bool,
5157
...@@ -59,18 +65,21 @@ zig_cache_directory: Directory,...@@ -59,18 +65,21 @@ zig_cache_directory: Directory,
59libc_include_dir_list: []const []const u8,65libc_include_dir_list: []const []const u8,
60rand: *std.rand.Random,66rand: *std.rand.Random,
6167
62/// Populated when we build libc++.a. A Job to build this is placed in the queue68/// Populated when we build the libc++ static library. A Job to build this is placed in the queue
63/// and resolved before calling linker.flush().69/// and resolved before calling linker.flush().
64libcxx_static_lib: ?[]const u8 = null,70libcxx_static_lib: ?[]const u8 = null,
65/// Populated when we build libc++abi.a. A Job to build this is placed in the queue71/// Populated when we build the libc++abi static library. A Job to build this is placed in the queue
66/// and resolved before calling linker.flush().72/// and resolved before calling linker.flush().
67libcxxabi_static_lib: ?[]const u8 = null,73libcxxabi_static_lib: ?[]const u8 = null,
68/// Populated when we build libunwind.a. A Job to build this is placed in the queue74/// Populated when we build the libunwind static library. A Job to build this is placed in the queue
69/// and resolved before calling linker.flush().75/// and resolved before calling linker.flush().
70libunwind_static_lib: ?CRTFile = null,76libunwind_static_lib: ?CRTFile = null,
71/// Populated when we build c.a. A Job to build this is placed in the queue77/// Populated when we build the libc static library. A Job to build this is placed in the queue
72/// and resolved before calling linker.flush().78/// and resolved before calling linker.flush().
73libc_static_lib: ?[]const u8 = null,79libc_static_lib: ?CRTFile = null,
80/// Populated when we build the libcompiler_rt static library. A Job to build this is placed in the queue
81/// and resolved before calling linker.flush().
82compiler_rt_static_lib: ?CRTFile = null,
7483
75glibc_so_files: ?glibc.BuiltSharedObjects = null,84glibc_so_files: ?glibc.BuiltSharedObjects = null,
7685
...@@ -121,6 +130,11 @@ const Job = union(enum) {...@@ -121,6 +130,11 @@ const Job = union(enum) {
121 glibc_shared_objects,130 glibc_shared_objects,
122 /// libunwind.a, usually needed when linking libc131 /// libunwind.a, usually needed when linking libc
123 libunwind: void,132 libunwind: void,
133 /// needed when producing a dynamic library or executable
134 libcompiler_rt: void,
135 /// needed when not linking libc and using LLVM for code generation because it generates
136 /// calls to, for example, memcpy and memset.
137 zig_libc: void,
124138
125 /// Generate builtin.zig source code and write it into the correct place.139 /// Generate builtin.zig source code and write it into the correct place.
126 generate_builtin_zig: void,140 generate_builtin_zig: void,
...@@ -310,6 +324,15 @@ pub const InitOptions = struct {...@@ -310,6 +324,15 @@ pub const InitOptions = struct {
310};324};
311325
312pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {326pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
327 const is_dyn_lib = switch (options.output_mode) {
328 .Obj, .Exe => false,
329 .Lib => (options.link_mode orelse .Static) == .Dynamic,
330 };
331 const is_exe_or_dyn_lib = switch (options.output_mode) {
332 .Obj => false,
333 .Lib => is_dyn_lib,
334 .Exe => true,
335 };
313 const comp: *Compilation = comp: {336 const comp: *Compilation = comp: {
314 // For allocations that have the same lifetime as Compilation. This arena is used only during this337 // For allocations that have the same lifetime as Compilation. This arena is used only during this
315 // initialization and then is freed in deinit().338 // initialization and then is freed in deinit().
...@@ -375,15 +398,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -375,15 +398,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
375 return error.MachineCodeModelNotSupported;398 return error.MachineCodeModelNotSupported;
376 }399 }
377400
378 const is_dyn_lib = switch (options.output_mode) {
379 .Obj, .Exe => false,
380 .Lib => (options.link_mode orelse .Static) == .Dynamic,
381 };
382 const is_exe_or_dyn_lib = switch (options.output_mode) {
383 .Obj => false,
384 .Lib => is_dyn_lib,
385 .Exe => true,
386 };
387 const must_dynamic_link = dl: {401 const must_dynamic_link = dl: {
388 if (target_util.cannotDynamicLink(options.target))402 if (target_util.cannotDynamicLink(options.target))
389 break :dl false;403 break :dl false;
...@@ -461,6 +475,27 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -461,6 +475,27 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
461 };475 };
462476
463 const single_threaded = options.single_threaded or target_util.isSingleThreaded(options.target);477 const single_threaded = options.single_threaded or target_util.isSingleThreaded(options.target);
478 const function_sections = options.function_sections orelse false;
479
480 const llvm_cpu_features: ?[*:0]const u8 = if (build_options.have_llvm and use_llvm) blk: {
481 var buf = std.ArrayList(u8).init(arena);
482 for (options.target.cpu.arch.allFeaturesList()) |feature, index_usize| {
483 const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize);
484 const is_enabled = options.target.cpu.features.isEnabled(index);
485
486 if (feature.llvm_name) |llvm_name| {
487 const plus_or_minus = "-+"[@boolToInt(is_enabled)];
488 try buf.ensureCapacity(buf.items.len + 2 + llvm_name.len);
489 buf.appendAssumeCapacity(plus_or_minus);
490 buf.appendSliceAssumeCapacity(llvm_name);
491 buf.appendSliceAssumeCapacity(",");
492 }
493 }
494 assert(mem.endsWith(u8, buf.items, ","));
495 buf.items[buf.items.len - 1] = 0;
496 buf.shrink(buf.items.len);
497 break :blk buf.items[0 .. buf.items.len - 1 :0].ptr;
498 } else null;
464499
465 // We put everything into the cache hash that *cannot be modified during an incremental update*.500 // We put everything into the cache hash that *cannot be modified during an incremental update*.
466 // For example, one cannot change the target between updates, but one can change source files,501 // For example, one cannot change the target between updates, but one can change source files,
...@@ -489,8 +524,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -489,8 +524,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
489 cache.hash.add(pic);524 cache.hash.add(pic);
490 cache.hash.add(stack_check);525 cache.hash.add(stack_check);
491 cache.hash.add(link_mode);526 cache.hash.add(link_mode);
527 cache.hash.add(function_sections);
492 cache.hash.add(options.strip);528 cache.hash.add(options.strip);
493 cache.hash.add(options.link_libc);529 cache.hash.add(options.link_libc);
530 cache.hash.add(options.link_libcpp);
494 cache.hash.add(options.output_mode);531 cache.hash.add(options.output_mode);
495 cache.hash.add(options.machine_code_model);532 cache.hash.add(options.machine_code_model);
496 // TODO audit this and make sure everything is in it533 // TODO audit this and make sure everything is in it
...@@ -596,10 +633,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -596,10 +633,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
596 owned_link_dir = artifact_dir;633 owned_link_dir = artifact_dir;
597 const link_artifact_directory: Directory = .{634 const link_artifact_directory: Directory = .{
598 .handle = artifact_dir,635 .handle = artifact_dir,
599 .path = if (options.zig_cache_directory.path) |p|636 .path = try options.zig_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
600 try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir })
601 else
602 artifact_sub_dir,
603 };637 };
604 break :blk link_artifact_directory;638 break :blk link_artifact_directory;
605 };639 };
...@@ -609,26 +643,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -609,26 +643,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
609 .ReleaseFast, .ReleaseSmall => false,643 .ReleaseFast, .ReleaseSmall => false,
610 };644 };
611645
612 const llvm_cpu_features: ?[*:0]const u8 = if (build_options.have_llvm and use_llvm) blk: {
613 var buf = std.ArrayList(u8).init(arena);
614 for (options.target.cpu.arch.allFeaturesList()) |feature, index_usize| {
615 const index = @intCast(Target.Cpu.Feature.Set.Index, index_usize);
616 const is_enabled = options.target.cpu.features.isEnabled(index);
617
618 if (feature.llvm_name) |llvm_name| {
619 const plus_or_minus = "-+"[@boolToInt(is_enabled)];
620 try buf.ensureCapacity(buf.items.len + 2 + llvm_name.len);
621 buf.appendAssumeCapacity(plus_or_minus);
622 buf.appendSliceAssumeCapacity(llvm_name);
623 buf.appendSliceAssumeCapacity(",");
624 }
625 }
626 assert(mem.endsWith(u8, buf.items, ","));
627 buf.items[buf.items.len - 1] = 0;
628 buf.shrink(buf.items.len);
629 break :blk buf.items[0 .. buf.items.len - 1 :0].ptr;
630 } else null;
631
632 const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: {646 const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: {
633 // Here we use the legacy stage1 C++ compiler to compile Zig code.647 // Here we use the legacy stage1 C++ compiler to compile Zig code.
634 const stage2_target = try arena.create(stage1.Stage2Target);648 const stage2_target = try arena.create(stage1.Stage2Target);
...@@ -646,7 +660,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -646,7 +660,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
646 if (options.color == .Off) progress.terminal = null;660 if (options.color == .Off) progress.terminal = null;
647661
648 const mod = module.?;662 const mod = module.?;
649 const main_zig_file = mod.root_pkg.root_src_path;663 const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{
664 mod.root_pkg.root_src_path,
665 });
650 const zig_lib_dir = options.zig_lib_directory.path.?;666 const zig_lib_dir = options.zig_lib_directory.path.?;
651 const builtin_sub = &[_][]const u8{"builtin.zig"};667 const builtin_sub = &[_][]const u8{"builtin.zig"};
652 const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub);668 const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub);
...@@ -700,7 +716,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -700,7 +716,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
700 .dll_export_fns = dll_export_fns,716 .dll_export_fns = dll_export_fns,
701 .link_mode_dynamic = link_mode == .Dynamic,717 .link_mode_dynamic = link_mode == .Dynamic,
702 .valgrind_enabled = valgrind,718 .valgrind_enabled = valgrind,
703 .function_sections = options.function_sections orelse false,719 .function_sections = function_sections,
704 .enable_stack_probing = stack_check,720 .enable_stack_probing = stack_check,
705 .enable_time_report = options.time_report,721 .enable_time_report = options.time_report,
706 .enable_stack_report = false,722 .enable_stack_report = false,
...@@ -790,6 +806,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -790,6 +806,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
790 .rand = options.rand,806 .rand = options.rand,
791 .clang_passthrough_mode = options.clang_passthrough_mode,807 .clang_passthrough_mode = options.clang_passthrough_mode,
792 .verbose_cc = options.verbose_cc,808 .verbose_cc = options.verbose_cc,
809 .verbose_tokenize = options.verbose_tokenize,
810 .verbose_ast = options.verbose_ast,
811 .verbose_ir = options.verbose_ir,
812 .verbose_llvm_ir = options.verbose_llvm_ir,
813 .verbose_cimport = options.verbose_cimport,
814 .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features,
793 .disable_c_depfile = options.disable_c_depfile,815 .disable_c_depfile = options.disable_c_depfile,
794 .owned_link_dir = owned_link_dir,816 .owned_link_dir = owned_link_dir,
795 .is_test = options.is_test,817 .is_test = options.is_test,
...@@ -823,10 +845,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -823,10 +845,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
823 if (comp.wantBuildLibUnwindFromSource()) {845 if (comp.wantBuildLibUnwindFromSource()) {
824 try comp.work_queue.writeItem(.{ .libunwind = {} });846 try comp.work_queue.writeItem(.{ .libunwind = {} });
825 }847 }
826
827 if (comp.stage1_module) |module| {848 if (comp.stage1_module) |module| {
828 try comp.work_queue.writeItem(.{ .stage1_module = {} });849 try comp.work_queue.writeItem(.{ .stage1_module = {} });
829 }850 }
851 if (is_exe_or_dyn_lib) {
852 try comp.work_queue.writeItem(.{ .libcompiler_rt = {} });
853 if (!comp.bin_file.options.link_libc) {
854 try comp.work_queue.writeItem(.{ .zig_libc = {} });
855 }
856 }
830857
831 return comp;858 return comp;
832}859}
...@@ -852,8 +879,14 @@ pub fn destroy(self: *Compilation) void {...@@ -852,8 +879,14 @@ pub fn destroy(self: *Compilation) void {
852 self.crt_files.deinit(gpa);879 self.crt_files.deinit(gpa);
853 }880 }
854881
855 if (self.libunwind_static_lib) |*unwind_crt_file| {882 if (self.libunwind_static_lib) |*crt_file| {
856 unwind_crt_file.deinit(gpa);883 crt_file.deinit(gpa);
884 }
885 if (self.compiler_rt_static_lib) |*crt_file| {
886 crt_file.deinit(gpa);
887 }
888 if (self.libc_static_lib) |*crt_file| {
889 crt_file.deinit(gpa);
857 }890 }
858891
859 for (self.c_object_table.items()) |entry| {892 for (self.c_object_table.items()) |entry| {
...@@ -889,41 +922,46 @@ pub fn update(self: *Compilation) !void {...@@ -889,41 +922,46 @@ pub fn update(self: *Compilation) !void {
889 self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key });922 self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key });
890 }923 }
891924
892 if (self.bin_file.options.module) |module| {925 const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
893 module.generation += 1;926 if (!use_stage1) {
894927 if (self.bin_file.options.module) |module| {
895 // TODO Detect which source files changed.928 module.generation += 1;
896 // Until then we simulate a full cache miss. Source files could have been loaded for any reason;929
897 // to force a refresh we unload now.930 // TODO Detect which source files changed.
898 if (module.root_scope.cast(Module.Scope.File)) |zig_file| {931 // Until then we simulate a full cache miss. Source files could have been loaded for any reason;
899 zig_file.unload(module.gpa);932 // to force a refresh we unload now.
900 module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) {933 if (module.root_scope.cast(Module.Scope.File)) |zig_file| {
901 error.AnalysisFail => {934 zig_file.unload(module.gpa);
902 assert(self.totalErrorCount() != 0);935 module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) {
903 },936 error.AnalysisFail => {
904 else => |e| return e,937 assert(self.totalErrorCount() != 0);
905 };938 },
906 } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| {939 else => |e| return e,
907 zir_module.unload(module.gpa);940 };
908 module.analyzeRootZIRModule(zir_module) catch |err| switch (err) {941 } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| {
909 error.AnalysisFail => {942 zir_module.unload(module.gpa);
910 assert(self.totalErrorCount() != 0);943 module.analyzeRootZIRModule(zir_module) catch |err| switch (err) {
911 },944 error.AnalysisFail => {
912 else => |e| return e,945 assert(self.totalErrorCount() != 0);
913 };946 },
947 else => |e| return e,
948 };
949 }
914 }950 }
915 }951 }
916952
917 try self.performAllTheWork();953 try self.performAllTheWork();
918954
919 if (self.bin_file.options.module) |module| {955 if (!use_stage1) {
920 // Process the deletion set.956 if (self.bin_file.options.module) |module| {
921 while (module.deletion_set.popOrNull()) |decl| {957 // Process the deletion set.
922 if (decl.dependants.items().len != 0) {958 while (module.deletion_set.popOrNull()) |decl| {
923 decl.deletion_flag = false;959 if (decl.dependants.items().len != 0) {
924 continue;960 decl.deletion_flag = false;
961 continue;
962 }
963 try module.deleteDecl(decl);
925 }964 }
926 try module.deleteDecl(decl);
927 }965 }
928 }966 }
929967
...@@ -1142,6 +1180,18 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {...@@ -1142,6 +1180,18 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
1142 fatal("unable to build libunwind: {}", .{@errorName(err)});1180 fatal("unable to build libunwind: {}", .{@errorName(err)});
1143 };1181 };
1144 },1182 },
1183 .libcompiler_rt => {
1184 self.buildStaticLibFromZig("compiler_rt.zig", &self.compiler_rt_static_lib) catch |err| {
1185 // TODO Expose this as a normal compile error rather than crashing here.
1186 fatal("unable to build compiler_rt: {}", .{@errorName(err)});
1187 };
1188 },
1189 .zig_libc => {
1190 self.buildStaticLibFromZig("c.zig", &self.libc_static_lib) catch |err| {
1191 // TODO Expose this as a normal compile error rather than crashing here.
1192 fatal("unable to build zig's multitarget libc: {}", .{@errorName(err)});
1193 };
1194 },
1145 .generate_builtin_zig => {1195 .generate_builtin_zig => {
1146 // This Job is only queued up if there is a zig module.1196 // This Job is only queued up if there is a zig module.
1147 self.updateBuiltinZigFile(self.bin_file.options.module.?) catch |err| {1197 self.updateBuiltinZigFile(self.bin_file.options.module.?) catch |err| {
...@@ -1691,7 +1741,7 @@ pub fn classifyFileExt(filename: []const u8) FileExt {...@@ -1691,7 +1741,7 @@ pub fn classifyFileExt(filename: []const u8) FileExt {
1691 } else if (mem.endsWith(u8, filename, ".zig")) {1741 } else if (mem.endsWith(u8, filename, ".zig")) {
1692 return .zig;1742 return .zig;
1693 } else if (mem.endsWith(u8, filename, ".zir")) {1743 } else if (mem.endsWith(u8, filename, ".zir")) {
1694 return .zig;1744 return .zir;
1695 } else if (hasSharedLibraryExt(filename)) {1745 } else if (hasSharedLibraryExt(filename)) {
1696 return .shared_library;1746 return .shared_library;
1697 } else if (hasStaticLibraryExt(filename)) {1747 } else if (hasStaticLibraryExt(filename)) {
...@@ -2030,3 +2080,97 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8...@@ -2030,3 +2080,97 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
2030 });2080 });
2031 return buffer.toOwnedSlice();2081 return buffer.toOwnedSlice();
2032}2082}
2083
2084pub fn updateSubCompilation(sub_compilation: *Compilation) !void {
2085 try sub_compilation.update();
2086
2087 // Look for compilation errors in this sub_compilation
2088 var errors = try sub_compilation.getAllErrorsAlloc();
2089 defer errors.deinit(sub_compilation.gpa);
2090
2091 if (errors.list.len != 0) {
2092 for (errors.list) |full_err_msg| {
2093 std.log.err("{}:{}:{}: {}\n", .{
2094 full_err_msg.src_path,
2095 full_err_msg.line + 1,
2096 full_err_msg.column + 1,
2097 full_err_msg.msg,
2098 });
2099 }
2100 return error.BuildingLibCObjectFailed;
2101 }
2102}
2103
2104fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFile) !void {
2105 const tracy = trace(@src());
2106 defer tracy.end();
2107
2108 const special_sub = "std" ++ std.fs.path.sep_str ++ "special";
2109 const special_path = try comp.zig_lib_directory.join(comp.gpa, &[_][]const u8{special_sub});
2110 defer comp.gpa.free(special_path);
2111
2112 var special_dir = try comp.zig_lib_directory.handle.openDir(special_sub, .{});
2113 defer special_dir.close();
2114
2115 var root_pkg: Package = .{
2116 .root_src_directory = .{
2117 .path = special_path,
2118 .handle = special_dir,
2119 },
2120 .root_src_path = basename,
2121 };
2122
2123 const emit_bin = Compilation.EmitLoc{
2124 .directory = null, // Put it in the cache directory.
2125 .basename = basename,
2126 };
2127 const optimize_mode: std.builtin.Mode = blk: {
2128 if (comp.is_test)
2129 break :blk comp.bin_file.options.optimize_mode;
2130 switch (comp.bin_file.options.optimize_mode) {
2131 .Debug, .ReleaseFast, .ReleaseSafe => break :blk .ReleaseFast,
2132 .ReleaseSmall => break :blk .ReleaseSmall,
2133 }
2134 };
2135 const sub_compilation = try Compilation.create(comp.gpa, .{
2136 // TODO use the global cache directory here
2137 .zig_cache_directory = comp.zig_cache_directory,
2138 .zig_lib_directory = comp.zig_lib_directory,
2139 .target = comp.getTarget(),
2140 .root_name = mem.split(basename, ".").next().?,
2141 .root_pkg = &root_pkg,
2142 .output_mode = .Lib,
2143 .rand = comp.rand,
2144 .libc_installation = comp.bin_file.options.libc_installation,
2145 .emit_bin = emit_bin,
2146 .optimize_mode = optimize_mode,
2147 .link_mode = .Static,
2148 .function_sections = true,
2149 .want_sanitize_c = false,
2150 .want_stack_check = false,
2151 .want_valgrind = false,
2152 .want_pic = comp.bin_file.options.pic,
2153 .emit_h = null,
2154 .strip = comp.bin_file.options.strip,
2155 .is_native_os = comp.bin_file.options.is_native_os,
2156 .self_exe_path = comp.self_exe_path,
2157 .verbose_cc = comp.verbose_cc,
2158 .verbose_link = comp.bin_file.options.verbose_link,
2159 .verbose_tokenize = comp.verbose_tokenize,
2160 .verbose_ast = comp.verbose_ast,
2161 .verbose_ir = comp.verbose_ir,
2162 .verbose_llvm_ir = comp.verbose_llvm_ir,
2163 .verbose_cimport = comp.verbose_cimport,
2164 .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
2165 .clang_passthrough_mode = comp.clang_passthrough_mode,
2166 });
2167 defer sub_compilation.destroy();
2168
2169 try sub_compilation.updateSubCompilation();
2170
2171 assert(out.* == null);
2172 out.* = Compilation.CRTFile{
2173 .full_object_path = try sub_compilation.bin_file.options.directory.join(comp.gpa, &[_][]const u8{basename}),
2174 .lock = sub_compilation.bin_file.toOwnedLock(),
2175 };
2176}
src-self-hosted/Package.zig+18-16
...@@ -1,32 +1,41 @@...@@ -1,32 +1,41 @@
1pub const Table = std.StringHashMapUnmanaged(*Package);1pub const Table = std.StringHashMapUnmanaged(*Package);
22
3root_src_directory: Compilation.Directory,3root_src_directory: Compilation.Directory,
4/// Relative to `root_src_directory`.4/// Relative to `root_src_directory`. May contain path separators.
5root_src_path: []u8,5root_src_path: []const u8,
6table: Table,6table: Table = .{},
7
8const std = @import("std");
9const mem = std.mem;
10const Allocator = std.mem.Allocator;
11const assert = std.debug.assert;
12const Package = @This();
13const Compilation = @import("Compilation.zig");
714
8/// No references to `root_src_dir` and `root_src_path` are kept.15/// No references to `root_src_dir` and `root_src_path` are kept.
9pub fn create(16pub fn create(
10 gpa: *Allocator,17 gpa: *Allocator,
11 base_dir: std.fs.Dir,18 base_directory: Compilation.Directory,
12 /// Relative to `base_dir`.19 /// Relative to `base_directory`.
13 root_src_dir: []const u8,20 root_src_dir: []const u8,
14 /// Relative to `root_src_dir`.21 /// Relative to `root_src_dir`.
15 root_src_path: []const u8,22 root_src_path: []const u8,
16) !*Package {23) !*Package {
17 const ptr = try gpa.create(Package);24 const ptr = try gpa.create(Package);
18 errdefer gpa.destroy(ptr);25 errdefer gpa.destroy(ptr);
26
27 const root_src_dir_path = try base_directory.join(gpa, &[_][]const u8{root_src_dir});
28 errdefer gpa.free(root_src_dir_path);
29
19 const root_src_path_dupe = try mem.dupe(gpa, u8, root_src_path);30 const root_src_path_dupe = try mem.dupe(gpa, u8, root_src_path);
20 errdefer gpa.free(root_src_path_dupe);31 errdefer gpa.free(root_src_path_dupe);
21 const root_src_dir_path = try mem.dupe(gpa, u8, root_src_dir);32
22 errdefer gpa.free(root_src_dir_path);
23 ptr.* = .{33 ptr.* = .{
24 .root_src_directory = .{34 .root_src_directory = .{
25 .path = root_src_dir_path,35 .path = root_src_dir_path,
26 .handle = try base_dir.openDir(root_src_dir, .{}),36 .handle = try base_directory.handle.openDir(root_src_dir, .{}),
27 },37 },
28 .root_src_path = root_src_path_dupe,38 .root_src_path = root_src_path_dupe,
29 .table = .{},
30 };39 };
31 return ptr;40 return ptr;
32}41}
...@@ -50,10 +59,3 @@ pub fn add(pkg: *Package, gpa: *Allocator, name: []const u8, package: *Package)...@@ -50,10 +59,3 @@ pub fn add(pkg: *Package, gpa: *Allocator, name: []const u8, package: *Package)
50 const name_dupe = try mem.dupe(gpa, u8, name);59 const name_dupe = try mem.dupe(gpa, u8, name);
51 pkg.table.putAssumeCapacityNoClobber(name_dupe, package);60 pkg.table.putAssumeCapacityNoClobber(name_dupe, package);
52}61}
53
54const std = @import("std");
55const mem = std.mem;
56const Allocator = std.mem.Allocator;
57const assert = std.debug.assert;
58const Package = @This();
59const Compilation = @import("Compilation.zig");
src-self-hosted/glibc.zig+14-22
...@@ -713,11 +713,17 @@ fn build_crt_file(...@@ -713,11 +713,17 @@ fn build_crt_file(
713 .c_source_files = c_source_files,713 .c_source_files = c_source_files,
714 .verbose_cc = comp.verbose_cc,714 .verbose_cc = comp.verbose_cc,
715 .verbose_link = comp.bin_file.options.verbose_link,715 .verbose_link = comp.bin_file.options.verbose_link,
716 .verbose_tokenize = comp.verbose_tokenize,
717 .verbose_ast = comp.verbose_ast,
718 .verbose_ir = comp.verbose_ir,
719 .verbose_llvm_ir = comp.verbose_llvm_ir,
720 .verbose_cimport = comp.verbose_cimport,
721 .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
716 .clang_passthrough_mode = comp.clang_passthrough_mode,722 .clang_passthrough_mode = comp.clang_passthrough_mode,
717 });723 });
718 defer sub_compilation.destroy();724 defer sub_compilation.destroy();
719725
720 try updateSubCompilation(sub_compilation);726 try sub_compilation.updateSubCompilation();
721727
722 try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);728 try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
723 const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p|729 const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p|
...@@ -989,6 +995,12 @@ fn buildSharedLib(...@@ -989,6 +995,12 @@ fn buildSharedLib(
989 .self_exe_path = comp.self_exe_path,995 .self_exe_path = comp.self_exe_path,
990 .verbose_cc = comp.verbose_cc,996 .verbose_cc = comp.verbose_cc,
991 .verbose_link = comp.bin_file.options.verbose_link,997 .verbose_link = comp.bin_file.options.verbose_link,
998 .verbose_tokenize = comp.verbose_tokenize,
999 .verbose_ast = comp.verbose_ast,
1000 .verbose_ir = comp.verbose_ir,
1001 .verbose_llvm_ir = comp.verbose_llvm_ir,
1002 .verbose_cimport = comp.verbose_cimport,
1003 .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
992 .clang_passthrough_mode = comp.clang_passthrough_mode,1004 .clang_passthrough_mode = comp.clang_passthrough_mode,
993 .version = version,1005 .version = version,
994 .version_script = map_file_path,1006 .version_script = map_file_path,
...@@ -997,25 +1009,5 @@ fn buildSharedLib(...@@ -997,25 +1009,5 @@ fn buildSharedLib(
997 });1009 });
998 defer sub_compilation.destroy();1010 defer sub_compilation.destroy();
9991011
1000 try updateSubCompilation(sub_compilation);1012 try sub_compilation.updateSubCompilation();
1001}
1002
1003fn updateSubCompilation(sub_compilation: *Compilation) !void {
1004 try sub_compilation.update();
1005
1006 // Look for compilation errors in this sub_compilation
1007 var errors = try sub_compilation.getAllErrorsAlloc();
1008 defer errors.deinit(sub_compilation.gpa);
1009
1010 if (errors.list.len != 0) {
1011 for (errors.list) |full_err_msg| {
1012 std.log.err("{}:{}:{}: {}\n", .{
1013 full_err_msg.src_path,
1014 full_err_msg.line + 1,
1015 full_err_msg.column + 1,
1016 full_err_msg.msg,
1017 });
1018 }
1019 return error.BuildingLibCObjectFailed;
1020 }
1021}1013}
src-self-hosted/libunwind.zig+7-21
...@@ -109,12 +109,18 @@ pub fn buildStaticLib(comp: *Compilation) !void {...@@ -109,12 +109,18 @@ pub fn buildStaticLib(comp: *Compilation) !void {
109 .c_source_files = &c_source_files,109 .c_source_files = &c_source_files,
110 .verbose_cc = comp.verbose_cc,110 .verbose_cc = comp.verbose_cc,
111 .verbose_link = comp.bin_file.options.verbose_link,111 .verbose_link = comp.bin_file.options.verbose_link,
112 .verbose_tokenize = comp.verbose_tokenize,
113 .verbose_ast = comp.verbose_ast,
114 .verbose_ir = comp.verbose_ir,
115 .verbose_llvm_ir = comp.verbose_llvm_ir,
116 .verbose_cimport = comp.verbose_cimport,
117 .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
112 .clang_passthrough_mode = comp.clang_passthrough_mode,118 .clang_passthrough_mode = comp.clang_passthrough_mode,
113 .link_libc = true,119 .link_libc = true,
114 });120 });
115 defer sub_compilation.destroy();121 defer sub_compilation.destroy();
116122
117 try updateSubCompilation(sub_compilation);123 try sub_compilation.updateSubCompilation();
118124
119 assert(comp.libunwind_static_lib == null);125 assert(comp.libunwind_static_lib == null);
120 comp.libunwind_static_lib = Compilation.CRTFile{126 comp.libunwind_static_lib = Compilation.CRTFile{
...@@ -122,23 +128,3 @@ pub fn buildStaticLib(comp: *Compilation) !void {...@@ -122,23 +128,3 @@ pub fn buildStaticLib(comp: *Compilation) !void {
122 .lock = sub_compilation.bin_file.toOwnedLock(),128 .lock = sub_compilation.bin_file.toOwnedLock(),
123 };129 };
124}130}
125
126fn updateSubCompilation(sub_compilation: *Compilation) !void {
127 try sub_compilation.update();
128
129 // Look for compilation errors in this sub_compilation
130 var errors = try sub_compilation.getAllErrorsAlloc();
131 defer errors.deinit(sub_compilation.gpa);
132
133 if (errors.list.len != 0) {
134 for (errors.list) |full_err_msg| {
135 std.log.err("{}:{}:{}: {}\n", .{
136 full_err_msg.src_path,
137 full_err_msg.line + 1,
138 full_err_msg.column + 1,
139 full_err_msg.msg,
140 });
141 }
142 return error.BuildingLibCObjectFailed;
143 }
144}
src-self-hosted/link.zig+7-5
...@@ -387,13 +387,15 @@ pub const File = struct {...@@ -387,13 +387,15 @@ pub const File = struct {
387 // If there is no Zig code to compile, then we should skip flushing the output file because it387 // If there is no Zig code to compile, then we should skip flushing the output file because it
388 // will not be part of the linker line anyway.388 // will not be part of the linker line anyway.
389 const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: {389 const module_obj_path: ?[]const u8 = if (base.options.module) |module| blk: {
390 const use_stage1 = build_options.is_stage1 and base.options.use_llvm;
391 if (use_stage1) {
392 const obj_basename = try std.fmt.allocPrint(arena, "{}.o", .{base.options.root_name});
393 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
394 break :blk full_obj_path;
395 }
390 try base.flushModule(comp);396 try base.flushModule(comp);
391
392 const obj_basename = base.intermediary_basename.?;397 const obj_basename = base.intermediary_basename.?;
393 const full_obj_path = if (directory.path) |dir_path|398 const full_obj_path = try directory.join(arena, &[_][]const u8{obj_basename});
394 try std.fs.path.join(arena, &[_][]const u8{ dir_path, obj_basename })
395 else
396 obj_basename;
397 break :blk full_obj_path;399 break :blk full_obj_path;
398 } else null;400 } else null;
399401
src-self-hosted/link/Elf.zig+10-12
...@@ -1253,8 +1253,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1253,8 +1253,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
12531253
1254 const is_lib = self.base.options.output_mode == .Lib;1254 const is_lib = self.base.options.output_mode == .Lib;
1255 const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;1255 const is_dyn_lib = self.base.options.link_mode == .Dynamic and is_lib;
1256 const is_exe_or_dyn_lib = is_dyn_lib or self.base.options.output_mode == .Exe;
1256 const have_dynamic_linker = self.base.options.link_libc and1257 const have_dynamic_linker = self.base.options.link_libc and
1257 self.base.options.link_mode == .Dynamic and (is_dyn_lib or self.base.options.output_mode == .Exe);1258 self.base.options.link_mode == .Dynamic and is_exe_or_dyn_lib;
12581259
1259 try ch.addOptionalFile(self.base.options.linker_script);1260 try ch.addOptionalFile(self.base.options.linker_script);
1260 try ch.addOptionalFile(self.base.options.version_script);1261 try ch.addOptionalFile(self.base.options.version_script);
...@@ -1489,16 +1490,13 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1489,16 +1490,13 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1489 try argv.append(p);1490 try argv.append(p);
1490 }1491 }
14911492
1492 // TODO compiler-rt and libc1493 // compiler-rt and libc
1493 //if (!g->is_dummy_so && (g->out_type == OutTypeExe || is_dyn_lib)) {1494 if (is_exe_or_dyn_lib) {
1494 // if (g->libc_link_lib == nullptr) {1495 if (!self.base.options.link_libc) {
1495 // Buf *libc_a_path = build_c(g, OutTypeLib, lj->build_dep_prog_node);1496 try argv.append(comp.libc_static_lib.?.full_object_path);
1496 // try argv.append(buf_ptr(libc_a_path));1497 }
1497 // }1498 try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
14981499 }
1499 // Buf *compiler_rt_o_path = build_compiler_rt(g, OutTypeLib, lj->build_dep_prog_node);
1500 // try argv.append(buf_ptr(compiler_rt_o_path));
1501 //}
15021500
1503 // Shared libraries.1501 // Shared libraries.
1504 try argv.ensureCapacity(argv.items.len + self.base.options.system_libs.len);1502 try argv.ensureCapacity(argv.items.len + self.base.options.system_libs.len);
...@@ -1545,7 +1543,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1545,7 +1543,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1545 try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a"));1543 try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a"));
1546 } else if (target.isMusl()) {1544 } else if (target.isMusl()) {
1547 try argv.append(comp.libunwind_static_lib.?.full_object_path);1545 try argv.append(comp.libunwind_static_lib.?.full_object_path);
1548 try argv.append(comp.libc_static_lib.?);1546 try argv.append(comp.libc_static_lib.?.full_object_path);
1549 } else if (self.base.options.link_libcpp) {1547 } else if (self.base.options.link_libcpp) {
1550 try argv.append(comp.libunwind_static_lib.?.full_object_path);1548 try argv.append(comp.libunwind_static_lib.?.full_object_path);
1551 } else {1549 } else {
src-self-hosted/main.zig+8-2
...@@ -1108,8 +1108,14 @@ pub fn buildOutputType(...@@ -1108,8 +1108,14 @@ pub fn buildOutputType(
1108 .yes => |p| p,1108 .yes => |p| p,
1109 };1109 };
11101110
1111 const root_pkg = if (root_src_file) |src_path| try Package.create(gpa, fs.cwd(), ".", src_path) else null;1111 var root_pkg_memory: Package = undefined;
1112 defer if (root_pkg) |pkg| pkg.destroy(gpa);1112 const root_pkg: ?*Package = if (root_src_file) |src_path| blk: {
1113 root_pkg_memory = .{
1114 .root_src_directory = .{ .path = null, .handle = fs.cwd() },
1115 .root_src_path = src_path,
1116 };
1117 break :blk &root_pkg_memory;
1118 } else null;
11131119
1114 const self_exe_path = try fs.selfExePathAlloc(arena);1120 const self_exe_path = try fs.selfExePathAlloc(arena);
1115 var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {1121 var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {