authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 03:02:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 03:02:46-07:00
log95941c4e7021588f224f0559c6f66de3a205b972
tree230b70c9dc2504be1c7525ade95dbc5b4af97435
parentda0fde59b6ffde5d505f664aa27b728aa2b1cc2a

stage2: building glibc shared objects

* caching system: use 16 bytes siphash final(), there was a bug in the std lib that wasn't catching undefined values for 18 bytes. fixed in master branch. * fix caching system unit test logic to not cause error.TextBusy on windows * port the logic from stage1 for building glibc shared objects * add is_native_os to the base cache hash * fix incorrectly freeing crt_files key (which is always a reference to global static constant data) * fix 2 use-after-free in loading glibc metadata * fix memory leak in buildCRTFile (errdefer instead of defer on arena)

5 files changed, 425 insertions(+), 165 deletions(-)

BRANCH_TODO+10-1
...@@ -1,4 +1,7 @@...@@ -1,4 +1,7 @@
1 * glibc .so files1 * glibc .so files
2 - stage1 C++ code integration
3 - ok file
4 * use hex for cache hash file paths
2 * support rpaths in ELF linker code5 * support rpaths in ELF linker code
3 * build & link against compiler-rt6 * build & link against compiler-rt
4 * build & link againstn freestanding libc7 * build & link againstn freestanding libc
...@@ -21,7 +24,6 @@...@@ -21,7 +24,6 @@
21 * implement proper parsing of LLD stderr/stdout and exposing compile errors24 * implement proper parsing of LLD stderr/stdout and exposing compile errors
22 * implement proper parsing of clang stderr/stdout and exposing compile errors25 * implement proper parsing of clang stderr/stdout and exposing compile errors
23 * implement proper compile errors for failing to build glibc crt files and shared libs26 * implement proper compile errors for failing to build glibc crt files and shared libs
24 * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
25 * self-host link.cpp and building libcs (#4313 and #4314). using the `zig cc` command will set a flag indicating a preference for the llvm backend, which will include linking with LLD. At least for now. If zig's self-hosted linker ever gets on par with the likes of ld and lld, we can make it always be used even for zig cc.27 * self-host link.cpp and building libcs (#4313 and #4314). using the `zig cc` command will set a flag indicating a preference for the llvm backend, which will include linking with LLD. At least for now. If zig's self-hosted linker ever gets on par with the likes of ld and lld, we can make it always be used even for zig cc.
26 * improve the stage2 tests to support testing with LLVM extensions enabled28 * improve the stage2 tests to support testing with LLVM extensions enabled
27 * multi-thread building C objects29 * multi-thread building C objects
...@@ -47,3 +49,10 @@...@@ -47,3 +49,10 @@
47 * libc_installation.zig: make it look for msvc only if msvc abi is chosen49 * libc_installation.zig: make it look for msvc only if msvc abi is chosen
48 * switch the default C ABI for windows to be mingw-w6450 * switch the default C ABI for windows to be mingw-w64
49 * port windows_sdk.cpp to zig51 * port windows_sdk.cpp to zig
52 * change glibc log errors to normal exposed compile errors
53 * update Package to use Compilation.Directory in create()
54 - skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
55 (maybe make it an explicit option and have main.zig disable it)
56 - make it possible for Package to not openDir and reference already existing resources.
57 * rename src/ to src/stage1/
58 * rename src-self-hosted/ to src/
ci/azure/windows_mingw_script+4
...@@ -18,4 +18,8 @@ cmake .. -G 'MSYS Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo $CMAKEFLAGS -DCMA...@@ -18,4 +18,8 @@ cmake .. -G 'MSYS Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo $CMAKEFLAGS -DCMA
1818
19make -j$(nproc) install19make -j$(nproc) install
2020
21# I saw a failure due to `git diff` being > 400 KB instead of empty as expected so this is to debug it.
22git status
23git diff | head -n100
24
21./zig build test-behavior -Dskip-non-native -Dskip-release25./zig build test-behavior -Dskip-non-native -Dskip-release
src-self-hosted/Cache.zig+136-115
...@@ -26,9 +26,9 @@ pub fn obtain(cache: *const Cache) CacheHash {...@@ -26,9 +26,9 @@ pub fn obtain(cache: *const Cache) CacheHash {
2626
27pub const base64_encoder = fs.base64_encoder;27pub const base64_encoder = fs.base64_encoder;
28pub const base64_decoder = fs.base64_decoder;28pub const base64_decoder = fs.base64_decoder;
29/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-629/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
30/// We round up to 18 to avoid the `==` padding after base64 encoding.30/// Currently we use SipHash and so this value must be 16 not any higher.
31pub const BIN_DIGEST_LEN = 18;31pub const BIN_DIGEST_LEN = 16;
32pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);32pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
3333
34const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;34const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
...@@ -87,14 +87,29 @@ pub const HashHelper = struct {...@@ -87,14 +87,29 @@ pub const HashHelper = struct {
87 hh.add(x.major);87 hh.add(x.major);
88 hh.add(x.minor);88 hh.add(x.minor);
89 hh.add(x.patch);89 hh.add(x.patch);
90 return;
91 },90 },
92 else => {},91 std.Target.Os.TaggedVersionRange => {
93 }92 switch (x) {
9493 .linux => |linux| {
95 switch (@typeInfo(@TypeOf(x))) {94 hh.add(linux.range.min);
96 .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),95 hh.add(linux.range.max);
97 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),96 hh.add(linux.glibc);
97 },
98 .windows => |windows| {
99 hh.add(windows.min);
100 hh.add(windows.max);
101 },
102 .semver => |semver| {
103 hh.add(semver.min);
104 hh.add(semver.max);
105 },
106 .none => {},
107 }
108 },
109 else => switch (@typeInfo(@TypeOf(x))) {
110 .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
111 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
112 },
98 }113 }
99 }114 }
100115
...@@ -613,44 +628,46 @@ test "cache file and then recall it" {...@@ -613,44 +628,46 @@ test "cache file and then recall it" {
613 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;628 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
614 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;629 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
615630
616 var cache = Cache{
617 .gpa = testing.allocator,
618 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
619 };
620 defer cache.manifest_dir.close();
621
622 {631 {
623 var ch = cache.obtain();632 var cache = Cache{
624 defer ch.deinit();633 .gpa = testing.allocator,
634 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
635 };
636 defer cache.manifest_dir.close();
625637
626 ch.hash.add(true);638 {
627 ch.hash.add(@as(u16, 1234));639 var ch = cache.obtain();
628 ch.hash.addBytes("1234");640 defer ch.deinit();
629 _ = try ch.addFile(temp_file, null);
630641
631 // There should be nothing in the cache642 ch.hash.add(true);
632 testing.expectEqual(false, try ch.hit());643 ch.hash.add(@as(u16, 1234));
644 ch.hash.addBytes("1234");
645 _ = try ch.addFile(temp_file, null);
633646
634 digest1 = ch.final();647 // There should be nothing in the cache
635 try ch.writeManifest();648 testing.expectEqual(false, try ch.hit());
636 }
637 {
638 var ch = cache.obtain();
639 defer ch.deinit();
640649
641 ch.hash.add(true);650 digest1 = ch.final();
642 ch.hash.add(@as(u16, 1234));651 try ch.writeManifest();
643 ch.hash.addBytes("1234");652 }
644 _ = try ch.addFile(temp_file, null);653 {
654 var ch = cache.obtain();
655 defer ch.deinit();
645656
646 // Cache hit! We just "built" the same file657 ch.hash.add(true);
647 testing.expect(try ch.hit());658 ch.hash.add(@as(u16, 1234));
648 digest2 = ch.final();659 ch.hash.addBytes("1234");
660 _ = try ch.addFile(temp_file, null);
649661
650 try ch.writeManifest();662 // Cache hit! We just "built" the same file
651 }663 testing.expect(try ch.hit());
664 digest2 = ch.final();
652665
653 testing.expectEqual(digest1, digest2);666 try ch.writeManifest();
667 }
668
669 testing.expectEqual(digest1, digest2);
670 }
654671
655 try cwd.deleteTree(temp_manifest_dir);672 try cwd.deleteTree(temp_manifest_dir);
656 try cwd.deleteFile(temp_file);673 try cwd.deleteFile(temp_file);
...@@ -693,50 +710,52 @@ test "check that changing a file makes cache fail" {...@@ -693,50 +710,52 @@ test "check that changing a file makes cache fail" {
693 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;710 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
694 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;711 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
695712
696 var cache = Cache{
697 .gpa = testing.allocator,
698 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
699 };
700 defer cache.manifest_dir.close();
701
702 {713 {
703 var ch = cache.obtain();714 var cache = Cache{
704 defer ch.deinit();715 .gpa = testing.allocator,
716 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
717 };
718 defer cache.manifest_dir.close();
705719
706 ch.hash.addBytes("1234");720 {
707 const temp_file_idx = try ch.addFile(temp_file, 100);721 var ch = cache.obtain();
722 defer ch.deinit();
708723
709 // There should be nothing in the cache724 ch.hash.addBytes("1234");
710 testing.expectEqual(false, try ch.hit());725 const temp_file_idx = try ch.addFile(temp_file, 100);
711726
712 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));727 // There should be nothing in the cache
728 testing.expectEqual(false, try ch.hit());
713729
714 digest1 = ch.final();730 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
715731
716 try ch.writeManifest();732 digest1 = ch.final();
717 }
718733
719 try cwd.writeFile(temp_file, updated_temp_file_contents);734 try ch.writeManifest();
735 }
720736
721 {737 try cwd.writeFile(temp_file, updated_temp_file_contents);
722 var ch = cache.obtain();
723 defer ch.deinit();
724738
725 ch.hash.addBytes("1234");739 {
726 const temp_file_idx = try ch.addFile(temp_file, 100);740 var ch = cache.obtain();
741 defer ch.deinit();
727742
728 // A file that we depend on has been updated, so the cache should not contain an entry for it743 ch.hash.addBytes("1234");
729 testing.expectEqual(false, try ch.hit());744 const temp_file_idx = try ch.addFile(temp_file, 100);
730745
731 // The cache system does not keep the contents of re-hashed input files.746 // A file that we depend on has been updated, so the cache should not contain an entry for it
732 testing.expect(ch.files.items[temp_file_idx].contents == null);747 testing.expectEqual(false, try ch.hit());
733748
734 digest2 = ch.final();749 // The cache system does not keep the contents of re-hashed input files.
750 testing.expect(ch.files.items[temp_file_idx].contents == null);
735751
736 try ch.writeManifest();752 digest2 = ch.final();
737 }753
754 try ch.writeManifest();
755 }
738756
739 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));757 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
758 }
740759
741 try cwd.deleteTree(temp_manifest_dir);760 try cwd.deleteTree(temp_manifest_dir);
742 try cwd.deleteTree(temp_file);761 try cwd.deleteTree(temp_file);
...@@ -749,7 +768,7 @@ test "no file inputs" {...@@ -749,7 +768,7 @@ test "no file inputs" {
749 }768 }
750 const cwd = fs.cwd();769 const cwd = fs.cwd();
751 const temp_manifest_dir = "no_file_inputs_manifest_dir";770 const temp_manifest_dir = "no_file_inputs_manifest_dir";
752 defer cwd.deleteTree(temp_manifest_dir) catch unreachable;771 defer cwd.deleteTree(temp_manifest_dir) catch {};
753772
754 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;773 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
755 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;774 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
...@@ -810,67 +829,69 @@ test "CacheHashes with files added after initial hash work" {...@@ -810,67 +829,69 @@ test "CacheHashes with files added after initial hash work" {
810 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;829 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
811 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;830 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
812831
813 var cache = Cache{
814 .gpa = testing.allocator,
815 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
816 };
817 defer cache.manifest_dir.close();
818
819 {832 {
820 var ch = cache.obtain();833 var cache = Cache{
821 defer ch.deinit();834 .gpa = testing.allocator,
835 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
836 };
837 defer cache.manifest_dir.close();
822838
823 ch.hash.addBytes("1234");839 {
824 _ = try ch.addFile(temp_file1, null);840 var ch = cache.obtain();
841 defer ch.deinit();
825842
826 // There should be nothing in the cache843 ch.hash.addBytes("1234");
827 testing.expectEqual(false, try ch.hit());844 _ = try ch.addFile(temp_file1, null);
828845
829 _ = try ch.addFilePost(temp_file2);846 // There should be nothing in the cache
847 testing.expectEqual(false, try ch.hit());
830848
831 digest1 = ch.final();849 _ = try ch.addFilePost(temp_file2);
832 try ch.writeManifest();
833 }
834 {
835 var ch = cache.obtain();
836 defer ch.deinit();
837850
838 ch.hash.addBytes("1234");851 digest1 = ch.final();
839 _ = try ch.addFile(temp_file1, null);852 try ch.writeManifest();
853 }
854 {
855 var ch = cache.obtain();
856 defer ch.deinit();
840857
841 testing.expect(try ch.hit());858 ch.hash.addBytes("1234");
842 digest2 = ch.final();859 _ = try ch.addFile(temp_file1, null);
843860
844 try ch.writeManifest();861 testing.expect(try ch.hit());
845 }862 digest2 = ch.final();
846 testing.expect(mem.eql(u8, &digest1, &digest2));
847863
848 // Modify the file added after initial hash864 try ch.writeManifest();
849 const ts2 = std.time.nanoTimestamp();865 }
850 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");866 testing.expect(mem.eql(u8, &digest1, &digest2));
851867
852 while (isProblematicTimestamp(ts2)) {868 // Modify the file added after initial hash
853 std.time.sleep(1);869 const ts2 = std.time.nanoTimestamp();
854 }870 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
855871
856 {872 while (isProblematicTimestamp(ts2)) {
857 var ch = cache.obtain();873 std.time.sleep(1);
858 defer ch.deinit();874 }
859875
860 ch.hash.addBytes("1234");876 {
861 _ = try ch.addFile(temp_file1, null);877 var ch = cache.obtain();
878 defer ch.deinit();
862879
863 // A file that we depend on has been updated, so the cache should not contain an entry for it880 ch.hash.addBytes("1234");
864 testing.expectEqual(false, try ch.hit());881 _ = try ch.addFile(temp_file1, null);
865882
866 _ = try ch.addFilePost(temp_file2);883 // A file that we depend on has been updated, so the cache should not contain an entry for it
884 testing.expectEqual(false, try ch.hit());
867885
868 digest3 = ch.final();886 _ = try ch.addFilePost(temp_file2);
869887
870 try ch.writeManifest();888 digest3 = ch.final();
871 }
872889
873 testing.expect(!mem.eql(u8, &digest1, &digest3));890 try ch.writeManifest();
891 }
892
893 testing.expect(!mem.eql(u8, &digest1, &digest3));
894 }
874895
875 try cwd.deleteTree(temp_manifest_dir);896 try cwd.deleteTree(temp_manifest_dir);
876 try cwd.deleteFile(temp_file1);897 try cwd.deleteFile(temp_file1);
src-self-hosted/Compilation.zig+19-32
...@@ -68,7 +68,9 @@ libunwind_static_lib: ?[]const u8 = null,...@@ -68,7 +68,9 @@ libunwind_static_lib: ?[]const u8 = null,
68/// and resolved before calling linker.flush().68/// and resolved before calling linker.flush().
69libc_static_lib: ?[]const u8 = null,69libc_static_lib: ?[]const u8 = null,
7070
71/// For example `Scrt1.o` and `libc.so.6`. These are populated after building libc from source,71glibc_so_files: ?glibc.BuiltSharedObjects = null,
72
73/// For example `Scrt1.o` and `libc_nonshared.a`. These are populated after building libc from source,
72/// The set of needed CRT (C runtime) files differs depending on the target and compilation settings.74/// The set of needed CRT (C runtime) files differs depending on the target and compilation settings.
73/// The key is the basename, and the value is the absolute path to the completed build artifact.75/// The key is the basename, and the value is the absolute path to the completed build artifact.
74crt_files: std.StringHashMapUnmanaged(CRTFile) = .{},76crt_files: std.StringHashMapUnmanaged(CRTFile) = .{},
...@@ -111,8 +113,8 @@ const WorkItem = union(enum) {...@@ -111,8 +113,8 @@ const WorkItem = union(enum) {
111113
112 /// one of the glibc static objects114 /// one of the glibc static objects
113 glibc_crt_file: glibc.CRTFile,115 glibc_crt_file: glibc.CRTFile,
114 /// one of the glibc shared objects116 /// all of the glibc shared objects
115 glibc_so: *const glibc.Lib,117 glibc_shared_objects,
116};118};
117119
118pub const CObject = struct {120pub const CObject = struct {
...@@ -272,6 +274,9 @@ pub const InitOptions = struct {...@@ -272,6 +274,9 @@ pub const InitOptions = struct {
272 version: ?std.builtin.Version = null,274 version: ?std.builtin.Version = null,
273 libc_installation: ?*const LibCInstallation = null,275 libc_installation: ?*const LibCInstallation = null,
274 machine_code_model: std.builtin.CodeModel = .default,276 machine_code_model: std.builtin.CodeModel = .default,
277 /// TODO Once self-hosted Zig is capable enough, we can remove this special-case
278 /// hack in favor of more general compilation options.
279 stage1_is_dummy_so: bool = false,
275};280};
276281
277pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {282pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
...@@ -421,6 +426,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -421,6 +426,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
421 cache.hash.addBytes(options.target.cpu.model.name);426 cache.hash.addBytes(options.target.cpu.model.name);
422 cache.hash.add(options.target.cpu.features.ints);427 cache.hash.add(options.target.cpu.features.ints);
423 cache.hash.add(options.target.os.tag);428 cache.hash.add(options.target.os.tag);
429 cache.hash.add(options.is_native_os);
424 cache.hash.add(options.target.abi);430 cache.hash.add(options.target.abi);
425 cache.hash.add(ofmt);431 cache.hash.add(ofmt);
426 cache.hash.add(pic);432 cache.hash.add(pic);
...@@ -446,22 +452,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -446,22 +452,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
446 hash.addOptionalBytes(root_pkg.root_src_directory.path);452 hash.addOptionalBytes(root_pkg.root_src_directory.path);
447 hash.add(valgrind);453 hash.add(valgrind);
448 hash.add(single_threaded);454 hash.add(single_threaded);
449 switch (options.target.os.getVersionRange()) {455 hash.add(options.target.os.getVersionRange());
450 .linux => |linux| {
451 hash.add(linux.range.min);
452 hash.add(linux.range.max);
453 hash.add(linux.glibc);
454 },
455 .windows => |windows| {
456 hash.add(windows.min);
457 hash.add(windows.max);
458 },
459 .semver => |semver| {
460 hash.add(semver.min);
461 hash.add(semver.max);
462 },
463 .none => {},
464 }
465456
466 const digest = hash.final();457 const digest = hash.final();
467 const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });458 const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
...@@ -660,7 +651,6 @@ pub fn destroy(self: *Compilation) void {...@@ -660,7 +651,6 @@ pub fn destroy(self: *Compilation) void {
660 {651 {
661 var it = self.crt_files.iterator();652 var it = self.crt_files.iterator();
662 while (it.next()) |entry| {653 while (it.next()) |entry| {
663 gpa.free(entry.key);
664 entry.value.deinit(gpa);654 entry.value.deinit(gpa);
665 }655 }
666 self.crt_files.deinit(gpa);656 self.crt_files.deinit(gpa);
...@@ -936,14 +926,15 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {...@@ -936,14 +926,15 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
936 },926 },
937 .glibc_crt_file => |crt_file| {927 .glibc_crt_file => |crt_file| {
938 glibc.buildCRTFile(self, crt_file) catch |err| {928 glibc.buildCRTFile(self, crt_file) catch |err| {
939 // This is a problem with the Zig installation. It's mostly OK to crash here,929 // TODO Expose this as a normal compile error rather than crashing here.
940 // but TODO because it would be even better if we could recover gracefully
941 // from temporary problems such as out-of-disk-space.
942 fatal("unable to build glibc CRT file: {}", .{@errorName(err)});930 fatal("unable to build glibc CRT file: {}", .{@errorName(err)});
943 };931 };
944 },932 },
945 .glibc_so => |glibc_lib| {933 .glibc_shared_objects => {
946 fatal("TODO build glibc shared object '{}.so.{}'", .{ glibc_lib.name, glibc_lib.sover });934 glibc.buildSharedObjects(self) catch |err| {
935 // TODO Expose this as a normal compile error rather than crashing here.
936 fatal("unable to build glibc shared objects: {}", .{@errorName(err)});
937 };
947 },938 },
948 };939 };
949}940}
...@@ -1587,17 +1578,13 @@ pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []cons...@@ -1587,17 +1578,13 @@ pub fn get_libc_crt_file(comp: *Compilation, arena: *Allocator, basename: []cons
1587}1578}
15881579
1589fn addBuildingGLibCWorkItems(comp: *Compilation) !void {1580fn addBuildingGLibCWorkItems(comp: *Compilation) !void {
1590 const static_file_work_items = [_]WorkItem{1581 try comp.work_queue.write(&[_]WorkItem{
1591 .{ .glibc_crt_file = .crti_o },1582 .{ .glibc_crt_file = .crti_o },
1592 .{ .glibc_crt_file = .crtn_o },1583 .{ .glibc_crt_file = .crtn_o },
1593 .{ .glibc_crt_file = .scrt1_o },1584 .{ .glibc_crt_file = .scrt1_o },
1594 .{ .glibc_crt_file = .libc_nonshared_a },1585 .{ .glibc_crt_file = .libc_nonshared_a },
1595 };1586 .{ .glibc_shared_objects = {} },
1596 try comp.work_queue.ensureUnusedCapacity(static_file_work_items.len + glibc.libs.len);1587 });
1597 comp.work_queue.writeAssumeCapacity(&static_file_work_items);
1598 for (glibc.libs) |*glibc_so| {
1599 comp.work_queue.writeItemAssumeCapacity(.{ .glibc_so = glibc_so });
1600 }
1601}1588}
16021589
1603fn wantBuildGLibCFromSource(comp: *Compilation) bool {1590fn wantBuildGLibCFromSource(comp: *Compilation) bool {
src-self-hosted/glibc.zig+256-17
...@@ -1,11 +1,15 @@...@@ -1,11 +1,15 @@
1const std = @import("std");1const std = @import("std");
2const Allocator = std.mem.Allocator;2const Allocator = std.mem.Allocator;
3const target_util = @import("target.zig");
4const mem = std.mem;3const mem = std.mem;
5const Compilation = @import("Compilation.zig");
6const path = std.fs.path;4const path = std.fs.path;
5const assert = std.debug.assert;
6
7const target_util = @import("target.zig");
8const Compilation = @import("Compilation.zig");
7const build_options = @import("build_options");9const build_options = @import("build_options");
8const trace = @import("tracy.zig").trace;10const trace = @import("tracy.zig").trace;
11const Cache = @import("Cache.zig");
12const Package = @import("Package.zig");
913
10pub const Lib = struct {14pub const Lib = struct {
11 name: []const u8,15 name: []const u8,
...@@ -83,14 +87,14 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -83,14 +87,14 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
83 };87 };
84 defer gpa.free(vers_txt_contents);88 defer gpa.free(vers_txt_contents);
8589
86 const fns_txt_contents = glibc_dir.readFileAlloc(gpa, "fns.txt", max_txt_size) catch |err| switch (err) {90 // Arena allocated because the result contains references to function names.
91 const fns_txt_contents = glibc_dir.readFileAlloc(arena, "fns.txt", max_txt_size) catch |err| switch (err) {
87 error.OutOfMemory => return error.OutOfMemory,92 error.OutOfMemory => return error.OutOfMemory,
88 else => {93 else => {
89 std.log.err("unable to read fns.txt: {}", .{@errorName(err)});94 std.log.err("unable to read fns.txt: {}", .{@errorName(err)});
90 return error.ZigInstallationCorrupt;95 return error.ZigInstallationCorrupt;
91 },96 },
92 };97 };
93 defer gpa.free(fns_txt_contents);
9498
95 const abi_txt_contents = glibc_dir.readFileAlloc(gpa, "abi.txt", max_txt_size) catch |err| switch (err) {99 const abi_txt_contents = glibc_dir.readFileAlloc(gpa, "abi.txt", max_txt_size) catch |err| switch (err) {
96 error.OutOfMemory => return error.OutOfMemory,100 error.OutOfMemory => return error.OutOfMemory,
...@@ -183,7 +187,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -183,7 +187,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
183 .os = .linux,187 .os = .linux,
184 .abi = abi_tag,188 .abi = abi_tag,
185 };189 };
186 try version_table.put(arena, triple, ver_list_base.ptr);190 try version_table.put(gpa, triple, ver_list_base.ptr);
187 }191 }
188 break :blk ver_list_base;192 break :blk ver_list_base;
189 };193 };
...@@ -250,7 +254,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {...@@ -250,7 +254,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
250 }254 }
251 const gpa = comp.gpa;255 const gpa = comp.gpa;
252 var arena_allocator = std.heap.ArenaAllocator.init(gpa);256 var arena_allocator = std.heap.ArenaAllocator.init(gpa);
253 errdefer arena_allocator.deinit();257 defer arena_allocator.deinit();
254 const arena = &arena_allocator.allocator;258 const arena = &arena_allocator.allocator;
255259
256 switch (crt_file) {260 switch (crt_file) {
...@@ -713,6 +717,252 @@ fn build_crt_file(...@@ -713,6 +717,252 @@ fn build_crt_file(
713 });717 });
714 defer sub_compilation.destroy();718 defer sub_compilation.destroy();
715719
720 try updateSubCompilation(sub_compilation);
721
722 try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
723 const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p|
724 try path.join(comp.gpa, &[_][]const u8{ p, basename })
725 else
726 try comp.gpa.dupe(u8, basename);
727
728 comp.crt_files.putAssumeCapacityNoClobber(basename, .{
729 .full_object_path = artifact_path,
730 .lock = sub_compilation.bin_file.toOwnedLock(),
731 });
732}
733
734pub const BuiltSharedObjects = struct {
735 lock: Cache.Lock,
736 dir_path: []u8,
737
738 pub fn deinit(self: *BuiltSharedObjects, gpa: *Allocator) void {
739 self.lock.release();
740 gpa.free(self.dir_path);
741 self.* = undefined;
742 }
743};
744
745const all_map_basename = "all.map";
746
747pub fn buildSharedObjects(comp: *Compilation) !void {
748 const tracy = trace(@src());
749 defer tracy.end();
750
751 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
752 defer arena_allocator.deinit();
753 const arena = &arena_allocator.allocator;
754
755 const target = comp.getTarget();
756 const target_version = target.os.version_range.linux.glibc;
757
758 // TODO use the global cache directory here
759 var cache_parent: Cache = .{
760 .gpa = comp.gpa,
761 .manifest_dir = comp.cache_parent.manifest_dir,
762 };
763 var cache = cache_parent.obtain();
764 defer cache.deinit();
765 cache.hash.addBytes(build_options.version);
766 cache.hash.addBytes(comp.zig_lib_directory.path orelse ".");
767 cache.hash.add(target.cpu.arch);
768 cache.hash.addBytes(target.cpu.model.name);
769 cache.hash.add(target.cpu.features.ints);
770 cache.hash.add(target.abi);
771 cache.hash.add(target_version);
772
773 const hit = try cache.hit();
774 const digest = cache.final();
775 const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest });
776 if (!hit) {
777 var o_directory: Compilation.Directory = .{
778 .handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}),
779 .path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }),
780 };
781 defer o_directory.handle.close();
782
783 const metadata = try loadMetaData(comp.gpa, comp.zig_lib_directory.handle);
784 defer metadata.destroy(comp.gpa);
785
786 const ver_list_base = metadata.version_table.get(.{
787 .arch = target.cpu.arch,
788 .os = target.os.tag,
789 .abi = target.abi,
790 }) orelse return error.GLibCUnavailableForThisTarget;
791 const target_ver_index = for (metadata.all_versions) |ver, i| {
792 switch (ver.order(target_version)) {
793 .eq => break i,
794 .lt => continue,
795 .gt => {
796 // TODO Expose via compile error mechanism instead of log.
797 std.log.warn("invalid target glibc version: {}", .{target_version});
798 return error.InvalidTargetGLibCVersion;
799 },
800 }
801 } else blk: {
802 const latest_index = metadata.all_versions.len - 1;
803 std.log.warn("zig cannot build new glibc version {}; providing instead {}", .{
804 target_version, metadata.all_versions[latest_index],
805 });
806 break :blk latest_index;
807 };
808 {
809 var map_contents = std.ArrayList(u8).init(arena);
810 for (metadata.all_versions) |ver| {
811 if (ver.patch == 0) {
812 try map_contents.writer().print("GLIBC_{d}.{d} {{ }};\n", .{ ver.major, ver.minor });
813 } else {
814 try map_contents.writer().print("GLIBC_{d}.{d}.{d} {{ }};\n", .{ ver.major, ver.minor, ver.patch });
815 }
816 }
817 try o_directory.handle.writeFile(all_map_basename, map_contents.items);
818 map_contents.deinit(); // The most recent allocation of an arena can be freed :)
819 }
820 var zig_body = std.ArrayList(u8).init(comp.gpa);
821 defer zig_body.deinit();
822 var zig_footer = std.ArrayList(u8).init(comp.gpa);
823 defer zig_footer.deinit();
824 for (libs) |*lib| {
825 zig_body.shrinkRetainingCapacity(0);
826 zig_footer.shrinkRetainingCapacity(0);
827
828 try zig_body.appendSlice(
829 \\comptime {
830 \\ asm (
831 \\
832 );
833 for (metadata.all_functions) |*libc_fn, fn_i| {
834 if (libc_fn.lib != lib) continue;
835
836 const ver_list = ver_list_base[fn_i];
837 // Pick the default symbol version:
838 // - If there are no versions, don't emit it
839 // - Take the greatest one <= than the target one
840 // - If none of them is <= than the
841 // specified one don't pick any default version
842 if (ver_list.len == 0) continue;
843 var chosen_def_ver_index: u8 = 255;
844 {
845 var ver_i: u8 = 0;
846 while (ver_i < ver_list.len) : (ver_i += 1) {
847 const ver_index = ver_list.versions[ver_i];
848 if ((chosen_def_ver_index == 255 or ver_index > chosen_def_ver_index) and
849 target_ver_index >= ver_index)
850 {
851 chosen_def_ver_index = ver_index;
852 }
853 }
854 }
855 {
856 var ver_i: u8 = 0;
857 while (ver_i < ver_list.len) : (ver_i += 1) {
858 const ver_index = ver_list.versions[ver_i];
859 const ver = metadata.all_versions[ver_index];
860 const sym_name = libc_fn.name;
861 const stub_name = if (ver.patch == 0)
862 try std.fmt.allocPrint(arena, "{s}_{d}_{d}", .{ sym_name, ver.major, ver.minor })
863 else
864 try std.fmt.allocPrint(arena, "{s}_{d}_{d}_{d}", .{ sym_name, ver.major, ver.minor, ver.patch });
865
866 try zig_footer.writer().print("export fn {s}() void {{}}\n", .{stub_name});
867
868 // Default symbol version definition vs normal symbol version definition
869 const want_two_ats = chosen_def_ver_index != 255 and ver_index == chosen_def_ver_index;
870 const at_sign_str = "@@"[0 .. @boolToInt(want_two_ats) + @as(usize, 1)];
871 if (ver.patch == 0) {
872 try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}\n", .{
873 stub_name, sym_name, at_sign_str, ver.major, ver.minor,
874 });
875 } else {
876 try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}.{d}\n", .{
877 stub_name, sym_name, at_sign_str, ver.major, ver.minor, ver.patch,
878 });
879 }
880 // Hide the stub to keep the symbol table clean
881 try zig_body.writer().print(" \\\\ .hidden {s}\n", .{stub_name});
882 }
883 }
884 }
885
886 try zig_body.appendSlice(
887 \\ );
888 \\}
889 \\
890 );
891 try zig_body.appendSlice(zig_footer.items);
892
893 var lib_name_buf: [32]u8 = undefined; // Larger than each of the names "c", "pthread", etc.
894 const zig_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.zig", .{lib.name}) catch unreachable;
895 try o_directory.handle.writeFile(zig_file_basename, zig_body.items);
896
897 try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, zig_file_basename, lib);
898 }
899 cache.writeManifest() catch |err| {
900 std.log.warn("glibc shared objects: failed to write cache manifest: {}", .{@errorName(err)});
901 };
902 }
903
904 assert(comp.glibc_so_files == null);
905 comp.glibc_so_files = BuiltSharedObjects{
906 .lock = cache.toOwnedLock(),
907 .dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }),
908 };
909}
910
911fn buildSharedLib(
912 comp: *Compilation,
913 arena: *Allocator,
914 zig_cache_directory: Compilation.Directory,
915 bin_directory: Compilation.Directory,
916 zig_file_basename: []const u8,
917 lib: *const Lib,
918) !void {
919 const tracy = trace(@src());
920 defer tracy.end();
921
922 const emit_bin = Compilation.EmitLoc{
923 .directory = bin_directory,
924 .basename = try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ lib.name, lib.sover }),
925 };
926 const version: std.builtin.Version = .{ .major = lib.sover, .minor = 0, .patch = 0 };
927 const ld_basename = path.basename(comp.getTarget().standardDynamicLinkerPath().get().?);
928 const override_soname = if (mem.eql(u8, lib.name, "ld")) ld_basename else null;
929 const map_file_path = try path.join(arena, &[_][]const u8{ bin_directory.path.?, all_map_basename });
930 // TODO we should be able to just give the open directory to Package
931 const root_pkg = try Package.create(comp.gpa, std.fs.cwd(), bin_directory.path.?, zig_file_basename);
932 defer root_pkg.destroy(comp.gpa);
933 const sub_compilation = try Compilation.create(comp.gpa, .{
934 .zig_cache_directory = zig_cache_directory,
935 .zig_lib_directory = comp.zig_lib_directory,
936 .target = comp.getTarget(),
937 .root_name = lib.name,
938 .root_pkg = null,
939 .output_mode = .Lib,
940 .link_mode = .Dynamic,
941 .rand = comp.rand,
942 .libc_installation = comp.bin_file.options.libc_installation,
943 .emit_bin = emit_bin,
944 .optimize_mode = comp.bin_file.options.optimize_mode,
945 .want_sanitize_c = false,
946 .want_stack_check = false,
947 .want_valgrind = false,
948 .emit_h = null,
949 .strip = comp.bin_file.options.strip,
950 .is_native_os = false,
951 .self_exe_path = comp.self_exe_path,
952 .debug_cc = comp.debug_cc,
953 .debug_link = comp.bin_file.options.debug_link,
954 .clang_passthrough_mode = comp.clang_passthrough_mode,
955 .version = version,
956 .stage1_is_dummy_so = true,
957 .version_script = map_file_path,
958 .override_soname = override_soname,
959 });
960 defer sub_compilation.destroy();
961
962 try updateSubCompilation(sub_compilation);
963}
964
965fn updateSubCompilation(sub_compilation: *Compilation) !void {
716 try sub_compilation.update();966 try sub_compilation.update();
717967
718 // Look for compilation errors in this sub_compilation968 // Look for compilation errors in this sub_compilation
...@@ -730,15 +980,4 @@ fn build_crt_file(...@@ -730,15 +980,4 @@ fn build_crt_file(
730 }980 }
731 return error.BuildingLibCObjectFailed;981 return error.BuildingLibCObjectFailed;
732 }982 }
733
734 try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
735 const artifact_path = if (sub_compilation.bin_file.options.directory.path) |p|
736 try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename })
737 else
738 try comp.gpa.dupe(u8, basename);
739
740 comp.crt_files.putAssumeCapacityNoClobber(basename, .{
741 .full_object_path = artifact_path,
742 .lock = sub_compilation.bin_file.toOwnedLock(),
743 });
744}983}