Commit 95941c4e7021588f224f0559c6f66de3a205b972 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-09-16 03:02:46-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-09-16 03:02:46-07:00
log	95941c4e7021588f224f0559c6f66de3a205b972
tree	230b70c9dc2504be1c7525ade95dbc5b4af97435
parent	da0fde59b6ffde5d505f664aa27b728aa2b1cc2a

stage2: building glibc shared objects

 * caching system: use 16 bytes siphash final(), there was a bug in the
   std lib that wasn't catching undefined values for 18 bytes. fixed in
   master branch.
 * fix caching system unit test logic to not cause error.TextBusy on windows
 * port the logic from stage1 for building glibc shared objects
 * add is_native_os to the base cache hash
 * fix incorrectly freeing crt_files key (which is always a reference to
   global static constant data)
 * fix 2 use-after-free in loading glibc metadata
 * fix memory leak in buildCRTFile (errdefer instead of defer on arena)

5 files changed, 425 insertions(+), 165 deletions(-)

BRANCH_TODO
ci
- azure
  - windows_mingw_script
src-self-hosted

BRANCH_TODO+10-1

...	...	@@ -1,4 +1,7 @@
1	1	* glibc .so files
	2	- stage1 C++ code integration
	3	- ok file
	4	* use hex for cache hash file paths
2	5	* support rpaths in ELF linker code
3	6	* build & link against compiler-rt
4	7	* build & link againstn freestanding libc
...	...	@@ -21,7 +24,6 @@
21	24	* implement proper parsing of LLD stderr/stdout and exposing compile errors
22	25	* implement proper parsing of clang stderr/stdout and exposing compile errors
23	26	* implement proper compile errors for failing to build glibc crt files and shared libs
24		* skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
25	27	* self-host link.cpp and building libcs (#4313 and #4314). using the `zig cc` command will set a flag indicating a preference for the llvm backend, which will include linking with LLD. At least for now. If zig's self-hosted linker ever gets on par with the likes of ld and lld, we can make it always be used even for zig cc.
26	28	* improve the stage2 tests to support testing with LLVM extensions enabled
27	29	* multi-thread building C objects
...	...	@@ -47,3 +49,10 @@
47	49	* libc_installation.zig: make it look for msvc only if msvc abi is chosen
48	50	* switch the default C ABI for windows to be mingw-w64
49	51	* port windows_sdk.cpp to zig
	52	* change glibc log errors to normal exposed compile errors
	53	* update Package to use Compilation.Directory in create()
	54	- skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
	55	(maybe make it an explicit option and have main.zig disable it)
	56	- make it possible for Package to not openDir and reference already existing resources.
	57	* rename src/ to src/stage1/
	58	* rename src-self-hosted/ to src/

ci/azure/windows_mingw_script+4

...	...	@@ -18,4 +18,8 @@ cmake .. -G 'MSYS Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo $CMAKEFLAGS -DCMA
18	18
19	19	make -j$(nproc) install
20	20
	21	# I saw a failure due to `git diff` being > 400 KB instead of empty as expected so this is to debug it.
	22	git status
	23	git diff \| head -n100
	24
21	25	./zig build test-behavior -Dskip-non-native -Dskip-release

src-self-hosted/Cache.zig+136-115

...	...	@@ -26,9 +26,9 @@ pub fn obtain(cache: *const Cache) CacheHash {
26	26
27	27	pub const base64_encoder = fs.base64_encoder;
28	28	pub const base64_decoder = fs.base64_decoder;
29		/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
30		/// We round up to 18 to avoid the `==` padding after base64 encoding.
31		pub const BIN_DIGEST_LEN = 18;
	29	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
	30	/// Currently we use SipHash and so this value must be 16 not any higher.
	31	pub const BIN_DIGEST_LEN = 16;
32	32	pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
33	33
34	34	const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
...	...	@@ -87,14 +87,29 @@ pub const HashHelper = struct {
87	87	hh.add(x.major);
88	88	hh.add(x.minor);
89	89	hh.add(x.patch);
90		return;
91	90	},
92		else => {},
93		}
94
95		switch (@typeInfo(@TypeOf(x))) {
96		.Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
97		else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
	91	std.Target.Os.TaggedVersionRange => {
	92	switch (x) {
	93	.linux => \|linux\| {
	94	hh.add(linux.range.min);
	95	hh.add(linux.range.max);
	96	hh.add(linux.glibc);
	97	},
	98	.windows => \|windows\| {
	99	hh.add(windows.min);
	100	hh.add(windows.max);
	101	},
	102	.semver => \|semver\| {
	103	hh.add(semver.min);
	104	hh.add(semver.max);
	105	},
	106	.none => {},
	107	}
	108	},
	109	else => switch (@typeInfo(@TypeOf(x))) {
	110	.Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
	111	else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
	112	},
98	113	}
99	114	}
100	115
...	...	@@ -613,44 +628,46 @@ test "cache file and then recall it" {
613	628	var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
614	629	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
615	630
616		var cache = Cache{
617		.gpa = testing.allocator,
618		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
619		};
620		defer cache.manifest_dir.close();
621
622	631	{
623		var ch = cache.obtain();
624		defer ch.deinit();
	632	var cache = Cache{
	633	.gpa = testing.allocator,
	634	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	635	};
	636	defer cache.manifest_dir.close();
625	637
626		ch.hash.add(true);
627		ch.hash.add(@as(u16, 1234));
628		ch.hash.addBytes("1234");
629		_ = try ch.addFile(temp_file, null);
	638	{
	639	var ch = cache.obtain();
	640	defer ch.deinit();
630	641
631		// There should be nothing in the cache
632		testing.expectEqual(false, try ch.hit());
	642	ch.hash.add(true);
	643	ch.hash.add(@as(u16, 1234));
	644	ch.hash.addBytes("1234");
	645	_ = try ch.addFile(temp_file, null);
633	646
634		digest1 = ch.final();
635		try ch.writeManifest();
636		}
637		{
638		var ch = cache.obtain();
639		defer ch.deinit();
	647	// There should be nothing in the cache
	648	testing.expectEqual(false, try ch.hit());
640	649
641		ch.hash.add(true);
642		ch.hash.add(@as(u16, 1234));
643		ch.hash.addBytes("1234");
644		_ = try ch.addFile(temp_file, null);
	650	digest1 = ch.final();
	651	try ch.writeManifest();
	652	}
	653	{
	654	var ch = cache.obtain();
	655	defer ch.deinit();
645	656
646		// Cache hit! We just "built" the same file
647		testing.expect(try ch.hit());
648		digest2 = ch.final();
	657	ch.hash.add(true);
	658	ch.hash.add(@as(u16, 1234));
	659	ch.hash.addBytes("1234");
	660	_ = try ch.addFile(temp_file, null);
649	661
650		try ch.writeManifest();
651		}
	662	// Cache hit! We just "built" the same file
	663	testing.expect(try ch.hit());
	664	digest2 = ch.final();
652	665
653		testing.expectEqual(digest1, digest2);
	666	try ch.writeManifest();
	667	}
	668
	669	testing.expectEqual(digest1, digest2);
	670	}
654	671
655	672	try cwd.deleteTree(temp_manifest_dir);
656	673	try cwd.deleteFile(temp_file);
...	...	@@ -693,50 +710,52 @@ test "check that changing a file makes cache fail" {
693	710	var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
694	711	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
695	712
696		var cache = Cache{
697		.gpa = testing.allocator,
698		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
699		};
700		defer cache.manifest_dir.close();
701
702	713	{
703		var ch = cache.obtain();
704		defer ch.deinit();
	714	var cache = Cache{
	715	.gpa = testing.allocator,
	716	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	717	};
	718	defer cache.manifest_dir.close();
705	719
706		ch.hash.addBytes("1234");
707		const temp_file_idx = try ch.addFile(temp_file, 100);
	720	{
	721	var ch = cache.obtain();
	722	defer ch.deinit();
708	723
709		// There should be nothing in the cache
710		testing.expectEqual(false, try ch.hit());
	724	ch.hash.addBytes("1234");
	725	const temp_file_idx = try ch.addFile(temp_file, 100);
711	726
712		testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
	727	// There should be nothing in the cache
	728	testing.expectEqual(false, try ch.hit());
713	729
714		digest1 = ch.final();
	730	testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
715	731
716		try ch.writeManifest();
717		}
	732	digest1 = ch.final();
718	733
719		try cwd.writeFile(temp_file, updated_temp_file_contents);
	734	try ch.writeManifest();
	735	}
720	736
721		{
722		var ch = cache.obtain();
723		defer ch.deinit();
	737	try cwd.writeFile(temp_file, updated_temp_file_contents);
724	738
725		ch.hash.addBytes("1234");
726		const temp_file_idx = try ch.addFile(temp_file, 100);
	739	{
	740	var ch = cache.obtain();
	741	defer ch.deinit();
727	742
728		// A file that we depend on has been updated, so the cache should not contain an entry for it
729		testing.expectEqual(false, try ch.hit());
	743	ch.hash.addBytes("1234");
	744	const temp_file_idx = try ch.addFile(temp_file, 100);
730	745
731		// The cache system does not keep the contents of re-hashed input files.
732		testing.expect(ch.files.items[temp_file_idx].contents == null);
	746	// A file that we depend on has been updated, so the cache should not contain an entry for it
	747	testing.expectEqual(false, try ch.hit());
733	748
734		digest2 = ch.final();
	749	// The cache system does not keep the contents of re-hashed input files.
	750	testing.expect(ch.files.items[temp_file_idx].contents == null);
735	751
736		try ch.writeManifest();
737		}
	752	digest2 = ch.final();
	753
	754	try ch.writeManifest();
	755	}
738	756
739		testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
	757	testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
	758	}
740	759
741	760	try cwd.deleteTree(temp_manifest_dir);
742	761	try cwd.deleteTree(temp_file);
...	...	@@ -749,7 +768,7 @@ test "no file inputs" {
749	768	}
750	769	const cwd = fs.cwd();
751	770	const temp_manifest_dir = "no_file_inputs_manifest_dir";
752		defer cwd.deleteTree(temp_manifest_dir) catch unreachable;
	771	defer cwd.deleteTree(temp_manifest_dir) catch {};
753	772
754	773	var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
755	774	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
...	...	@@ -810,67 +829,69 @@ test "CacheHashes with files added after initial hash work" {
810	829	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
811	830	var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
812	831
813		var cache = Cache{
814		.gpa = testing.allocator,
815		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
816		};
817		defer cache.manifest_dir.close();
818
819	832	{
820		var ch = cache.obtain();
821		defer ch.deinit();
	833	var cache = Cache{
	834	.gpa = testing.allocator,
	835	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	836	};
	837	defer cache.manifest_dir.close();
822	838
823		ch.hash.addBytes("1234");
824		_ = try ch.addFile(temp_file1, null);
	839	{
	840	var ch = cache.obtain();
	841	defer ch.deinit();
825	842
826		// There should be nothing in the cache
827		testing.expectEqual(false, try ch.hit());
	843	ch.hash.addBytes("1234");
	844	_ = try ch.addFile(temp_file1, null);
828	845
829		_ = try ch.addFilePost(temp_file2);
	846	// There should be nothing in the cache
	847	testing.expectEqual(false, try ch.hit());
830	848
831		digest1 = ch.final();
832		try ch.writeManifest();
833		}
834		{
835		var ch = cache.obtain();
836		defer ch.deinit();
	849	_ = try ch.addFilePost(temp_file2);
837	850
838		ch.hash.addBytes("1234");
839		_ = try ch.addFile(temp_file1, null);
	851	digest1 = ch.final();
	852	try ch.writeManifest();
	853	}
	854	{
	855	var ch = cache.obtain();
	856	defer ch.deinit();
840	857
841		testing.expect(try ch.hit());
842		digest2 = ch.final();
	858	ch.hash.addBytes("1234");
	859	_ = try ch.addFile(temp_file1, null);
843	860
844		try ch.writeManifest();
845		}
846		testing.expect(mem.eql(u8, &digest1, &digest2));
	861	testing.expect(try ch.hit());
	862	digest2 = ch.final();
847	863
848		// Modify the file added after initial hash
849		const ts2 = std.time.nanoTimestamp();
850		try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
	864	try ch.writeManifest();
	865	}
	866	testing.expect(mem.eql(u8, &digest1, &digest2));
851	867
852		while (isProblematicTimestamp(ts2)) {
853		std.time.sleep(1);
854		}
	868	// Modify the file added after initial hash
	869	const ts2 = std.time.nanoTimestamp();
	870	try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
855	871
856		{
857		var ch = cache.obtain();
858		defer ch.deinit();
	872	while (isProblematicTimestamp(ts2)) {
	873	std.time.sleep(1);
	874	}
859	875
860		ch.hash.addBytes("1234");
861		_ = try ch.addFile(temp_file1, null);
	876	{
	877	var ch = cache.obtain();
	878	defer ch.deinit();
862	879
863		// A file that we depend on has been updated, so the cache should not contain an entry for it
864		testing.expectEqual(false, try ch.hit());
	880	ch.hash.addBytes("1234");
	881	_ = try ch.addFile(temp_file1, null);
865	882
866		_ = try ch.addFilePost(temp_file2);
	883	// A file that we depend on has been updated, so the cache should not contain an entry for it
	884	testing.expectEqual(false, try ch.hit());
867	885
868		digest3 = ch.final();
	886	_ = try ch.addFilePost(temp_file2);
869	887
870		try ch.writeManifest();
871		}
	888	digest3 = ch.final();
872	889
873		testing.expect(!mem.eql(u8, &digest1, &digest3));
	890	try ch.writeManifest();
	891	}
	892
	893	testing.expect(!mem.eql(u8, &digest1, &digest3));
	894	}
874	895
875	896	try cwd.deleteTree(temp_manifest_dir);
876	897	try cwd.deleteFile(temp_file1);

src-self-hosted/Compilation.zig+19-32

...	...	@@ -68,7 +68,9 @@ libunwind_static_lib: ?[]const u8 = null,
68	68	/// and resolved before calling linker.flush().
69	69	libc_static_lib: ?[]const u8 = null,
70	70
71		/// For example `Scrt1.o` and `libc.so.6`. These are populated after building libc from source,
	71	glibc_so_files: ?glibc.BuiltSharedObjects = null,
	72
	73	/// For example `Scrt1.o` and `libc_nonshared.a`. These are populated after building libc from source,
72	74	/// The set of needed CRT (C runtime) files differs depending on the target and compilation settings.
73	75	/// The key is the basename, and the value is the absolute path to the completed build artifact.
74	76	crt_files: std.StringHashMapUnmanaged(CRTFile) = .{},
...	...	@@ -111,8 +113,8 @@ const WorkItem = union(enum) {
111	113
112	114	/// one of the glibc static objects
113	115	glibc_crt_file: glibc.CRTFile,
114		/// one of the glibc shared objects
115		glibc_so: *const glibc.Lib,
	116	/// all of the glibc shared objects
	117	glibc_shared_objects,
116	118	};
117	119
118	120	pub const CObject = struct {
...	...	@@ -272,6 +274,9 @@ pub const InitOptions = struct {
272	274	version: ?std.builtin.Version = null,
273	275	libc_installation: ?*const LibCInstallation = null,
274	276	machine_code_model: std.builtin.CodeModel = .default,
	277	/// TODO Once self-hosted Zig is capable enough, we can remove this special-case
	278	/// hack in favor of more general compilation options.
	279	stage1_is_dummy_so: bool = false,
275	280	};
276	281
277	282	pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
...	...	@@ -421,6 +426,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
421	426	cache.hash.addBytes(options.target.cpu.model.name);
422	427	cache.hash.add(options.target.cpu.features.ints);
423	428	cache.hash.add(options.target.os.tag);
	429	cache.hash.add(options.is_native_os);
424	430	cache.hash.add(options.target.abi);
425	431	cache.hash.add(ofmt);
426	432	cache.hash.add(pic);
...	...	@@ -446,22 +452,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
446	452	hash.addOptionalBytes(root_pkg.root_src_directory.path);
447	453	hash.add(valgrind);
448	454	hash.add(single_threaded);
449		switch (options.target.os.getVersionRange()) {
450		.linux => \|linux\| {
451		hash.add(linux.range.min);
452		hash.add(linux.range.max);
453		hash.add(linux.glibc);
454		},
455		.windows => \|windows\| {
456		hash.add(windows.min);
457		hash.add(windows.max);
458		},
459		.semver => \|semver\| {
460		hash.add(semver.min);
461		hash.add(semver.max);
462		},
463		.none => {},
464		}
	455	hash.add(options.target.os.getVersionRange());
465	456
466	457	const digest = hash.final();
467	458	const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
...	...	@@ -660,7 +651,6 @@ pub fn destroy(self: *Compilation) void {
660	651	{
661	652	var it = self.crt_files.iterator();
662	653	while (it.next()) \|entry\| {
663		gpa.free(entry.key);
664	654	entry.value.deinit(gpa);
665	655	}
666	656	self.crt_files.deinit(gpa);
...	...	@@ -936,14 +926,15 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
936	926	},
937	927	.glibc_crt_file => \|crt_file\| {
938	928	glibc.buildCRTFile(self, crt_file) catch \|err\| {
939		// This is a problem with the Zig installation. It's mostly OK to crash here,
940		// but TODO because it would be even better if we could recover gracefully
941		// from temporary problems such as out-of-disk-space.
	929	// TODO Expose this as a normal compile error rather than crashing here.
942	930	fatal("unable to build glibc CRT file: {}", .{@errorName(err)});
943	931	};
944	932	},
945		.glibc_so => \|glibc_lib\| {
946		fatal("TODO build glibc shared object '{}.so.{}'", .{ glibc_lib.name, glibc_lib.sover });
	933	.glibc_shared_objects => {
	934	glibc.buildSharedObjects(self) catch \|err\| {
	935	// TODO Expose this as a normal compile error rather than crashing here.
	936	fatal("unable to build glibc shared objects: {}", .{@errorName(err)});
	937	};
947	938	},
948	939	};
949	940	}
...	...	@@ -1587,17 +1578,13 @@ pub fn get_libc_crt_file(comp: Compilation, arena: Allocator, basename: []cons
1587	1578	}
1588	1579
1589	1580	fn addBuildingGLibCWorkItems(comp: *Compilation) !void {
1590		const static_file_work_items = [_]WorkItem{
	1581	try comp.work_queue.write(&[_]WorkItem{
1591	1582	.{ .glibc_crt_file = .crti_o },
1592	1583	.{ .glibc_crt_file = .crtn_o },
1593	1584	.{ .glibc_crt_file = .scrt1_o },
1594	1585	.{ .glibc_crt_file = .libc_nonshared_a },
1595		};
1596		try comp.work_queue.ensureUnusedCapacity(static_file_work_items.len + glibc.libs.len);
1597		comp.work_queue.writeAssumeCapacity(&static_file_work_items);
1598		for (glibc.libs) \|*glibc_so\| {
1599		comp.work_queue.writeItemAssumeCapacity(.{ .glibc_so = glibc_so });
1600		}
	1586	.{ .glibc_shared_objects = {} },
	1587	});
1601	1588	}
1602	1589
1603	1590	fn wantBuildGLibCFromSource(comp: *Compilation) bool {

src-self-hosted/glibc.zig+256-17

...	...	@@ -1,11 +1,15 @@
1	1	const std = @import("std");
2	2	const Allocator = std.mem.Allocator;
3		const target_util = @import("target.zig");
4	3	const mem = std.mem;
5		const Compilation = @import("Compilation.zig");
6	4	const path = std.fs.path;
	5	const assert = std.debug.assert;
	6
	7	const target_util = @import("target.zig");
	8	const Compilation = @import("Compilation.zig");
7	9	const build_options = @import("build_options");
8	10	const trace = @import("tracy.zig").trace;
	11	const Cache = @import("Cache.zig");
	12	const Package = @import("Package.zig");
9	13
10	14	pub const Lib = struct {
11	15	name: []const u8,
...	...	@@ -83,14 +87,14 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
83	87	};
84	88	defer gpa.free(vers_txt_contents);
85	89
86		const fns_txt_contents = glibc_dir.readFileAlloc(gpa, "fns.txt", max_txt_size) catch \|err\| switch (err) {
	90	// Arena allocated because the result contains references to function names.
	91	const fns_txt_contents = glibc_dir.readFileAlloc(arena, "fns.txt", max_txt_size) catch \|err\| switch (err) {
87	92	error.OutOfMemory => return error.OutOfMemory,
88	93	else => {
89	94	std.log.err("unable to read fns.txt: {}", .{@errorName(err)});
90	95	return error.ZigInstallationCorrupt;
91	96	},
92	97	};
93		defer gpa.free(fns_txt_contents);
94	98
95	99	const abi_txt_contents = glibc_dir.readFileAlloc(gpa, "abi.txt", max_txt_size) catch \|err\| switch (err) {
96	100	error.OutOfMemory => return error.OutOfMemory,
...	...	@@ -183,7 +187,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
183	187	.os = .linux,
184	188	.abi = abi_tag,
185	189	};
186		try version_table.put(arena, triple, ver_list_base.ptr);
	190	try version_table.put(gpa, triple, ver_list_base.ptr);
187	191	}
188	192	break :blk ver_list_base;
189	193	};
...	...	@@ -250,7 +254,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
250	254	}
251	255	const gpa = comp.gpa;
252	256	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
253		errdefer arena_allocator.deinit();
	257	defer arena_allocator.deinit();
254	258	const arena = &arena_allocator.allocator;
255	259
256	260	switch (crt_file) {
...	...	@@ -713,6 +717,252 @@ fn build_crt_file(
713	717	});
714	718	defer sub_compilation.destroy();
715	719
	720	try updateSubCompilation(sub_compilation);
	721
	722	try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
	723	const artifact_path = if (sub_compilation.bin_file.options.directory.path) \|p\|
	724	try path.join(comp.gpa, &[_][]const u8{ p, basename })
	725	else
	726	try comp.gpa.dupe(u8, basename);
	727
	728	comp.crt_files.putAssumeCapacityNoClobber(basename, .{
	729	.full_object_path = artifact_path,
	730	.lock = sub_compilation.bin_file.toOwnedLock(),
	731	});
	732	}
	733
	734	pub const BuiltSharedObjects = struct {
	735	lock: Cache.Lock,
	736	dir_path: []u8,
	737
	738	pub fn deinit(self: BuiltSharedObjects, gpa: Allocator) void {
	739	self.lock.release();
	740	gpa.free(self.dir_path);
	741	self.* = undefined;
	742	}
	743	};
	744
	745	const all_map_basename = "all.map";
	746
	747	pub fn buildSharedObjects(comp: *Compilation) !void {
	748	const tracy = trace(@src());
	749	defer tracy.end();
	750
	751	var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
	752	defer arena_allocator.deinit();
	753	const arena = &arena_allocator.allocator;
	754
	755	const target = comp.getTarget();
	756	const target_version = target.os.version_range.linux.glibc;
	757
	758	// TODO use the global cache directory here
	759	var cache_parent: Cache = .{
	760	.gpa = comp.gpa,
	761	.manifest_dir = comp.cache_parent.manifest_dir,
	762	};
	763	var cache = cache_parent.obtain();
	764	defer cache.deinit();
	765	cache.hash.addBytes(build_options.version);
	766	cache.hash.addBytes(comp.zig_lib_directory.path orelse ".");
	767	cache.hash.add(target.cpu.arch);
	768	cache.hash.addBytes(target.cpu.model.name);
	769	cache.hash.add(target.cpu.features.ints);
	770	cache.hash.add(target.abi);
	771	cache.hash.add(target_version);
	772
	773	const hit = try cache.hit();
	774	const digest = cache.final();
	775	const o_sub_path = try path.join(arena, &[_][]const u8{ "o", &digest });
	776	if (!hit) {
	777	var o_directory: Compilation.Directory = .{
	778	.handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}),
	779	.path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }),
	780	};
	781	defer o_directory.handle.close();
	782
	783	const metadata = try loadMetaData(comp.gpa, comp.zig_lib_directory.handle);
	784	defer metadata.destroy(comp.gpa);
	785
	786	const ver_list_base = metadata.version_table.get(.{
	787	.arch = target.cpu.arch,
	788	.os = target.os.tag,
	789	.abi = target.abi,
	790	}) orelse return error.GLibCUnavailableForThisTarget;
	791	const target_ver_index = for (metadata.all_versions) \|ver, i\| {
	792	switch (ver.order(target_version)) {
	793	.eq => break i,
	794	.lt => continue,
	795	.gt => {
	796	// TODO Expose via compile error mechanism instead of log.
	797	std.log.warn("invalid target glibc version: {}", .{target_version});
	798	return error.InvalidTargetGLibCVersion;
	799	},
	800	}
	801	} else blk: {
	802	const latest_index = metadata.all_versions.len - 1;
	803	std.log.warn("zig cannot build new glibc version {}; providing instead {}", .{
	804	target_version, metadata.all_versions[latest_index],
	805	});
	806	break :blk latest_index;
	807	};
	808	{
	809	var map_contents = std.ArrayList(u8).init(arena);
	810	for (metadata.all_versions) \|ver\| {
	811	if (ver.patch == 0) {
	812	try map_contents.writer().print("GLIBC_{d}.{d} {{ }};\n", .{ ver.major, ver.minor });
	813	} else {
	814	try map_contents.writer().print("GLIBC_{d}.{d}.{d} {{ }};\n", .{ ver.major, ver.minor, ver.patch });
	815	}
	816	}
	817	try o_directory.handle.writeFile(all_map_basename, map_contents.items);
	818	map_contents.deinit(); // The most recent allocation of an arena can be freed :)
	819	}
	820	var zig_body = std.ArrayList(u8).init(comp.gpa);
	821	defer zig_body.deinit();
	822	var zig_footer = std.ArrayList(u8).init(comp.gpa);
	823	defer zig_footer.deinit();
	824	for (libs) \|*lib\| {
	825	zig_body.shrinkRetainingCapacity(0);
	826	zig_footer.shrinkRetainingCapacity(0);
	827
	828	try zig_body.appendSlice(
	829	\\comptime {
	830	\\ asm (
	831	\\
	832	);
	833	for (metadata.all_functions) \|*libc_fn, fn_i\| {
	834	if (libc_fn.lib != lib) continue;
	835
	836	const ver_list = ver_list_base[fn_i];
	837	// Pick the default symbol version:
	838	// - If there are no versions, don't emit it
	839	// - Take the greatest one <= than the target one
	840	// - If none of them is <= than the
	841	// specified one don't pick any default version
	842	if (ver_list.len == 0) continue;
	843	var chosen_def_ver_index: u8 = 255;
	844	{
	845	var ver_i: u8 = 0;
	846	while (ver_i < ver_list.len) : (ver_i += 1) {
	847	const ver_index = ver_list.versions[ver_i];
	848	if ((chosen_def_ver_index == 255 or ver_index > chosen_def_ver_index) and
	849	target_ver_index >= ver_index)
	850	{
	851	chosen_def_ver_index = ver_index;
	852	}
	853	}
	854	}
	855	{
	856	var ver_i: u8 = 0;
	857	while (ver_i < ver_list.len) : (ver_i += 1) {
	858	const ver_index = ver_list.versions[ver_i];
	859	const ver = metadata.all_versions[ver_index];
	860	const sym_name = libc_fn.name;
	861	const stub_name = if (ver.patch == 0)
	862	try std.fmt.allocPrint(arena, "{s}_{d}_{d}", .{ sym_name, ver.major, ver.minor })
	863	else
	864	try std.fmt.allocPrint(arena, "{s}_{d}_{d}_{d}", .{ sym_name, ver.major, ver.minor, ver.patch });
	865
	866	try zig_footer.writer().print("export fn {s}() void {{}}\n", .{stub_name});
	867
	868	// Default symbol version definition vs normal symbol version definition
	869	const want_two_ats = chosen_def_ver_index != 255 and ver_index == chosen_def_ver_index;
	870	const at_sign_str = "@@"[0 .. @boolToInt(want_two_ats) + @as(usize, 1)];
	871	if (ver.patch == 0) {
	872	try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}\n", .{
	873	stub_name, sym_name, at_sign_str, ver.major, ver.minor,
	874	});
	875	} else {
	876	try zig_body.writer().print(" \\\\ .symver {s}, {s}{s}GLIBC_{d}.{d}.{d}\n", .{
	877	stub_name, sym_name, at_sign_str, ver.major, ver.minor, ver.patch,
	878	});
	879	}
	880	// Hide the stub to keep the symbol table clean
	881	try zig_body.writer().print(" \\\\ .hidden {s}\n", .{stub_name});
	882	}
	883	}
	884	}
	885
	886	try zig_body.appendSlice(
	887	\\ );
	888	\\}
	889	\\
	890	);
	891	try zig_body.appendSlice(zig_footer.items);
	892
	893	var lib_name_buf: [32]u8 = undefined; // Larger than each of the names "c", "pthread", etc.
	894	const zig_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.zig", .{lib.name}) catch unreachable;
	895	try o_directory.handle.writeFile(zig_file_basename, zig_body.items);
	896
	897	try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, zig_file_basename, lib);
	898	}
	899	cache.writeManifest() catch \|err\| {
	900	std.log.warn("glibc shared objects: failed to write cache manifest: {}", .{@errorName(err)});
	901	};
	902	}
	903
	904	assert(comp.glibc_so_files == null);
	905	comp.glibc_so_files = BuiltSharedObjects{
	906	.lock = cache.toOwnedLock(),
	907	.dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }),
	908	};
	909	}
	910
	911	fn buildSharedLib(
	912	comp: *Compilation,
	913	arena: *Allocator,
	914	zig_cache_directory: Compilation.Directory,
	915	bin_directory: Compilation.Directory,
	916	zig_file_basename: []const u8,
	917	lib: *const Lib,
	918	) !void {
	919	const tracy = trace(@src());
	920	defer tracy.end();
	921
	922	const emit_bin = Compilation.EmitLoc{
	923	.directory = bin_directory,
	924	.basename = try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ lib.name, lib.sover }),
	925	};
	926	const version: std.builtin.Version = .{ .major = lib.sover, .minor = 0, .patch = 0 };
	927	const ld_basename = path.basename(comp.getTarget().standardDynamicLinkerPath().get().?);
	928	const override_soname = if (mem.eql(u8, lib.name, "ld")) ld_basename else null;
	929	const map_file_path = try path.join(arena, &[_][]const u8{ bin_directory.path.?, all_map_basename });
	930	// TODO we should be able to just give the open directory to Package
	931	const root_pkg = try Package.create(comp.gpa, std.fs.cwd(), bin_directory.path.?, zig_file_basename);
	932	defer root_pkg.destroy(comp.gpa);
	933	const sub_compilation = try Compilation.create(comp.gpa, .{
	934	.zig_cache_directory = zig_cache_directory,
	935	.zig_lib_directory = comp.zig_lib_directory,
	936	.target = comp.getTarget(),
	937	.root_name = lib.name,
	938	.root_pkg = null,
	939	.output_mode = .Lib,
	940	.link_mode = .Dynamic,
	941	.rand = comp.rand,
	942	.libc_installation = comp.bin_file.options.libc_installation,
	943	.emit_bin = emit_bin,
	944	.optimize_mode = comp.bin_file.options.optimize_mode,
	945	.want_sanitize_c = false,
	946	.want_stack_check = false,
	947	.want_valgrind = false,
	948	.emit_h = null,
	949	.strip = comp.bin_file.options.strip,
	950	.is_native_os = false,
	951	.self_exe_path = comp.self_exe_path,
	952	.debug_cc = comp.debug_cc,
	953	.debug_link = comp.bin_file.options.debug_link,
	954	.clang_passthrough_mode = comp.clang_passthrough_mode,
	955	.version = version,
	956	.stage1_is_dummy_so = true,
	957	.version_script = map_file_path,
	958	.override_soname = override_soname,
	959	});
	960	defer sub_compilation.destroy();
	961
	962	try updateSubCompilation(sub_compilation);
	963	}
	964
	965	fn updateSubCompilation(sub_compilation: *Compilation) !void {
716	966	try sub_compilation.update();
717	967
718	968	// Look for compilation errors in this sub_compilation
...	...	@@ -730,15 +980,4 @@ fn build_crt_file(
730	980	}
731	981	return error.BuildingLibCObjectFailed;
732	982	}
733
734		try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1);
735		const artifact_path = if (sub_compilation.bin_file.options.directory.path) \|p\|
736		try std.fs.path.join(comp.gpa, &[_][]const u8{ p, basename })
737		else
738		try comp.gpa.dupe(u8, basename);
739
740		comp.crt_files.putAssumeCapacityNoClobber(basename, .{
741		.full_object_path = artifact_path,
742		.lock = sub_compilation.bin_file.toOwnedLock(),
743		});
744	983	}