authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-11-20 17:23:44-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-11-20 17:23:44-07:00
log4e5a88b28882eda156a46ecc7f70887a7bc0b49b
tree1ccdc54fdb432ff72692e9f10f4f0640ef6869cd
parenta699d678b2c81dcf333a4f4bb84676836849a618

stage2: default dynamic libraries to be linked as needed

After this change, the default for dynamic libraries (`-l` or `--library`) is to only link them if they end up being actually used. With the Zig CLI, the new options `-needed-l` or `--needed-library` can be used to force link against a dynamic library. With `zig cc`, this behavior can be overridden with `-Wl,--no-as-needed` (and restored with `-Wl,--as-needed`). Closes #10164

7 files changed, 102 insertions(+), 47 deletions(-)

src/Cache.zig-8
......@@ -90,14 +90,6 @@ pub const HashHelper = struct {
9090 for (list_of_bytes) |bytes| hh.addBytes(bytes);
9191 }
9292
93 pub fn addStringSet(hh: *HashHelper, hm: std.StringArrayHashMapUnmanaged(void)) void {
94 const keys = hm.keys();
95 hh.add(keys.len);
96 for (keys) |key| {
97 hh.addBytes(key);
98 }
99 }
100
10193 /// Convert the input value into bytes and record it as a dependency of the process being cached.
10294 pub fn add(hh: *HashHelper, x: anytype) void {
10395 switch (@TypeOf(x)) {
src/Compilation.zig+11-8
......@@ -627,6 +627,8 @@ pub const ClangPreprocessorMode = enum {
627627 stdout,
628628};
629629
630pub const SystemLib = link.SystemLib;
631
630632pub const InitOptions = struct {
631633 zig_lib_directory: Directory,
632634 local_cache_directory: Directory,
......@@ -672,7 +674,8 @@ pub const InitOptions = struct {
672674 link_objects: []const []const u8 = &[0][]const u8{},
673675 framework_dirs: []const []const u8 = &[0][]const u8{},
674676 frameworks: []const []const u8 = &[0][]const u8{},
675 system_libs: []const []const u8 = &[0][]const u8{},
677 system_lib_names: []const []const u8 = &.{},
678 system_lib_infos: []const SystemLib = &.{},
676679 /// These correspond to the WASI libc emulated subcomponents including:
677680 /// * process clocks
678681 /// * getpid
......@@ -935,7 +938,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
935938 if (options.link_objects.len != 0 or
936939 options.c_source_files.len != 0 or
937940 options.frameworks.len != 0 or
938 options.system_libs.len != 0 or
941 options.system_lib_names.len != 0 or
939942 options.link_libc or options.link_libcpp or
940943 link_eh_frame_hdr or
941944 options.link_emit_relocs or
......@@ -1003,7 +1006,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
10031006 break :dl true;
10041007 }
10051008 const any_dyn_libs: bool = x: {
1006 if (options.system_libs.len != 0)
1009 if (options.system_lib_names.len != 0)
10071010 break :x true;
10081011 for (options.link_objects) |obj| {
10091012 switch (classifyFileExt(obj)) {
......@@ -1050,7 +1053,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
10501053 options.target,
10511054 options.is_native_abi,
10521055 link_libc,
1053 options.system_libs.len != 0 or options.frameworks.len != 0,
1056 options.system_lib_names.len != 0 or options.frameworks.len != 0,
10541057 options.libc_installation,
10551058 );
10561059
......@@ -1372,11 +1375,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
13721375 };
13731376 };
13741377
1375 var system_libs: std.StringArrayHashMapUnmanaged(void) = .{};
1378 var system_libs: std.StringArrayHashMapUnmanaged(SystemLib) = .{};
13761379 errdefer system_libs.deinit(gpa);
1377 try system_libs.ensureTotalCapacity(gpa, options.system_libs.len);
1378 for (options.system_libs) |lib_name| {
1379 system_libs.putAssumeCapacity(lib_name, {});
1380 try system_libs.ensureTotalCapacity(gpa, options.system_lib_names.len);
1381 for (options.system_lib_names) |lib_name, i| {
1382 system_libs.putAssumeCapacity(lib_name, options.system_lib_infos[i]);
13801383 }
13811384
13821385 const bin_file = try link.File.openPath(gpa, .{
src/link.zig+17-1
......@@ -18,6 +18,22 @@ const wasi_libc = @import("wasi_libc.zig");
1818const Air = @import("Air.zig");
1919const Liveness = @import("Liveness.zig");
2020
21pub const SystemLib = struct {
22 needed: bool = false,
23};
24
25pub fn hashAddSystemLibs(
26 hh: *Cache.HashHelper,
27 hm: std.StringArrayHashMapUnmanaged(SystemLib),
28) void {
29 const keys = hm.keys();
30 hh.add(keys.len);
31 hh.addListOfBytes(keys);
32 for (hm.values()) |value| {
33 hh.add(value.needed);
34 }
35}
36
2137pub const producer_string = if (builtin.is_test) "zig test" else "zig " ++ build_options.version;
2238
2339pub const Emit = struct {
......@@ -121,7 +137,7 @@ pub const Options = struct {
121137 objects: []const []const u8,
122138 framework_dirs: []const []const u8,
123139 frameworks: []const []const u8,
124 system_libs: std.StringArrayHashMapUnmanaged(void),
140 system_libs: std.StringArrayHashMapUnmanaged(SystemLib),
125141 wasi_emulated_libs: []const wasi_libc.CRTFile,
126142 lib_dirs: []const []const u8,
127143 rpath_list: []const []const u8,
src/link/Coff.zig+1-1
......@@ -940,7 +940,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
940940 }
941941 }
942942 }
943 man.hash.addStringSet(self.base.options.system_libs);
943 link.hashAddSystemLibs(&man.hash, self.base.options.system_libs);
944944 man.hash.addOptional(self.base.options.subsystem);
945945 man.hash.add(self.base.options.is_test);
946946 man.hash.add(self.base.options.tsaware);
src/link/Elf.zig+35-7
......@@ -1345,7 +1345,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
13451345 }
13461346 man.hash.addOptionalBytes(self.base.options.soname);
13471347 man.hash.addOptional(self.base.options.version);
1348 man.hash.addStringSet(self.base.options.system_libs);
1348 link.hashAddSystemLibs(&man.hash, self.base.options.system_libs);
13491349 man.hash.add(allow_shlib_undefined);
13501350 man.hash.add(self.base.options.bind_global_refs_locally);
13511351 man.hash.add(self.base.options.tsan);
......@@ -1550,7 +1550,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
15501550 for (self.base.options.system_libs.keys()) |link_lib| {
15511551 test_path.shrinkRetainingCapacity(0);
15521552 const sep = fs.path.sep_str;
1553 try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ lib_dir_path, link_lib });
1553 try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{
1554 lib_dir_path, link_lib,
1555 });
15541556 fs.cwd().access(test_path.items, .{}) catch |err| switch (err) {
15551557 error.FileNotFound => continue,
15561558 else => |e| return e,
......@@ -1627,16 +1629,42 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
16271629 // Shared libraries.
16281630 if (is_exe_or_dyn_lib) {
16291631 const system_libs = self.base.options.system_libs.keys();
1630 try argv.ensureUnusedCapacity(system_libs.len);
1631 for (system_libs) |link_lib| {
1632 // By this time, we depend on these libs being dynamically linked libraries and not static libraries
1633 // (the check for that needs to be earlier), but they could be full paths to .so files, in which
1634 // case we want to avoid prepending "-l".
1632 const system_libs_values = self.base.options.system_libs.values();
1633
1634 // Worst-case, we need an --as-needed argument for every lib, as well
1635 // as one before and one after.
1636 try argv.ensureUnusedCapacity(system_libs.len * 2 + 2);
1637 argv.appendAssumeCapacity("--as-needed");
1638 var as_needed = true;
1639
1640 for (system_libs) |link_lib, i| {
1641 const lib_as_needed = !system_libs_values[i].needed;
1642 switch ((@as(u2, @boolToInt(lib_as_needed)) << 1) | @boolToInt(as_needed)) {
1643 0b00, 0b11 => {},
1644 0b01 => {
1645 argv.appendAssumeCapacity("--no-as-needed");
1646 as_needed = false;
1647 },
1648 0b10 => {
1649 argv.appendAssumeCapacity("--as-needed");
1650 as_needed = true;
1651 },
1652 }
1653
1654 // By this time, we depend on these libs being dynamically linked
1655 // libraries and not static libraries (the check for that needs to be earlier),
1656 // but they could be full paths to .so files, in which case we
1657 // want to avoid prepending "-l".
16351658 const ext = Compilation.classifyFileExt(link_lib);
16361659 const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib});
16371660 argv.appendAssumeCapacity(arg);
16381661 }
16391662
1663 if (!as_needed) {
1664 argv.appendAssumeCapacity("--as-needed");
1665 as_needed = true;
1666 }
1667
16401668 // libc++ dep
16411669 if (self.base.options.link_libcpp) {
16421670 try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
src/link/MachO.zig+1-1
......@@ -471,7 +471,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void {
471471 if (is_dyn_lib) {
472472 man.hash.addOptional(self.base.options.version);
473473 }
474 man.hash.addStringSet(self.base.options.system_libs);
474 link.hashAddSystemLibs(&man.hash, self.base.options.system_libs);
475475 man.hash.addOptionalBytes(self.base.options.sysroot);
476476
477477 // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
src/main.zig+37-21
......@@ -387,7 +387,9 @@ const usage_build_generic =
387387 \\ -ffunction-sections Places each function in a separate section
388388 \\
389389 \\Link Options:
390 \\ -l[lib], --library [lib] Link against system library
390 \\ -l[lib], --library [lib] Link against system library (only if actually used)
391 \\ -needed-l[lib], Link against system library (even if unused)
392 \\ --needed-library [lib]
391393 \\ -L[d], --library-directory [d] Add a directory to the library search path
392394 \\ -T[script], --script [script] Use a custom linker script
393395 \\ --version-script [path] Provide a version .map file
......@@ -655,7 +657,7 @@ fn buildOutputType(
655657 var wasi_exec_model: ?std.builtin.WasiExecModel = null;
656658 var enable_link_snapshots: bool = false;
657659
658 var system_libs = std.ArrayList([]const u8).init(gpa);
660 var system_libs = std.StringArrayHashMap(Compilation.SystemLib).init(gpa);
659661 defer system_libs.deinit();
660662
661663 var wasi_emulated_libs = std.ArrayList(wasi_libc.CRTFile).init(gpa);
......@@ -860,10 +862,14 @@ fn buildOutputType(
860862 version_script = args[i];
861863 } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) {
862864 if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
863 // We don't know whether this library is part of libc or libc++ until we resolve the target.
864 // So we simply append to the list for now.
865 // We don't know whether this library is part of libc or libc++ until
866 // we resolve the target, so we simply append to the list for now.
865867 i += 1;
866 try system_libs.append(args[i]);
868 try system_libs.put(args[i], .{ .needed = false });
869 } else if (mem.eql(u8, arg, "--needed-library") or mem.eql(u8, arg, "-needed-l")) {
870 if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
871 i += 1;
872 try system_libs.put(args[i], .{ .needed = true });
867873 } else if (mem.eql(u8, arg, "-D") or
868874 mem.eql(u8, arg, "-isystem") or
869875 mem.eql(u8, arg, "-I") or
......@@ -1164,9 +1170,11 @@ fn buildOutputType(
11641170 } else if (mem.startsWith(u8, arg, "-F")) {
11651171 try framework_dirs.append(arg[2..]);
11661172 } else if (mem.startsWith(u8, arg, "-l")) {
1167 // We don't know whether this library is part of libc or libc++ until we resolve the target.
1168 // So we simply append to the list for now.
1169 try system_libs.append(arg[2..]);
1173 // We don't know whether this library is part of libc or libc++ until
1174 // we resolve the target, so we simply append to the list for now.
1175 try system_libs.put(arg["-l".len..], .{ .needed = false });
1176 } else if (mem.startsWith(u8, arg, "-needed-l")) {
1177 try system_libs.put(arg["-needed-l".len..], .{ .needed = true });
11701178 } else if (mem.startsWith(u8, arg, "-D") or
11711179 mem.startsWith(u8, arg, "-I"))
11721180 {
......@@ -1230,6 +1238,7 @@ fn buildOutputType(
12301238 var linker_args = std.ArrayList([]const u8).init(arena);
12311239 var it = ClangArgIterator.init(arena, all_args);
12321240 var emit_llvm = false;
1241 var needed = false;
12331242 while (it.has_next) {
12341243 it.next() catch |err| {
12351244 fatal("unable to parse command line parameters: {s}", .{@errorName(err)});
......@@ -1262,9 +1271,9 @@ fn buildOutputType(
12621271 },
12631272 .l => {
12641273 // -l
1265 // We don't know whether this library is part of libc or libc++ until we resolve the target.
1266 // So we simply append to the list for now.
1267 try system_libs.append(it.only_arg);
1274 // We don't know whether this library is part of libc or libc++ until
1275 // we resolve the target, so we simply append to the list for now.
1276 try system_libs.put(it.only_arg, .{ .needed = needed });
12681277 },
12691278 .ignore => {},
12701279 .driver_punt => {
......@@ -1302,8 +1311,13 @@ fn buildOutputType(
13021311 continue;
13031312 }
13041313 }
1305
1306 try linker_args.append(linker_arg);
1314 if (mem.eql(u8, linker_arg, "--as-needed")) {
1315 needed = false;
1316 } else if (mem.eql(u8, linker_arg, "--no-as-needed")) {
1317 needed = true;
1318 } else {
1319 try linker_args.append(linker_arg);
1320 }
13071321 }
13081322 },
13091323 .optimize => {
......@@ -1725,21 +1739,22 @@ fn buildOutputType(
17251739 // existence via flags instead.
17261740 {
17271741 var i: usize = 0;
1728 while (i < system_libs.items.len) {
1729 const lib_name = system_libs.items[i];
1742 while (i < system_libs.count()) {
1743 const lib_name = system_libs.keys()[i];
1744
17301745 if (target_util.is_libc_lib_name(target_info.target, lib_name)) {
17311746 link_libc = true;
1732 _ = system_libs.orderedRemove(i);
1747 _ = system_libs.orderedRemove(lib_name);
17331748 continue;
17341749 }
17351750 if (target_util.is_libcpp_lib_name(target_info.target, lib_name)) {
17361751 link_libcpp = true;
1737 _ = system_libs.orderedRemove(i);
1752 _ = system_libs.orderedRemove(lib_name);
17381753 continue;
17391754 }
17401755 if (mem.eql(u8, lib_name, "unwind")) {
17411756 link_libunwind = true;
1742 _ = system_libs.orderedRemove(i);
1757 _ = system_libs.orderedRemove(lib_name);
17431758 continue;
17441759 }
17451760 if (std.fs.path.isAbsolute(lib_name)) {
......@@ -1748,7 +1763,7 @@ fn buildOutputType(
17481763 if (target_info.target.os.tag == .wasi) {
17491764 if (wasi_libc.getEmulatedLibCRTFile(lib_name)) |crt_file| {
17501765 try wasi_emulated_libs.append(crt_file);
1751 _ = system_libs.orderedRemove(i);
1766 _ = system_libs.orderedRemove(lib_name);
17521767 continue;
17531768 }
17541769 }
......@@ -1777,7 +1792,7 @@ fn buildOutputType(
17771792 const is_darwin_on_darwin = (comptime builtin.target.isDarwin()) and cross_target.isDarwin();
17781793
17791794 if (sysroot == null and (cross_target.isNativeOs() or is_darwin_on_darwin) and
1780 (system_libs.items.len != 0 or want_native_include_dirs))
1795 (system_libs.count() != 0 or want_native_include_dirs))
17811796 {
17821797 const paths = std.zig.system.NativePaths.detect(arena, target_info) catch |err| {
17831798 fatal("unable to detect native system paths: {s}", .{@errorName(err)});
......@@ -2144,7 +2159,8 @@ fn buildOutputType(
21442159 .link_objects = link_objects.items,
21452160 .framework_dirs = framework_dirs.items,
21462161 .frameworks = frameworks.items,
2147 .system_libs = system_libs.items,
2162 .system_lib_names = system_libs.keys(),
2163 .system_lib_infos = system_libs.values(),
21482164 .wasi_emulated_libs = wasi_emulated_libs.items,
21492165 .link_libc = link_libc,
21502166 .link_libcpp = link_libcpp,