authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-06 20:20:06+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-12 13:55:40+01:00
log808c15dd397f995d9bdf43664ee5644b39c9c863
tree68ca882637676a120b3acac7ca4708d7271abb44
parent89ba8859704486d526a75434f18d5a25ff89d57b
signaturelock-open Commit is signed but in an unrecognized format.

link.Lld: remove dead caching logic

It turns out that LLD caching hasn't been in use for a while. On master, it is currently only enabled when you compile via the build system, passing `-fincremental`, using LLD (and so LLVM if there's a ZCU). That case never happens, because `-fincremental` is only useful when you're using a backend *other* than the LLVM backend. My previous commits accidentally re-enabled this logic in some cases, exposing bugs; that ultimately led to this realisation. So, let's just delete that logic -- less LLVM-related cruft to maintain.

1 files changed, 0 insertions(+), 371 deletions(-)

src/link/Lld.zig-371
......@@ -312,59 +312,8 @@ fn linkAsArchive(lld: *Lld, arena: Allocator) !void {
312312 // insight as to what's going on here you can read that function body which is more
313313 // well-commented.
314314
315 const id_symlink_basename = "llvm-ar.id";
316
317 var man: Cache.Manifest = undefined;
318 defer if (!lld.disable_caching) man.deinit();
319
320315 const link_inputs = comp.link_inputs;
321316
322 var digest: [Cache.hex_digest_len]u8 = undefined;
323
324 if (!lld.disable_caching) {
325 man = comp.cache_parent.obtain();
326
327 // We are about to obtain this lock, so here we give other processes a chance first.
328 base.releaseLock();
329
330 try link.hashInputs(&man, link_inputs);
331
332 for (comp.c_object_table.keys()) |key| {
333 _ = try man.addFilePath(key.status.success.object_path, null);
334 }
335 for (comp.win32_resource_table.keys()) |key| {
336 _ = try man.addFile(key.status.success.res_path, null);
337 }
338 try man.addOptionalFile(zcu_obj_path);
339 try man.addOptionalFilePath(compiler_rt_path);
340 try man.addOptionalFilePath(ubsan_rt_path);
341
342 // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
343 _ = try man.hit();
344 digest = man.final();
345
346 var prev_digest_buf: [digest.len]u8 = undefined;
347 const prev_digest: []u8 = Cache.readSmallFile(
348 directory.handle,
349 id_symlink_basename,
350 &prev_digest_buf,
351 ) catch |err| b: {
352 log.debug("archive new_digest={s} readFile error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });
353 break :b prev_digest_buf[0..0];
354 };
355 if (mem.eql(u8, prev_digest, &digest)) {
356 log.debug("archive digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
357 base.lock = man.toOwnedLock();
358 return;
359 }
360
361 // We are about to change the output file to be different, so we invalidate the build hash now.
362 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
363 error.FileNotFound => {},
364 else => |e| return e,
365 };
366 }
367
368317 var object_files: std.ArrayListUnmanaged([*:0]const u8) = .empty;
369318
370319 try object_files.ensureUnusedCapacity(arena, link_inputs.len);
......@@ -408,20 +357,6 @@ fn linkAsArchive(lld: *Lld, arena: Allocator) !void {
408357 },
409358 );
410359 if (bad) return error.UnableToWriteArchive;
411
412 if (!lld.disable_caching) {
413 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
414 log.warn("failed to save archive hash digest file: {s}", .{@errorName(err)});
415 };
416
417 if (man.have_exclusive_lock) {
418 man.writeManifest() catch |err| {
419 log.warn("failed to write cache manifest when archiving: {s}", .{@errorName(err)});
420 };
421 }
422
423 base.lock = man.toOwnedLock();
424 }
425360}
426361
427362fn coffLink(lld: *Lld, arena: Allocator) !void {
......@@ -457,90 +392,6 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
457392 .named => |name| name,
458393 };
459394
460 // See link/Elf.zig for comments on how this mechanism works.
461 const id_symlink_basename = "lld.id";
462
463 var man: Cache.Manifest = undefined;
464 defer if (!lld.disable_caching) man.deinit();
465
466 var digest: [Cache.hex_digest_len]u8 = undefined;
467
468 if (!lld.disable_caching) {
469 man = comp.cache_parent.obtain();
470 base.releaseLock();
471
472 comptime assert(Compilation.link_hash_implementation_version == 14);
473
474 try link.hashInputs(&man, comp.link_inputs);
475 for (comp.c_object_table.keys()) |key| {
476 _ = try man.addFilePath(key.status.success.object_path, null);
477 }
478 for (comp.win32_resource_table.keys()) |key| {
479 _ = try man.addFile(key.status.success.res_path, null);
480 }
481 try man.addOptionalFile(module_obj_path);
482 man.hash.addOptionalBytes(entry_name);
483 man.hash.add(base.stack_size);
484 man.hash.add(coff.image_base);
485 man.hash.add(base.build_id);
486 {
487 // TODO remove this, libraries must instead be resolved by the frontend.
488 for (coff.lib_directories) |lib_directory| man.hash.addOptionalBytes(lib_directory.path);
489 }
490 man.hash.add(comp.skip_linker_dependencies);
491 if (comp.config.link_libc) {
492 man.hash.add(comp.libc_installation != null);
493 if (comp.libc_installation) |libc_installation| {
494 man.hash.addBytes(libc_installation.crt_dir.?);
495 if (target.abi == .msvc or target.abi == .itanium) {
496 man.hash.addBytes(libc_installation.msvc_lib_dir.?);
497 man.hash.addBytes(libc_installation.kernel32_lib_dir.?);
498 }
499 }
500 }
501 man.hash.addListOfBytes(comp.windows_libs.keys());
502 man.hash.addListOfBytes(comp.force_undefined_symbols.keys());
503 man.hash.addOptional(coff.subsystem);
504 man.hash.add(comp.config.is_test);
505 man.hash.add(coff.tsaware);
506 man.hash.add(coff.nxcompat);
507 man.hash.add(coff.dynamicbase);
508 man.hash.add(base.allow_shlib_undefined);
509 // strip does not need to go into the linker hash because it is part of the hash namespace
510 man.hash.add(coff.major_subsystem_version);
511 man.hash.add(coff.minor_subsystem_version);
512 man.hash.add(coff.repro);
513 man.hash.addOptional(comp.version);
514 try man.addOptionalFile(coff.module_definition_file);
515
516 // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
517 _ = try man.hit();
518 digest = man.final();
519 var prev_digest_buf: [digest.len]u8 = undefined;
520 const prev_digest: []u8 = Cache.readSmallFile(
521 directory.handle,
522 id_symlink_basename,
523 &prev_digest_buf,
524 ) catch |err| blk: {
525 log.debug("COFF LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });
526 // Handle this as a cache miss.
527 break :blk prev_digest_buf[0..0];
528 };
529 if (mem.eql(u8, prev_digest, &digest)) {
530 log.debug("COFF LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
531 // Hot diggity dog! The output binary is already there.
532 base.lock = man.toOwnedLock();
533 return;
534 }
535 log.debug("COFF LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) });
536
537 // We are about to change the output file to be different, so we invalidate the build hash now.
538 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
539 error.FileNotFound => {},
540 else => |e| return e,
541 };
542 }
543
544395 if (comp.config.output_mode == .Obj) {
545396 // LLD's COFF driver does not support the equivalent of `-r` so we do a simple file copy
546397 // here. TODO: think carefully about how we can avoid this redundant operation when doing
......@@ -935,21 +786,6 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
935786
936787 try spawnLld(comp, arena, argv.items);
937788 }
938
939 if (!lld.disable_caching) {
940 // Update the file with the digest. If it fails we can continue; it only
941 // means that the next invocation will have an unnecessary cache miss.
942 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
943 log.warn("failed to save linking hash digest file: {s}", .{@errorName(err)});
944 };
945 // Again failure here only means an unnecessary cache miss.
946 man.writeManifest() catch |err| {
947 log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
948 };
949 // We hang on to this lock so that the output file path can be used without
950 // other processes clobbering it.
951 base.lock = man.toOwnedLock();
952 }
953789}
954790fn findLib(arena: Allocator, name: []const u8, lib_directories: []const Cache.Directory) !?[]const u8 {
955791 for (lib_directories) |lib_directory| {
......@@ -1001,118 +837,6 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
1001837 break :blk null;
1002838 };
1003839
1004 // Here we want to determine whether we can save time by not invoking LLD when the
1005 // output is unchanged. None of the linker options or the object files that are being
1006 // linked are in the hash that namespaces the directory we are outputting to. Therefore,
1007 // we must hash those now, and the resulting digest will form the "id" of the linking
1008 // job we are about to perform.
1009 // After a successful link, we store the id in the metadata of a symlink named "lld.id" in
1010 // the artifact directory. So, now, we check if this symlink exists, and if it matches
1011 // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD.
1012 const id_symlink_basename = "lld.id";
1013
1014 var man: std.Build.Cache.Manifest = undefined;
1015 defer if (!lld.disable_caching) man.deinit();
1016
1017 var digest: [std.Build.Cache.hex_digest_len]u8 = undefined;
1018
1019 if (!lld.disable_caching) {
1020 man = comp.cache_parent.obtain();
1021
1022 // We are about to obtain this lock, so here we give other processes a chance first.
1023 base.releaseLock();
1024
1025 comptime assert(Compilation.link_hash_implementation_version == 14);
1026
1027 try man.addOptionalFile(elf.linker_script);
1028 try man.addOptionalFile(elf.version_script);
1029 man.hash.add(elf.allow_undefined_version);
1030 man.hash.addOptional(elf.enable_new_dtags);
1031 try link.hashInputs(&man, comp.link_inputs);
1032 for (comp.c_object_table.keys()) |key| {
1033 _ = try man.addFilePath(key.status.success.object_path, null);
1034 }
1035 try man.addOptionalFile(module_obj_path);
1036 try man.addOptionalFilePath(compiler_rt_path);
1037 try man.addOptionalFilePath(ubsan_rt_path);
1038 try man.addOptionalFilePath(if (comp.tsan_lib) |l| l.full_object_path else null);
1039 try man.addOptionalFilePath(if (comp.fuzzer_lib) |l| l.full_object_path else null);
1040
1041 // We can skip hashing libc and libc++ components that we are in charge of building from Zig
1042 // installation sources because they are always a product of the compiler version + target information.
1043 man.hash.addOptionalBytes(elf.entry_name);
1044 man.hash.add(elf.image_base);
1045 man.hash.add(base.gc_sections);
1046 man.hash.addOptional(elf.sort_section);
1047 man.hash.add(comp.link_eh_frame_hdr);
1048 man.hash.add(elf.emit_relocs);
1049 man.hash.add(comp.config.rdynamic);
1050 man.hash.addListOfBytes(elf.rpath_list);
1051 if (output_mode == .Exe) {
1052 man.hash.add(base.stack_size);
1053 }
1054 man.hash.add(base.build_id);
1055 man.hash.addListOfBytes(elf.symbol_wrap_set);
1056 man.hash.add(comp.skip_linker_dependencies);
1057 man.hash.add(elf.z_nodelete);
1058 man.hash.add(elf.z_notext);
1059 man.hash.add(elf.z_defs);
1060 man.hash.add(elf.z_origin);
1061 man.hash.add(elf.z_nocopyreloc);
1062 man.hash.add(elf.z_now);
1063 man.hash.add(elf.z_relro);
1064 man.hash.add(elf.z_common_page_size orelse 0);
1065 man.hash.add(elf.z_max_page_size orelse 0);
1066 man.hash.add(elf.hash_style);
1067 // strip does not need to go into the linker hash because it is part of the hash namespace
1068 if (comp.config.link_libc) {
1069 man.hash.add(comp.libc_installation != null);
1070 if (comp.libc_installation) |libc_installation| {
1071 man.hash.addBytes(libc_installation.crt_dir.?);
1072 }
1073 }
1074 if (have_dynamic_linker) {
1075 man.hash.addOptionalBytes(target.dynamic_linker.get());
1076 }
1077 man.hash.addOptionalBytes(elf.soname);
1078 man.hash.addOptional(comp.version);
1079 man.hash.addListOfBytes(comp.force_undefined_symbols.keys());
1080 man.hash.add(base.allow_shlib_undefined);
1081 man.hash.add(elf.bind_global_refs_locally);
1082 man.hash.add(elf.compress_debug_sections);
1083 man.hash.add(comp.config.any_sanitize_thread);
1084 man.hash.add(comp.config.any_fuzz);
1085 man.hash.addOptionalBytes(comp.sysroot);
1086
1087 // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
1088 _ = try man.hit();
1089 digest = man.final();
1090
1091 var prev_digest_buf: [digest.len]u8 = undefined;
1092 const prev_digest: []u8 = std.Build.Cache.readSmallFile(
1093 directory.handle,
1094 id_symlink_basename,
1095 &prev_digest_buf,
1096 ) catch |err| blk: {
1097 log.debug("ELF LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });
1098 // Handle this as a cache miss.
1099 break :blk prev_digest_buf[0..0];
1100 };
1101 if (mem.eql(u8, prev_digest, &digest)) {
1102 log.debug("ELF LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
1103 // Hot diggity dog! The output binary is already there.
1104 base.lock = man.toOwnedLock();
1105 return;
1106 }
1107 log.debug("ELF LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) });
1108
1109 // We are about to change the output file to be different, so we invalidate the build hash now.
1110 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
1111 error.FileNotFound => {},
1112 else => |e| return e,
1113 };
1114 }
1115
1116840 // Due to a deficiency in LLD, we need to special-case BPF to a simple file
1117841 // copy when generating relocatables. Normally, we would expect `lld -r` to work.
1118842 // However, because LLD wants to resolve BPF relocations which it shouldn't, it fails
......@@ -1570,21 +1294,6 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
15701294
15711295 try spawnLld(comp, arena, argv.items);
15721296 }
1573
1574 if (!lld.disable_caching) {
1575 // Update the file with the digest. If it fails we can continue; it only
1576 // means that the next invocation will have an unnecessary cache miss.
1577 std.Build.Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
1578 log.warn("failed to save linking hash digest file: {s}", .{@errorName(err)});
1579 };
1580 // Again failure here only means an unnecessary cache miss.
1581 man.writeManifest() catch |err| {
1582 log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
1583 };
1584 // We hang on to this lock so that the output file path can be used without
1585 // other processes clobbering it.
1586 base.lock = man.toOwnedLock();
1587 }
15881297}
15891298fn getLDMOption(target: std.Target) ?[]const u8 {
15901299 // This should only return emulations understood by LLD's parseEmulation().
......@@ -1700,71 +1409,6 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void {
17001409 break :blk null;
17011410 };
17021411
1703 const id_symlink_basename = "lld.id";
1704
1705 var man: Cache.Manifest = undefined;
1706 defer if (!lld.disable_caching) man.deinit();
1707
1708 var digest: [Cache.hex_digest_len]u8 = undefined;
1709
1710 if (!lld.disable_caching) {
1711 man = comp.cache_parent.obtain();
1712
1713 // We are about to obtain this lock, so here we give other processes a chance first.
1714 base.releaseLock();
1715
1716 comptime assert(Compilation.link_hash_implementation_version == 14);
1717
1718 try link.hashInputs(&man, comp.link_inputs);
1719 for (comp.c_object_table.keys()) |key| {
1720 _ = try man.addFilePath(key.status.success.object_path, null);
1721 }
1722 try man.addOptionalFile(module_obj_path);
1723 try man.addOptionalFilePath(compiler_rt_path);
1724 try man.addOptionalFilePath(ubsan_rt_path);
1725 man.hash.addOptionalBytes(wasm.entry_name);
1726 man.hash.add(base.stack_size);
1727 man.hash.add(base.build_id);
1728 man.hash.add(import_memory);
1729 man.hash.add(export_memory);
1730 man.hash.add(wasm.import_table);
1731 man.hash.add(wasm.export_table);
1732 man.hash.addOptional(wasm.initial_memory);
1733 man.hash.addOptional(wasm.max_memory);
1734 man.hash.add(shared_memory);
1735 man.hash.addOptional(wasm.global_base);
1736 man.hash.addListOfBytes(wasm.export_symbol_names);
1737 // strip does not need to go into the linker hash because it is part of the hash namespace
1738
1739 // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
1740 _ = try man.hit();
1741 digest = man.final();
1742
1743 var prev_digest_buf: [digest.len]u8 = undefined;
1744 const prev_digest: []u8 = Cache.readSmallFile(
1745 directory.handle,
1746 id_symlink_basename,
1747 &prev_digest_buf,
1748 ) catch |err| blk: {
1749 log.debug("WASM LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });
1750 // Handle this as a cache miss.
1751 break :blk prev_digest_buf[0..0];
1752 };
1753 if (mem.eql(u8, prev_digest, &digest)) {
1754 log.debug("WASM LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
1755 // Hot diggity dog! The output binary is already there.
1756 base.lock = man.toOwnedLock();
1757 return;
1758 }
1759 log.debug("WASM LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) });
1760
1761 // We are about to change the output file to be different, so we invalidate the build hash now.
1762 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
1763 error.FileNotFound => {},
1764 else => |e| return e,
1765 };
1766 }
1767
17681412 if (is_obj) {
17691413 // LLD's WASM driver does not support the equivalent of `-r` so we do a simple file copy
17701414 // here. TODO: think carefully about how we can avoid this redundant operation when doing
......@@ -1998,21 +1642,6 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void {
19981642 };
19991643 }
20001644 }
2001
2002 if (!lld.disable_caching) {
2003 // Update the file with the digest. If it fails we can continue; it only
2004 // means that the next invocation will have an unnecessary cache miss.
2005 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
2006 log.warn("failed to save linking hash digest symlink: {s}", .{@errorName(err)});
2007 };
2008 // Again failure here only means an unnecessary cache miss.
2009 man.writeManifest() catch |err| {
2010 log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
2011 };
2012 // We hang on to this lock so that the output file path can be used without
2013 // other processes clobbering it.
2014 base.lock = man.toOwnedLock();
2015 }
20161645}
20171646
20181647fn spawnLld(