authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-03-27 06:49:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-03-27 20:06:32+01:00
log4b85f05f060ee3365bd86e1885bbefb9e685c81b
tree12a8ceeca354e72fdcaa2eb1c83e471434ce1935
parentb490412cd28e4bc76de9dfe39ac1d99435a86814

Fetch: fix recompression not preserving symlinks

closes #31660

1 files changed, 50 insertions(+), 28 deletions(-)

src/Package/Fetch.zig+50-28
......@@ -383,7 +383,7 @@ pub const JobQueue = struct {
383383 // However, if we want Zig users to be able to share cached package
384384 // data with each other via peer-to-peer protocols, we benefit greatly
385385 // from the data being identical on everyone's computers.
386 var scanned_files: std.ArrayList([]const u8) = .empty;
386 var scanned_files: std.ArrayList(ScannedFile) = .empty;
387387 defer scanned_files.deinit(gpa);
388388
389389 var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true });
......@@ -394,22 +394,25 @@ pub const JobQueue = struct {
394394 defer walker.deinit();
395395
396396 while (try walker.next(io)) |entry| {
397 switch (entry.kind) {
397 const symlink = switch (entry.kind) {
398398 .directory => continue,
399 .file, .sym_link => {},
400 else => {
401 return error.IllegalFileType;
402 },
403 }
399 .file => false,
400 .sym_link => true,
401 else => return error.IllegalFileType,
402 };
404403 const entry_path = try arena.dupe(u8, entry.path);
405404 // If necessary, normalize path separators to POSIX-style since the tar format requires that.
406 if (comptime std.fs.path.sep != std.fs.path.sep_posix) {
405 if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) {
407406 std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix);
408407 }
409 try scanned_files.append(gpa, entry_path);
408 try scanned_files.append(gpa, .{
409 .ptr = entry_path.ptr,
410 .len = @intCast(entry_path.len),
411 .symlink = symlink,
412 });
410413 }
411414
412 std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp);
415 std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp);
413416 }
414417
415418 prog_node.setEstimatedTotalItems(scanned_files.items.len);
......@@ -432,16 +435,26 @@ pub const JobQueue = struct {
432435 archiver.prefix = pkg_hash_slice;
433436
434437 var file_read_buffer: [4096]u8 = undefined;
435
436 for (scanned_files.items) |entry_path| {
437 var file = try pkg_dir.openFile(io, entry_path, .{});
438 defer file.close(io);
439 var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
440 archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) {
441 error.ReadFailed => return file_reader.err.?,
442 error.WriteFailed => return file_writer.err.?,
443 else => |e| return e,
444 };
438 var link_buf: [fs.max_path_bytes]u8 = undefined;
439
440 for (scanned_files.items) |scanned_file| {
441 const entry_path = scanned_file.ptr[0..scanned_file.len];
442 if (scanned_file.symlink) {
443 const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)];
444 archiver.writeLink(entry_path, link_name, .{}) catch |err| switch (err) {
445 error.WriteFailed => return file_writer.err.?,
446 else => |e| return e,
447 };
448 } else {
449 var file = try pkg_dir.openFile(io, entry_path, .{});
450 defer file.close(io);
451 var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
452 archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) {
453 error.ReadFailed => return file_reader.err.?,
454 error.WriteFailed => return file_writer.err.?,
455 else => |e| return e,
456 };
457 }
445458 prog_node.completeOne();
446459 }
447460
......@@ -455,8 +468,14 @@ pub const JobQueue = struct {
455468 }
456469};
457470
458fn stringCmp(_: void, lhs: []const u8, rhs: []const u8) bool {
459 return std.mem.lessThan(u8, lhs, rhs);
471const ScannedFile = struct {
472 ptr: [*]const u8,
473 len: u32,
474 symlink: bool,
475};
476
477fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool {
478 return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]);
460479}
461480
462481pub const Location = union(enum) {
......@@ -1819,10 +1838,8 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute
18191838 assert(!f.job_queue.recursive);
18201839 // Print something to stdout that can be text diffed to figure out why
18211840 // the package hash is different.
1822 dumpHashInfo(io, all_files.items) catch |err| {
1823 std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)});
1824 std.process.exit(1);
1825 };
1841 dumpHashInfo(io, all_files.items) catch |err|
1842 std.process.fatal("unable to write to stdout: {t}", .{err});
18261843 }
18271844
18281845 return .{
......@@ -1834,11 +1851,16 @@ fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!Compute
18341851fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void {
18351852 var stdout_buffer: [1024]u8 = undefined;
18361853 var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer);
1837 const w = &stdout_writer.interface;
1854 dumpHashInfoWriter(&stdout_writer.interface, all_files) catch |err| switch (err) {
1855 error.WriteFailed => return stdout_writer.err.?,
1856 };
1857 try stdout_writer.flush();
1858}
1859
1860fn dumpHashInfoWriter(w: *Io.Writer, all_files: []const *const HashedFile) Io.Writer.Error!void {
18381861 for (all_files) |hashed_file| {
18391862 try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path });
18401863 }
1841 try w.flush();
18421864}
18431865
18441866fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void {