authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-04-03 21:24:30-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-04-03 21:24:30-07:00
loge5d900268a1d969dcaf4f2d657e96a5a5f217e7b
tree54eb6ff054d660c6304f9eb1aa94ac64b67a25a5
parentb88ae8dbd84886d3b9b26509034720f755a0e28a
parenta60b7af2c19ca14609bd052916299ffb64063856
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #19111 from ianic/no_strip_components

package manager: handle archives without leading root folder

2 files changed, 200 insertions(+), 60 deletions(-)

lib/std/tar.zig+102-8
......@@ -29,6 +29,9 @@ pub const Diagnostics = struct {
2929 allocator: std.mem.Allocator,
3030 errors: std.ArrayListUnmanaged(Error) = .{},
3131
32 root_entries: usize = 0,
33 root_dir: ?[]const u8 = null,
34
3235 pub const Error = union(enum) {
3336 unable_to_create_sym_link: struct {
3437 code: anyerror,
......@@ -45,6 +48,45 @@ pub const Diagnostics = struct {
4548 },
4649 };
4750
51 fn findRoot(d: *Diagnostics, path: []const u8, kind: FileKind) !void {
52 if (rootDir(path)) |root_dir| {
53 d.root_entries += 1;
54 if (kind == .directory and d.root_entries == 1) {
55 d.root_dir = try d.allocator.dupe(u8, root_dir);
56 return;
57 }
58 if (d.root_dir) |r| {
59 d.allocator.free(r);
60 d.root_dir = null;
61 }
62 }
63 }
64
65 // If path is package root returns root_dir name, otherwise null.
66 fn rootDir(path: []const u8) ?[]const u8 {
67 if (path.len == 0) return null;
68
69 const start_index: usize = if (path[0] == '/') 1 else 0;
70 const end_index: usize = if (path[path.len - 1] == '/') path.len - 1 else path.len;
71 const buf = path[start_index..end_index];
72 return if (std.mem.indexOfScalarPos(u8, buf, 0, '/') == null)
73 buf
74 else
75 null;
76 }
77
78 test rootDir {
79 const expectEqualStrings = testing.expectEqualStrings;
80 const expect = testing.expect;
81
82 try expectEqualStrings("a", rootDir("a").?);
83 try expectEqualStrings("b", rootDir("b").?);
84 try expectEqualStrings("c", rootDir("/c").?);
85 try expectEqualStrings("d", rootDir("/d/").?);
86 try expect(rootDir("a/b") == null);
87 try expect(rootDir("") == null);
88 }
89
4890 pub fn deinit(d: *Diagnostics) void {
4991 for (d.errors.items) |item| {
5092 switch (item) {
......@@ -61,6 +103,10 @@ pub const Diagnostics = struct {
61103 }
62104 }
63105 d.errors.deinit(d.allocator);
106 if (d.root_dir) |r| {
107 d.allocator.free(r);
108 d.root_dir = null;
109 }
64110 d.* = undefined;
65111 }
66112};
......@@ -580,19 +626,21 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)
580626 .link_name_buffer = &link_name_buffer,
581627 .diagnostics = options.diagnostics,
582628 });
629
583630 while (try iter.next()) |file| {
631 const file_name = stripComponents(file.name, options.strip_components);
632 if (options.diagnostics) |d| {
633 try d.findRoot(file_name, file.kind);
634 }
635
584636 switch (file.kind) {
585637 .directory => {
586 const file_name = stripComponents(file.name, options.strip_components);
587638 if (file_name.len != 0 and !options.exclude_empty_directories) {
588639 try dir.makePath(file_name);
589640 }
590641 },
591642 .file => {
592 if (file.size == 0 and file.name.len == 0) return;
593 const file_name = stripComponents(file.name, options.strip_components);
594643 if (file_name.len == 0) return error.BadFileName;
595
596644 if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) |fs_file| {
597645 defer fs_file.close();
598646 try file.writeAll(fs_file);
......@@ -605,12 +653,8 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)
605653 }
606654 },
607655 .sym_link => {
608 // The file system path of the symbolic link.
609 const file_name = stripComponents(file.name, options.strip_components);
610656 if (file_name.len == 0) return error.BadFileName;
611 // The data inside the symbolic link.
612657 const link_name = file.link_name;
613
614658 createDirAndSymlink(dir, link_name, file_name) catch |err| {
615659 const d = options.diagnostics orelse return error.UnableToCreateSymLink;
616660 try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{
......@@ -799,6 +843,7 @@ test PaxIterator {
799843
800844test {
801845 _ = @import("tar/test.zig");
846 _ = Diagnostics;
802847}
803848
804849test "header parse size" {
......@@ -993,6 +1038,55 @@ test pipeToFileSystem {
9931038 );
9941039}
9951040
1041test "pipeToFileSystem root_dir" {
1042 const data = @embedFile("tar/testdata/example.tar");
1043 var fbs = std.io.fixedBufferStream(data);
1044 const reader = fbs.reader();
1045
1046 // with strip_components = 1
1047 {
1048 var tmp = testing.tmpDir(.{ .no_follow = true });
1049 defer tmp.cleanup();
1050 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1051 defer diagnostics.deinit();
1052
1053 pipeToFileSystem(tmp.dir, reader, .{
1054 .strip_components = 1,
1055 .diagnostics = &diagnostics,
1056 }) catch |err| {
1057 // Skip on platform which don't support symlinks
1058 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
1059 return err;
1060 };
1061
1062 // there is no root_dir
1063 try testing.expect(diagnostics.root_dir == null);
1064 try testing.expectEqual(3, diagnostics.root_entries);
1065 }
1066
1067 // with strip_components = 0
1068 {
1069 fbs.reset();
1070 var tmp = testing.tmpDir(.{ .no_follow = true });
1071 defer tmp.cleanup();
1072 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1073 defer diagnostics.deinit();
1074
1075 pipeToFileSystem(tmp.dir, reader, .{
1076 .strip_components = 0,
1077 .diagnostics = &diagnostics,
1078 }) catch |err| {
1079 // Skip on platform which don't support symlinks
1080 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
1081 return err;
1082 };
1083
1084 // root_dir found
1085 try testing.expectEqualStrings("example", diagnostics.root_dir.?);
1086 try testing.expectEqual(1, diagnostics.root_entries);
1087 }
1088}
1089
9961090fn normalizePath(bytes: []u8) []u8 {
9971091 const canonical_sep = std.fs.path.sep_posix;
9981092 if (std.fs.path.sep == canonical_sep) return bytes;
src/Package/Fetch.zig+98-52
......@@ -441,7 +441,7 @@ fn runResource(
441441 const rand_int = std.crypto.random.int(u64);
442442 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
443443
444 {
444 const package_sub_path = blk: {
445445 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
446446 var tmp_directory: Cache.Directory = .{
447447 .path = tmp_directory_path,
......@@ -461,37 +461,50 @@ fn runResource(
461461 };
462462 defer tmp_directory.handle.close();
463463
464 try unpackResource(f, resource, uri_path, tmp_directory);
464 // Unpack resource into tmp_directory. A non-null return value means
465 // that the package contents are inside a `pkg_dir` sub-directory.
466 const pkg_dir = try unpackResource(f, resource, uri_path, tmp_directory);
467
468 var pkg_path: Cache.Path = .{
469 .root_dir = tmp_directory,
470 .sub_path = if (pkg_dir) |pkg_dir_name| pkg_dir_name else "",
471 };
472
473 // Apply btrfs workaround if needed. Reopen tmp_directory.
474 if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {
475 // https://github.com/ziglang/zig/issues/17095
476 pkg_path.root_dir.handle.close();
477 pkg_path.root_dir.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{
478 .iterate = true,
479 }) catch @panic("btrfs workaround failed");
480 }
465481
466482 // Load, parse, and validate the unpacked build.zig.zon file. It is allowed
467483 // for the file to be missing, in which case this fetched package is
468484 // considered to be a "naked" package.
469 try loadManifest(f, .{ .root_dir = tmp_directory });
470
471 // Apply the manifest's inclusion rules to the temporary directory by
472 // deleting excluded files. If any error occurred for files that were
473 // ultimately excluded, those errors should be ignored, such as failure to
474 // create symlinks that weren't supposed to be included anyway.
475
476 // Empty directories have already been omitted by `unpackResource`.
485 try loadManifest(f, pkg_path);
477486
478487 const filter: Filter = .{
479488 .include_paths = if (f.manifest) |m| m.paths else .{},
480489 };
481490
491 // TODO:
492 // If any error occurred for files that were ultimately excluded, those
493 // errors should be ignored, such as failure to create symlinks that
494 // weren't supposed to be included anyway.
495
496 // Apply the manifest's inclusion rules to the temporary directory by
497 // deleting excluded files.
498 // Empty directories have already been omitted by `unpackResource`.
482499 // Compute the package hash based on the remaining files in the temporary
483500 // directory.
501 f.actual_hash = try computeHash(f, pkg_path, filter);
484502
485 if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {
486 // https://github.com/ziglang/zig/issues/17095
487 tmp_directory.handle.close();
488 tmp_directory.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{
489 .iterate = true,
490 }) catch @panic("btrfs workaround failed");
491 }
492
493 f.actual_hash = try computeHash(f, tmp_directory, filter);
494 }
503 break :blk if (pkg_dir) |pkg_dir_name|
504 try fs.path.join(arena, &.{ tmp_dir_sub_path, pkg_dir_name })
505 else
506 tmp_dir_sub_path;
507 };
495508
496509 // Rename the temporary directory into the global zig package cache
497510 // directory. If the hash already exists, delete the temporary directory
......@@ -503,7 +516,7 @@ fn runResource(
503516 .root_dir = cache_root,
504517 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
505518 };
506 renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, f.package_root.sub_path) catch |err| {
519 renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
507520 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
508521 const dest = try cache_root.join(arena, &.{f.package_root.sub_path});
509522 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
......@@ -512,6 +525,10 @@ fn runResource(
512525 ) });
513526 return error.FetchFailed;
514527 };
528 // Remove temporary directory root if not already renamed to global cache.
529 if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) {
530 cache_root.handle.deleteDir(tmp_dir_sub_path) catch {};
531 }
515532
516533 // Validate the computed hash against the expected hash. If invalid, this
517534 // job is done.
......@@ -867,9 +884,9 @@ const FileType = enum {
867884 try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
868885 try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
869886 try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
887 try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
870888
871889 try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
872 try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null);
873890 try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
874891 try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
875892 try std.testing.expect(fromContentDisposition("inline; size=42") == null);
......@@ -1027,12 +1044,16 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
10271044 ));
10281045}
10291046
1047/// A `null` return value indicates the `tmp_directory` is populated directly
1048/// with the package contents.
1049/// A non-null return value means that the package contents are inside a
1050/// sub-directory indicated by the named path.
10301051fn unpackResource(
10311052 f: *Fetch,
10321053 resource: *Resource,
10331054 uri_path: []const u8,
10341055 tmp_directory: Cache.Directory,
1035) RunError!void {
1056) RunError!?[]const u8 {
10361057 const eb = &f.error_bundle;
10371058 const file_type = switch (resource.*) {
10381059 .file => FileType.fromPath(uri_path) orelse
......@@ -1093,21 +1114,24 @@ fn unpackResource(
10931114
10941115 .git => .git_pack,
10951116
1096 .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {
1097 return f.fail(f.location_tok, try eb.printString(
1098 "unable to copy directory '{s}': {s}",
1099 .{ uri_path, @errorName(err) },
1100 ));
1117 .dir => |dir| {
1118 f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {
1119 return f.fail(f.location_tok, try eb.printString(
1120 "unable to copy directory '{s}': {s}",
1121 .{ uri_path, @errorName(err) },
1122 ));
1123 };
1124 return null;
11011125 },
11021126 };
11031127
11041128 switch (file_type) {
1105 .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
1129 .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()),
11061130 .@"tar.gz" => {
11071131 const reader = resource.reader();
11081132 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
11091133 var dcp = std.compress.gzip.decompressor(br.reader());
1110 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1134 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
11111135 },
11121136 .@"tar.xz" => {
11131137 const gpa = f.arena.child_allocator;
......@@ -1120,7 +1144,7 @@ fn unpackResource(
11201144 ));
11211145 };
11221146 defer dcp.deinit();
1123 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1147 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
11241148 },
11251149 .@"tar.zst" => {
11261150 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
......@@ -1130,21 +1154,25 @@ fn unpackResource(
11301154 var dcp = std.compress.zstd.decompressor(br.reader(), .{
11311155 .window_buffer = window_buffer,
11321156 });
1133 return unpackTarball(f, tmp_directory.handle, dcp.reader());
1157 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
11341158 },
1135 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
1136 error.FetchFailed => return error.FetchFailed,
1137 error.OutOfMemory => return error.OutOfMemory,
1138 else => |e| return f.fail(f.location_tok, try eb.printString(
1139 "unable to unpack git files: {s}",
1140 .{@errorName(e)},
1141 )),
1159 .git_pack => {
1160 unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
1161 error.FetchFailed => return error.FetchFailed,
1162 error.OutOfMemory => return error.OutOfMemory,
1163 else => |e| return f.fail(f.location_tok, try eb.printString(
1164 "unable to unpack git files: {s}",
1165 .{@errorName(e)},
1166 )),
1167 };
1168 return null;
11421169 },
11431170 }
11441171}
11451172
1146fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
1173fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const u8 {
11471174 const eb = &f.error_bundle;
1175 const arena = f.arena.allocator();
11481176 const gpa = f.arena.child_allocator;
11491177
11501178 var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa };
......@@ -1152,7 +1180,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
11521180
11531181 std.tar.pipeToFileSystem(out_dir, reader, .{
11541182 .diagnostics = &diagnostics,
1155 .strip_components = 1,
1183 .strip_components = 0,
11561184 // https://github.com/ziglang/zig/issues/17463
11571185 .mode_mode = .ignore,
11581186 .exclude_empty_directories = true,
......@@ -1196,6 +1224,11 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
11961224 }
11971225 return error.FetchFailed;
11981226 }
1227
1228 return if (diagnostics.root_dir) |root_dir|
1229 return try arena.dupe(u8, root_dir)
1230 else
1231 null;
11991232}
12001233
12011234fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void {
......@@ -1341,7 +1374,7 @@ pub fn renameTmpIntoCache(
13411374/// function.
13421375fn computeHash(
13431376 f: *Fetch,
1344 tmp_directory: Cache.Directory,
1377 pkg_path: Cache.Path,
13451378 filter: Filter,
13461379) RunError!Manifest.Digest {
13471380 // All the path name strings need to be in memory for sorting.
......@@ -1349,6 +1382,7 @@ fn computeHash(
13491382 const gpa = f.arena.child_allocator;
13501383 const eb = &f.error_bundle;
13511384 const thread_pool = f.job_queue.thread_pool;
1385 const root_dir = pkg_path.root_dir.handle;
13521386
13531387 // Collect all files, recursively, then sort.
13541388 var all_files = std.ArrayList(*HashedFile).init(gpa);
......@@ -1362,7 +1396,7 @@ fn computeHash(
13621396 var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{};
13631397 defer sus_dirs.deinit(gpa);
13641398
1365 var walker = try tmp_directory.handle.walk(gpa);
1399 var walker = try root_dir.walk(gpa);
13661400 defer walker.deinit();
13671401
13681402 {
......@@ -1376,13 +1410,14 @@ fn computeHash(
13761410 while (walker.next() catch |err| {
13771411 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
13781412 "unable to walk temporary directory '{}': {s}",
1379 .{ tmp_directory, @errorName(err) },
1413 .{ pkg_path, @errorName(err) },
13801414 ) });
13811415 return error.FetchFailed;
13821416 }) |entry| {
13831417 if (entry.kind == .directory) continue;
13841418
1385 if (!filter.includePath(entry.path)) {
1419 const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
1420 if (!filter.includePath(entry_pkg_path)) {
13861421 // Delete instead of including in hash calculation.
13871422 const fs_path = try arena.dupe(u8, entry.path);
13881423
......@@ -1397,7 +1432,7 @@ fn computeHash(
13971432 };
13981433 wait_group.start();
13991434 try thread_pool.spawn(workerDeleteFile, .{
1400 tmp_directory.handle, deleted_file, &wait_group,
1435 root_dir, deleted_file, &wait_group,
14011436 });
14021437 try deleted_files.append(deleted_file);
14031438 continue;
......@@ -1420,14 +1455,14 @@ fn computeHash(
14201455 const hashed_file = try arena.create(HashedFile);
14211456 hashed_file.* = .{
14221457 .fs_path = fs_path,
1423 .normalized_path = try normalizePathAlloc(arena, fs_path),
1458 .normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
14241459 .kind = kind,
14251460 .hash = undefined, // to be populated by the worker
14261461 .failure = undefined, // to be populated by the worker
14271462 };
14281463 wait_group.start();
14291464 try thread_pool.spawn(workerHashFile, .{
1430 tmp_directory.handle, hashed_file, &wait_group,
1465 root_dir, hashed_file, &wait_group,
14311466 });
14321467 try all_files.append(hashed_file);
14331468 }
......@@ -1446,7 +1481,7 @@ fn computeHash(
14461481 var i: usize = 0;
14471482 while (i < sus_dirs.count()) : (i += 1) {
14481483 const sus_dir = sus_dirs.keys()[i];
1449 tmp_directory.handle.deleteDir(sus_dir) catch |err| switch (err) {
1484 root_dir.deleteDir(sus_dir) catch |err| switch (err) {
14501485 error.DirNotEmpty => continue,
14511486 error.FileNotFound => continue,
14521487 else => |e| {
......@@ -1610,11 +1645,22 @@ const HashedFile = struct {
16101645 }
16111646};
16121647
1648/// Strips root directory name from file system path.
1649fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
1650 if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
1651
1652 if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs_path[root_dir.len] == fs.path.sep) {
1653 return fs_path[root_dir.len + 1 ..];
1654 }
1655
1656 return fs_path;
1657}
1658
16131659/// Make a file system path identical independently of operating system path inconsistencies.
16141660/// This converts backslashes into forward slashes.
1615fn normalizePathAlloc(arena: Allocator, fs_path: []const u8) ![]const u8 {
1616 if (fs.path.sep == canonical_sep) return fs_path;
1617 const normalized = try arena.dupe(u8, fs_path);
1661fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
1662 const normalized = try arena.dupe(u8, pkg_path);
1663 if (fs.path.sep == canonical_sep) return normalized;
16181664 normalizePath(normalized);
16191665 return normalized;
16201666}