| ... | @@ -461,14 +461,10 @@ fn runResource( | ... | @@ -461,14 +461,10 @@ fn runResource( |
| 461 | }; | 461 | }; |
| 462 | defer tmp_directory.handle.close(); | 462 | defer tmp_directory.handle.close(); |
| 463 | | 463 | |
| 464 | // Unpack resource into tmp_directory. A non-null return value means | 464 | // Fetch and unpack a resource into a temporary directory. |
| 465 | // that the package contents are inside a `pkg_dir` sub-directory. | 465 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); |
| 466 | const pkg_dir = try unpackResource(f, resource, uri_path, tmp_directory); | | |
| 467 | | 466 | |
| 468 | var pkg_path: Cache.Path = .{ | 467 | var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; |
| 469 | .root_dir = tmp_directory, | | |
| 470 | .sub_path = if (pkg_dir) |pkg_dir_name| pkg_dir_name else "", | | |
| 471 | }; | | |
| 472 | | 468 | |
| 473 | // Apply btrfs workaround if needed. Reopen tmp_directory. | 469 | // Apply btrfs workaround if needed. Reopen tmp_directory. |
| 474 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { | 470 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { |
| ... | @@ -488,10 +484,9 @@ fn runResource( | ... | @@ -488,10 +484,9 @@ fn runResource( |
| 488 | .include_paths = if (f.manifest) |m| m.paths else .{}, | 484 | .include_paths = if (f.manifest) |m| m.paths else .{}, |
| 489 | }; | 485 | }; |
| 490 | | 486 | |
| 491 | // TODO: | 487 | // Ignore errors that were excluded by manifest, such as failure to |
| 492 | // If any error occurred for files that were ultimately excluded, those | 488 | // create symlinks that weren't supposed to be included anyway. |
| 493 | // errors should be ignored, such as failure to create symlinks that | 489 | try unpack_result.validate(f, filter); |
| 494 | // weren't supposed to be included anyway. | | |
| 495 | | 490 | |
| 496 | // Apply the manifest's inclusion rules to the temporary directory by | 491 | // Apply the manifest's inclusion rules to the temporary directory by |
| 497 | // deleting excluded files. | 492 | // deleting excluded files. |
| ... | @@ -500,8 +495,8 @@ fn runResource( | ... | @@ -500,8 +495,8 @@ fn runResource( |
| 500 | // directory. | 495 | // directory. |
| 501 | f.actual_hash = try computeHash(f, pkg_path, filter); | 496 | f.actual_hash = try computeHash(f, pkg_path, filter); |
| 502 | | 497 | |
| 503 | break :blk if (pkg_dir) |pkg_dir_name| | 498 | break :blk if (unpack_result.root_dir.len > 0) |
| 504 | try fs.path.join(arena, &.{ tmp_dir_sub_path, pkg_dir_name }) | 499 | try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) |
| 505 | else | 500 | else |
| 506 | tmp_dir_sub_path; | 501 | tmp_dir_sub_path; |
| 507 | }; | 502 | }; |
| ... | @@ -1044,16 +1039,12 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re | ... | @@ -1044,16 +1039,12 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re |
| 1044 | )); | 1039 | )); |
| 1045 | } | 1040 | } |
| 1046 | | 1041 | |
| 1047 | /// A `null` return value indicates the `tmp_directory` is populated directly | | |
| 1048 | /// with the package contents. | | |
| 1049 | /// A non-null return value means that the package contents are inside a | | |
| 1050 | /// sub-directory indicated by the named path. | | |
| 1051 | fn unpackResource( | 1042 | fn unpackResource( |
| 1052 | f: *Fetch, | 1043 | f: *Fetch, |
| 1053 | resource: *Resource, | 1044 | resource: *Resource, |
| 1054 | uri_path: []const u8, | 1045 | uri_path: []const u8, |
| 1055 | tmp_directory: Cache.Directory, | 1046 | tmp_directory: Cache.Directory, |
| 1056 | ) RunError!?[]const u8 { | 1047 | ) RunError!UnpackResult { |
| 1057 | const eb = &f.error_bundle; | 1048 | const eb = &f.error_bundle; |
| 1058 | const file_type = switch (resource.*) { | 1049 | const file_type = switch (resource.*) { |
| 1059 | .file => FileType.fromPath(uri_path) orelse | 1050 | .file => FileType.fromPath(uri_path) orelse |
| ... | @@ -1121,7 +1112,7 @@ fn unpackResource( | ... | @@ -1121,7 +1112,7 @@ fn unpackResource( |
| 1121 | .{ uri_path, @errorName(err) }, | 1112 | .{ uri_path, @errorName(err) }, |
| 1122 | )); | 1113 | )); |
| 1123 | }; | 1114 | }; |
| 1124 | return null; | 1115 | return .{}; |
| 1125 | }, | 1116 | }, |
| 1126 | }; | 1117 | }; |
| 1127 | | 1118 | |
| ... | @@ -1156,27 +1147,22 @@ fn unpackResource( | ... | @@ -1156,27 +1147,22 @@ fn unpackResource( |
| 1156 | }); | 1147 | }); |
| 1157 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | 1148 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1158 | }, | 1149 | }, |
| 1159 | .git_pack => { | 1150 | .git_pack => return unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { |
| 1160 | unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { | 1151 | error.FetchFailed => return error.FetchFailed, |
| 1161 | error.FetchFailed => return error.FetchFailed, | 1152 | error.OutOfMemory => return error.OutOfMemory, |
| 1162 | error.OutOfMemory => return error.OutOfMemory, | 1153 | else => |e| return f.fail(f.location_tok, try eb.printString( |
| 1163 | else => |e| return f.fail(f.location_tok, try eb.printString( | 1154 | "unable to unpack git files: {s}", |
| 1164 | "unable to unpack git files: {s}", | 1155 | .{@errorName(e)}, |
| 1165 | .{@errorName(e)}, | 1156 | )), |
| 1166 | )), | | |
| 1167 | }; | | |
| 1168 | return null; | | |
| 1169 | }, | 1157 | }, |
| 1170 | } | 1158 | } |
| 1171 | } | 1159 | } |
| 1172 | | 1160 | |
| 1173 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const u8 { | 1161 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { |
| 1174 | const eb = &f.error_bundle; | 1162 | const eb = &f.error_bundle; |
| 1175 | const arena = f.arena.allocator(); | 1163 | const arena = f.arena.allocator(); |
| 1176 | const gpa = f.arena.child_allocator; | | |
| 1177 | | 1164 | |
| 1178 | var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa }; | 1165 | var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; |
| 1179 | defer diagnostics.deinit(); | | |
| 1180 | | 1166 | |
| 1181 | std.tar.pipeToFileSystem(out_dir, reader, .{ | 1167 | std.tar.pipeToFileSystem(out_dir, reader, .{ |
| 1182 | .diagnostics = &diagnostics, | 1168 | .diagnostics = &diagnostics, |
| ... | @@ -1188,53 +1174,27 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const | ... | @@ -1188,53 +1174,27 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const |
| 1188 | .{@errorName(err)}, | 1174 | .{@errorName(err)}, |
| 1189 | )); | 1175 | )); |
| 1190 | | 1176 | |
| | 1177 | var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; |
| 1191 | if (diagnostics.errors.items.len > 0) { | 1178 | if (diagnostics.errors.items.len > 0) { |
| 1192 | const notes_len: u32 = @intCast(diagnostics.errors.items.len); | 1179 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); |
| 1193 | try eb.addRootErrorMessage(.{ | 1180 | for (diagnostics.errors.items) |item| { |
| 1194 | .msg = try eb.addString("unable to unpack tarball"), | | |
| 1195 | .src_loc = try f.srcLoc(f.location_tok), | | |
| 1196 | .notes_len = notes_len, | | |
| 1197 | }); | | |
| 1198 | const notes_start = try eb.reserveNotes(notes_len); | | |
| 1199 | for (diagnostics.errors.items, notes_start..) |item, note_i| { | | |
| 1200 | switch (item) { | 1181 | switch (item) { |
| 1201 | .unable_to_create_sym_link => |info| { | 1182 | .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), |
| 1202 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | 1183 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), |
| 1203 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | 1184 | .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), |
| 1204 | info.file_name, info.link_name, @errorName(info.code), | | |
| 1205 | }), | | |
| 1206 | })); | | |
| 1207 | }, | | |
| 1208 | .unable_to_create_file => |info| { | | |
| 1209 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | | |
| 1210 | .msg = try eb.printString("unable to create file '{s}': {s}", .{ | | |
| 1211 | info.file_name, @errorName(info.code), | | |
| 1212 | }), | | |
| 1213 | })); | | |
| 1214 | }, | | |
| 1215 | .unsupported_file_type => |info| { | | |
| 1216 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | | |
| 1217 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ | | |
| 1218 | info.file_name, @intFromEnum(info.file_type), | | |
| 1219 | }), | | |
| 1220 | })); | | |
| 1221 | }, | | |
| 1222 | } | 1185 | } |
| 1223 | } | 1186 | } |
| 1224 | return error.FetchFailed; | | |
| 1225 | } | 1187 | } |
| 1226 | | 1188 | return res; |
| 1227 | return if (diagnostics.root_dir) |root_dir| | | |
| 1228 | return try arena.dupe(u8, root_dir) | | |
| 1229 | else | | |
| 1230 | null; | | |
| 1231 | } | 1189 | } |
| 1232 | | 1190 | |
| 1233 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { | 1191 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!UnpackResult { |
| 1234 | const eb = &f.error_bundle; | 1192 | const arena = f.arena.allocator(); |
| 1235 | const gpa = f.arena.child_allocator; | 1193 | const gpa = f.arena.child_allocator; |
| 1236 | const want_oid = resource.git.want_oid; | 1194 | const want_oid = resource.git.want_oid; |
| 1237 | const reader = resource.git.fetch_stream.reader(); | 1195 | const reader = resource.git.fetch_stream.reader(); |
| | 1196 | |
| | 1197 | var res: UnpackResult = .{}; |
| 1238 | // The .git directory is used to store the packfile and associated index, but | 1198 | // The .git directory is used to store the packfile and associated index, but |
| 1239 | // we do not attempt to replicate the exact structure of a real .git | 1199 | // we do not attempt to replicate the exact structure of a real .git |
| 1240 | // directory, since that isn't relevant for fetching a package. | 1200 | // directory, since that isn't relevant for fetching a package. |
| ... | @@ -1265,35 +1225,23 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void | ... | @@ -1265,35 +1225,23 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void |
| 1265 | checkout_prog_node.activate(); | 1225 | checkout_prog_node.activate(); |
| 1266 | var repository = try git.Repository.init(gpa, pack_file, index_file); | 1226 | var repository = try git.Repository.init(gpa, pack_file, index_file); |
| 1267 | defer repository.deinit(); | 1227 | defer repository.deinit(); |
| 1268 | var diagnostics: git.Diagnostics = .{ .allocator = gpa }; | 1228 | var diagnostics: git.Diagnostics = .{ .allocator = arena }; |
| 1269 | defer diagnostics.deinit(); | | |
| 1270 | try repository.checkout(out_dir, want_oid, &diagnostics); | 1229 | try repository.checkout(out_dir, want_oid, &diagnostics); |
| 1271 | | 1230 | |
| 1272 | if (diagnostics.errors.items.len > 0) { | 1231 | if (diagnostics.errors.items.len > 0) { |
| 1273 | const notes_len: u32 = @intCast(diagnostics.errors.items.len); | 1232 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); |
| 1274 | try eb.addRootErrorMessage(.{ | 1233 | for (diagnostics.errors.items) |item| { |
| 1275 | .msg = try eb.addString("unable to unpack packfile"), | | |
| 1276 | .src_loc = try f.srcLoc(f.location_tok), | | |
| 1277 | .notes_len = notes_len, | | |
| 1278 | }); | | |
| 1279 | const notes_start = try eb.reserveNotes(notes_len); | | |
| 1280 | for (diagnostics.errors.items, notes_start..) |item, note_i| { | | |
| 1281 | switch (item) { | 1234 | switch (item) { |
| 1282 | .unable_to_create_sym_link => |info| { | 1235 | .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), |
| 1283 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | 1236 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), |
| 1284 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | | |
| 1285 | info.file_name, info.link_name, @errorName(info.code), | | |
| 1286 | }), | | |
| 1287 | })); | | |
| 1288 | }, | | |
| 1289 | } | 1237 | } |
| 1290 | } | 1238 | } |
| 1291 | return error.InvalidGitPack; | | |
| 1292 | } | 1239 | } |
| 1293 | } | 1240 | } |
| 1294 | } | 1241 | } |
| 1295 | | 1242 | |
| 1296 | try out_dir.deleteTree(".git"); | 1243 | try out_dir.deleteTree(".git"); |
| | 1244 | return res; |
| 1297 | } | 1245 | } |
| 1298 | | 1246 | |
| 1299 | fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { | 1247 | fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { |
| ... | @@ -1743,6 +1691,7 @@ const native_os = builtin.os.tag; | ... | @@ -1743,6 +1691,7 @@ const native_os = builtin.os.tag; |
| 1743 | test { | 1691 | test { |
| 1744 | _ = Filter; | 1692 | _ = Filter; |
| 1745 | _ = FileType; | 1693 | _ = FileType; |
| | 1694 | _ = UnpackResult; |
| 1746 | } | 1695 | } |
| 1747 | | 1696 | |
| 1748 | // Detects executable header: ELF magic header or shebang line. | 1697 | // Detects executable header: ELF magic header or shebang line. |
| ... | @@ -1778,3 +1727,472 @@ test FileHeader { | ... | @@ -1778,3 +1727,472 @@ test FileHeader { |
| 1778 | h.update(FileHeader.elf_magic[2..4]); | 1727 | h.update(FileHeader.elf_magic[2..4]); |
| 1779 | try std.testing.expect(h.isExecutable()); | 1728 | try std.testing.expect(h.isExecutable()); |
| 1780 | } | 1729 | } |
| | 1730 | |
| | 1731 | // Result of the `unpackResource` operation. Enables collecting errors from |
| | 1732 | // tar/git diagnostic, filtering that errors by manifest inclusion rules and |
| | 1733 | // emitting remaining errors to an `ErrorBundle`. |
| | 1734 | const UnpackResult = struct { |
| | 1735 | errors: []Error = undefined, |
| | 1736 | errors_count: usize = 0, |
| | 1737 | root_error_message: []const u8 = "", |
| | 1738 | |
| | 1739 | // A non empty value means that the package contents are inside a |
| | 1740 | // sub-directory indicated by the named path. |
| | 1741 | root_dir: []const u8 = "", |
| | 1742 | |
| | 1743 | const Error = union(enum) { |
| | 1744 | unable_to_create_sym_link: struct { |
| | 1745 | code: anyerror, |
| | 1746 | file_name: []const u8, |
| | 1747 | link_name: []const u8, |
| | 1748 | }, |
| | 1749 | unable_to_create_file: struct { |
| | 1750 | code: anyerror, |
| | 1751 | file_name: []const u8, |
| | 1752 | }, |
| | 1753 | unsupported_file_type: struct { |
| | 1754 | file_name: []const u8, |
| | 1755 | file_type: u8, |
| | 1756 | }, |
| | 1757 | |
| | 1758 | fn excluded(self: Error, filter: Filter) bool { |
| | 1759 | const file_name = switch (self) { |
| | 1760 | .unable_to_create_file => |info| info.file_name, |
| | 1761 | .unable_to_create_sym_link => |info| info.file_name, |
| | 1762 | .unsupported_file_type => |info| info.file_name, |
| | 1763 | }; |
| | 1764 | return !filter.includePath(file_name); |
| | 1765 | } |
| | 1766 | }; |
| | 1767 | |
| | 1768 | fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { |
| | 1769 | self.root_error_message = try arena.dupe(u8, root_error_message); |
| | 1770 | self.errors = try arena.alloc(UnpackResult.Error, n); |
| | 1771 | } |
| | 1772 | |
| | 1773 | fn hasErrors(self: *UnpackResult) bool { |
| | 1774 | return self.errors_count > 0; |
| | 1775 | } |
| | 1776 | |
| | 1777 | fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { |
| | 1778 | self.errors[self.errors_count] = .{ .unable_to_create_file = .{ |
| | 1779 | .code = err, |
| | 1780 | .file_name = file_name, |
| | 1781 | } }; |
| | 1782 | self.errors_count += 1; |
| | 1783 | } |
| | 1784 | |
| | 1785 | fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { |
| | 1786 | self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ |
| | 1787 | .code = err, |
| | 1788 | .file_name = file_name, |
| | 1789 | .link_name = link_name, |
| | 1790 | } }; |
| | 1791 | self.errors_count += 1; |
| | 1792 | } |
| | 1793 | |
| | 1794 | fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { |
| | 1795 | self.errors[self.errors_count] = .{ .unsupported_file_type = .{ |
| | 1796 | .file_name = file_name, |
| | 1797 | .file_type = file_type, |
| | 1798 | } }; |
| | 1799 | self.errors_count += 1; |
| | 1800 | } |
| | 1801 | |
| | 1802 | fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { |
| | 1803 | self.filterErrors(filter); |
| | 1804 | if (self.hasErrors()) { |
| | 1805 | const eb = &f.error_bundle; |
| | 1806 | try self.bundleErrors(eb, try f.srcLoc(f.location_tok)); |
| | 1807 | return error.FetchFailed; |
| | 1808 | } |
| | 1809 | } |
| | 1810 | |
| | 1811 | // Filter errors by manifest inclusion rules. |
| | 1812 | fn filterErrors(self: *UnpackResult, filter: Filter) void { |
| | 1813 | var i = self.errors_count; |
| | 1814 | while (i > 0) { |
| | 1815 | i -= 1; |
| | 1816 | if (self.errors[i].excluded(filter)) { |
| | 1817 | self.errors_count -= 1; |
| | 1818 | const tmp = self.errors[i]; |
| | 1819 | self.errors[i] = self.errors[self.errors_count]; |
| | 1820 | self.errors[self.errors_count] = tmp; |
| | 1821 | } |
| | 1822 | } |
| | 1823 | } |
| | 1824 | |
| | 1825 | // Emmit errors to an `ErrorBundle`. |
| | 1826 | fn bundleErrors( |
| | 1827 | self: *UnpackResult, |
| | 1828 | eb: *ErrorBundle.Wip, |
| | 1829 | src_loc: ErrorBundle.SourceLocationIndex, |
| | 1830 | ) !void { |
| | 1831 | if (self.errors_count == 0 and self.root_error_message.len == 0) |
| | 1832 | return; |
| | 1833 | |
| | 1834 | const notes_len: u32 = @intCast(self.errors_count); |
| | 1835 | try eb.addRootErrorMessage(.{ |
| | 1836 | .msg = try eb.addString(self.root_error_message), |
| | 1837 | .src_loc = src_loc, |
| | 1838 | .notes_len = notes_len, |
| | 1839 | }); |
| | 1840 | const notes_start = try eb.reserveNotes(notes_len); |
| | 1841 | for (self.errors, notes_start..) |item, note_i| { |
| | 1842 | switch (item) { |
| | 1843 | .unable_to_create_sym_link => |info| { |
| | 1844 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| | 1845 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ |
| | 1846 | info.file_name, info.link_name, @errorName(info.code), |
| | 1847 | }), |
| | 1848 | })); |
| | 1849 | }, |
| | 1850 | .unable_to_create_file => |info| { |
| | 1851 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| | 1852 | .msg = try eb.printString("unable to create file '{s}': {s}", .{ |
| | 1853 | info.file_name, @errorName(info.code), |
| | 1854 | }), |
| | 1855 | })); |
| | 1856 | }, |
| | 1857 | .unsupported_file_type => |info| { |
| | 1858 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ |
| | 1859 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ |
| | 1860 | info.file_name, info.file_type, |
| | 1861 | }), |
| | 1862 | })); |
| | 1863 | }, |
| | 1864 | } |
| | 1865 | } |
| | 1866 | } |
| | 1867 | |
| | 1868 | test filterErrors { |
| | 1869 | var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); |
| | 1870 | defer arena_instance.deinit(); |
| | 1871 | const arena = arena_instance.allocator(); |
| | 1872 | |
| | 1873 | // init |
| | 1874 | var res: UnpackResult = .{}; |
| | 1875 | try res.allocErrors(arena, 4, "error"); |
| | 1876 | try std.testing.expectEqual(0, res.errors_count); |
| | 1877 | |
| | 1878 | // create errors |
| | 1879 | res.unableToCreateFile("dir1/file1", error.File1); |
| | 1880 | res.unableToCreateSymLink("dir2/file2", "", error.File2); |
| | 1881 | res.unableToCreateFile("dir1/file3", error.File3); |
| | 1882 | res.unsupportedFileType("dir2/file4", 'x'); |
| | 1883 | try std.testing.expectEqual(4, res.errors_count); |
| | 1884 | |
| | 1885 | // filter errors |
| | 1886 | var filter: Filter = .{}; |
| | 1887 | try filter.include_paths.put(arena, "dir2", {}); |
| | 1888 | res.filterErrors(filter); |
| | 1889 | |
| | 1890 | try std.testing.expectEqual(2, res.errors_count); |
| | 1891 | try std.testing.expect(res.errors[0] == Error.unsupported_file_type); |
| | 1892 | try std.testing.expect(res.errors[1] == Error.unable_to_create_sym_link); |
| | 1893 | // filtered: moved to the list end |
| | 1894 | try std.testing.expect(res.errors[2] == Error.unable_to_create_file); |
| | 1895 | try std.testing.expect(res.errors[3] == Error.unable_to_create_file); |
| | 1896 | } |
| | 1897 | }; |
| | 1898 | |
| | 1899 | test "tarball with duplicate paths" { |
| | 1900 | // This tarball has duplicate path 'dir1/file1' to simulate case sensitve |
| | 1901 | // file system on any file sytstem. |
| | 1902 | // |
| | 1903 | // duplicate_paths/ |
| | 1904 | // duplicate_paths/dir1/ |
| | 1905 | // duplicate_paths/dir1/file1 |
| | 1906 | // duplicate_paths/dir1/file1 |
| | 1907 | // duplicate_paths/build.zig.zon |
| | 1908 | // duplicate_paths/src/ |
| | 1909 | // duplicate_paths/src/main.zig |
| | 1910 | // duplicate_paths/src/root.zig |
| | 1911 | // duplicate_paths/build.zig |
| | 1912 | // |
| | 1913 | |
| | 1914 | const gpa = std.testing.allocator; |
| | 1915 | var tmp = std.testing.tmpDir(.{}); |
| | 1916 | defer tmp.cleanup(); |
| | 1917 | |
| | 1918 | const tarball_name = "duplicate_paths.tar.gz"; |
| | 1919 | try saveEmbedFile(tarball_name, tmp.dir); |
| | 1920 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| | 1921 | defer gpa.free(tarball_path); |
| | 1922 | |
| | 1923 | // Run tarball fetch, expect to fail |
| | 1924 | var fb: TestFetchBuilder = undefined; |
| | 1925 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| | 1926 | defer fb.deinit(); |
| | 1927 | try std.testing.expectError(error.FetchFailed, fetch.run()); |
| | 1928 | |
| | 1929 | try fb.expectFetchErrors(1, |
| | 1930 | \\error: unable to unpack tarball |
| | 1931 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| | 1932 | \\ |
| | 1933 | ); |
| | 1934 | } |
| | 1935 | |
| | 1936 | test "tarball with excluded duplicate paths" { |
| | 1937 | // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. |
| | 1938 | // |
| | 1939 | // .paths = .{ |
| | 1940 | // "build.zig", |
| | 1941 | // "build.zig.zon", |
| | 1942 | // "src", |
| | 1943 | // } |
| | 1944 | // |
| | 1945 | |
| | 1946 | const gpa = std.testing.allocator; |
| | 1947 | var tmp = std.testing.tmpDir(.{}); |
| | 1948 | defer tmp.cleanup(); |
| | 1949 | |
| | 1950 | const tarball_name = "duplicate_paths_excluded.tar.gz"; |
| | 1951 | try saveEmbedFile(tarball_name, tmp.dir); |
| | 1952 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| | 1953 | defer gpa.free(tarball_path); |
| | 1954 | |
| | 1955 | // Run tarball fetch, should succeed |
| | 1956 | var fb: TestFetchBuilder = undefined; |
| | 1957 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| | 1958 | defer fb.deinit(); |
| | 1959 | try fetch.run(); |
| | 1960 | |
| | 1961 | const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); |
| | 1962 | try std.testing.expectEqualStrings( |
| | 1963 | "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", |
| | 1964 | &hex_digest, |
| | 1965 | ); |
| | 1966 | |
| | 1967 | const expected_files: []const []const u8 = &.{ |
| | 1968 | "build.zig", |
| | 1969 | "build.zig.zon", |
| | 1970 | "src/main.zig", |
| | 1971 | "src/root.zig", |
| | 1972 | }; |
| | 1973 | try fb.expectPackageFiles(expected_files); |
| | 1974 | } |
| | 1975 | |
| | 1976 | test "tarball without root folder" { |
| | 1977 | // Tarball with root folder. Manifest excludes dir1 and dir2. |
| | 1978 | // |
| | 1979 | // build.zig |
| | 1980 | // build.zig.zon |
| | 1981 | // dir1/ |
| | 1982 | // dir1/file2 |
| | 1983 | // dir1/file1 |
| | 1984 | // dir2/ |
| | 1985 | // dir2/file2 |
| | 1986 | // src/ |
| | 1987 | // src/main.zig |
| | 1988 | // |
| | 1989 | |
| | 1990 | const gpa = std.testing.allocator; |
| | 1991 | var tmp = std.testing.tmpDir(.{}); |
| | 1992 | defer tmp.cleanup(); |
| | 1993 | |
| | 1994 | const tarball_name = "no_root.tar.gz"; |
| | 1995 | try saveEmbedFile(tarball_name, tmp.dir); |
| | 1996 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| | 1997 | defer gpa.free(tarball_path); |
| | 1998 | |
| | 1999 | // Run tarball fetch, should succeed |
| | 2000 | var fb: TestFetchBuilder = undefined; |
| | 2001 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| | 2002 | defer fb.deinit(); |
| | 2003 | try fetch.run(); |
| | 2004 | |
| | 2005 | const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); |
| | 2006 | try std.testing.expectEqualStrings( |
| | 2007 | "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", |
| | 2008 | &hex_digest, |
| | 2009 | ); |
| | 2010 | |
| | 2011 | const expected_files: []const []const u8 = &.{ |
| | 2012 | "build.zig", |
| | 2013 | "build.zig.zon", |
| | 2014 | "src/main.zig", |
| | 2015 | }; |
| | 2016 | try fb.expectPackageFiles(expected_files); |
| | 2017 | } |
| | 2018 | |
| | 2019 | test "set executable bit based on file content" { |
| | 2020 | if (!std.fs.has_executable_bit) return error.SkipZigTest; |
| | 2021 | const gpa = std.testing.allocator; |
| | 2022 | var tmp = std.testing.tmpDir(.{}); |
| | 2023 | defer tmp.cleanup(); |
| | 2024 | |
| | 2025 | const tarball_name = "executables.tar.gz"; |
| | 2026 | try saveEmbedFile(tarball_name, tmp.dir); |
| | 2027 | const tarball_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| | 2028 | defer gpa.free(tarball_path); |
| | 2029 | |
| | 2030 | // $ tar -tvf executables.tar.gz |
| | 2031 | // drwxrwxr-x 0 executables/ |
| | 2032 | // -rwxrwxr-x 170 executables/hello |
| | 2033 | // lrwxrwxrwx 0 executables/hello_ln -> hello |
| | 2034 | // -rw-rw-r-- 0 executables/file1 |
| | 2035 | // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit |
| | 2036 | // -rwxrwxr-x 7 executables/script_without_shebang |
| | 2037 | // -rwxrwxr-x 17 executables/script |
| | 2038 | |
| | 2039 | var fb: TestFetchBuilder = undefined; |
| | 2040 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); |
| | 2041 | defer fb.deinit(); |
| | 2042 | |
| | 2043 | try fetch.run(); |
| | 2044 | try std.testing.expectEqualStrings( |
| | 2045 | "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", |
| | 2046 | &Manifest.hexDigest(fetch.actual_hash), |
| | 2047 | ); |
| | 2048 | |
| | 2049 | var out = try fb.packageDir(); |
| | 2050 | defer out.close(); |
| | 2051 | const S = std.posix.S; |
| | 2052 | // expect executable bit not set |
| | 2053 | try std.testing.expect((try out.statFile("file1")).mode & S.IXUSR == 0); |
| | 2054 | try std.testing.expect((try out.statFile("script_without_shebang")).mode & S.IXUSR == 0); |
| | 2055 | // expect executable bit set |
| | 2056 | try std.testing.expect((try out.statFile("hello")).mode & S.IXUSR != 0); |
| | 2057 | try std.testing.expect((try out.statFile("script")).mode & S.IXUSR != 0); |
| | 2058 | try std.testing.expect((try out.statFile("script_with_shebang_without_exec_bit")).mode & S.IXUSR != 0); |
| | 2059 | try std.testing.expect((try out.statFile("hello_ln")).mode & S.IXUSR != 0); |
| | 2060 | |
| | 2061 | // |
| | 2062 | // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3 |
| | 2063 | // -rw-rw-r-- 1 0 Apr file1 |
| | 2064 | // -rwxrwxr-x 1 170 Apr hello |
| | 2065 | // lrwxrwxrwx 1 5 Apr hello_ln -> hello |
| | 2066 | // -rwxrwxr-x 1 17 Apr script |
| | 2067 | // -rw-rw-r-- 1 7 Apr script_without_shebang |
| | 2068 | // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit |
| | 2069 | } |
| | 2070 | |
| | 2071 | fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void { |
| | 2072 | //const tarball_name = "duplicate_paths_excluded.tar.gz"; |
| | 2073 | const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name); |
| | 2074 | var tmp_file = try dir.createFile(tarball_name, .{}); |
| | 2075 | defer tmp_file.close(); |
| | 2076 | try tmp_file.writeAll(tarball_content); |
| | 2077 | } |
| | 2078 | |
| | 2079 | // Builds Fetch with required dependencies, clears dependencies on deinit(). |
| | 2080 | const TestFetchBuilder = struct { |
| | 2081 | thread_pool: ThreadPool, |
| | 2082 | http_client: std.http.Client, |
| | 2083 | global_cache_directory: Cache.Directory, |
| | 2084 | progress: std.Progress, |
| | 2085 | job_queue: Fetch.JobQueue, |
| | 2086 | fetch: Fetch, |
| | 2087 | |
| | 2088 | fn build( |
| | 2089 | self: *TestFetchBuilder, |
| | 2090 | allocator: std.mem.Allocator, |
| | 2091 | cache_parent_dir: std.fs.Dir, |
| | 2092 | path_or_url: []const u8, |
| | 2093 | ) !*Fetch { |
| | 2094 | const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{}); |
| | 2095 | |
| | 2096 | try self.thread_pool.init(.{ .allocator = allocator }); |
| | 2097 | self.http_client = .{ .allocator = allocator }; |
| | 2098 | self.global_cache_directory = .{ .handle = cache_dir, .path = null }; |
| | 2099 | |
| | 2100 | self.progress = .{ .dont_print_on_dumb = true }; |
| | 2101 | |
| | 2102 | self.job_queue = .{ |
| | 2103 | .http_client = &self.http_client, |
| | 2104 | .thread_pool = &self.thread_pool, |
| | 2105 | .global_cache = self.global_cache_directory, |
| | 2106 | .recursive = false, |
| | 2107 | .read_only = false, |
| | 2108 | .debug_hash = false, |
| | 2109 | .work_around_btrfs_bug = false, |
| | 2110 | }; |
| | 2111 | |
| | 2112 | self.fetch = .{ |
| | 2113 | .arena = std.heap.ArenaAllocator.init(allocator), |
| | 2114 | .location = .{ .path_or_url = path_or_url }, |
| | 2115 | .location_tok = 0, |
| | 2116 | .hash_tok = 0, |
| | 2117 | .name_tok = 0, |
| | 2118 | .lazy_status = .eager, |
| | 2119 | .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, |
| | 2120 | .parent_manifest_ast = null, |
| | 2121 | .prog_node = self.progress.start("Fetch", 0), |
| | 2122 | .job_queue = &self.job_queue, |
| | 2123 | .omit_missing_hash_error = true, |
| | 2124 | .allow_missing_paths_field = false, |
| | 2125 | |
| | 2126 | .package_root = undefined, |
| | 2127 | .error_bundle = undefined, |
| | 2128 | .manifest = null, |
| | 2129 | .manifest_ast = undefined, |
| | 2130 | .actual_hash = undefined, |
| | 2131 | .has_build_zig = false, |
| | 2132 | .oom_flag = false, |
| | 2133 | .module = null, |
| | 2134 | }; |
| | 2135 | return &self.fetch; |
| | 2136 | } |
| | 2137 | |
| | 2138 | fn deinit(self: *TestFetchBuilder) void { |
| | 2139 | self.fetch.deinit(); |
| | 2140 | self.job_queue.deinit(); |
| | 2141 | self.fetch.prog_node.end(); |
| | 2142 | self.global_cache_directory.handle.close(); |
| | 2143 | self.http_client.deinit(); |
| | 2144 | self.thread_pool.deinit(); |
| | 2145 | } |
| | 2146 | |
| | 2147 | fn packageDir(self: *TestFetchBuilder) !fs.Dir { |
| | 2148 | const root = self.fetch.package_root; |
| | 2149 | return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); |
| | 2150 | } |
| | 2151 | |
| | 2152 | // Test helper, asserts thet package dir constains expected_files. |
| | 2153 | // expected_files must be sorted. |
| | 2154 | fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { |
| | 2155 | var package_dir = try self.packageDir(); |
| | 2156 | defer package_dir.close(); |
| | 2157 | |
| | 2158 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; |
| | 2159 | defer actual_files.deinit(std.testing.allocator); |
| | 2160 | defer for (actual_files.items) |file| std.testing.allocator.free(file); |
| | 2161 | var walker = try package_dir.walk(std.testing.allocator); |
| | 2162 | defer walker.deinit(); |
| | 2163 | while (try walker.next()) |entry| { |
| | 2164 | if (entry.kind != .file) continue; |
| | 2165 | const path = try std.testing.allocator.dupe(u8, entry.path); |
| | 2166 | errdefer std.testing.allocator.free(path); |
| | 2167 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); |
| | 2168 | try actual_files.append(std.testing.allocator, path); |
| | 2169 | } |
| | 2170 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { |
| | 2171 | fn lessThan(_: void, a: []u8, b: []u8) bool { |
| | 2172 | return std.mem.lessThan(u8, a, b); |
| | 2173 | } |
| | 2174 | }.lessThan); |
| | 2175 | |
| | 2176 | try std.testing.expectEqual(expected_files.len, actual_files.items.len); |
| | 2177 | for (expected_files, 0..) |file_name, i| { |
| | 2178 | try std.testing.expectEqualStrings(file_name, actual_files.items[i]); |
| | 2179 | } |
| | 2180 | try std.testing.expectEqualDeep(expected_files, actual_files.items); |
| | 2181 | } |
| | 2182 | |
| | 2183 | // Test helper, asserts that fetch has failed with `msg` error message. |
| | 2184 | fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { |
| | 2185 | var errors = try self.fetch.error_bundle.toOwnedBundle(""); |
| | 2186 | defer errors.deinit(std.testing.allocator); |
| | 2187 | |
| | 2188 | const em = errors.getErrorMessage(errors.getMessages()[0]); |
| | 2189 | try std.testing.expectEqual(1, em.count); |
| | 2190 | if (notes_len > 0) { |
| | 2191 | try std.testing.expectEqual(notes_len, em.notes_len); |
| | 2192 | } |
| | 2193 | var al = std.ArrayList(u8).init(std.testing.allocator); |
| | 2194 | defer al.deinit(); |
| | 2195 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); |
| | 2196 | try std.testing.expectEqualStrings(msg, al.items); |
| | 2197 | } |
| | 2198 | }; |
Andrew Kelley <andrew@ziglang.org> 2024-04-09 13:22:48-07:00
GitHub <noreply@github.com> 2024-04-09 13:22:48-07:00
Signed by PGP key B5690EEEBB952194
lib