| ... | @@ -1193,14 +1193,14 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes | ... | @@ -1193,14 +1193,14 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes |
| 1193 | res.root_dir = try gpa.dupe(u8, root_dir); | 1193 | res.root_dir = try gpa.dupe(u8, root_dir); |
| 1194 | } | 1194 | } |
| 1195 | if (diagnostics.errors.items.len > 0) { | 1195 | if (diagnostics.errors.items.len > 0) { |
| | 1196 | try res.rootErrorMessage("unable to unpack tarball"); |
| 1196 | for (diagnostics.errors.items) |item| { | 1197 | for (diagnostics.errors.items) |item| { |
| 1197 | switch (item) { | 1198 | switch (item) { |
| 1198 | .unable_to_create_file => |i| try res.createFile(i.file_name, i.code), | 1199 | .unable_to_create_file => |i| try res.unableToCreateFile(i.file_name, i.code), |
| 1199 | .unable_to_create_sym_link => |i| try res.symLink(i.file_name, i.link_name, i.code), | 1200 | .unable_to_create_sym_link => |i| try res.unableToCreateSymLink(i.file_name, i.link_name, i.code), |
| 1200 | .unsupported_file_type => |i| try res.unsupportedFileType(i.file_name, @intFromEnum(i.file_type)), | 1201 | .unsupported_file_type => |i| try res.unsupportedFileType(i.file_name, @intFromEnum(i.file_type)), |
| 1201 | } | 1202 | } |
| 1202 | } | 1203 | } |
| 1203 | try res.rootErrorMessage("unable to unpack tarball"); | | |
| 1204 | } | 1204 | } |
| 1205 | return res; | 1205 | return res; |
| 1206 | } | 1206 | } |
| ... | @@ -1246,14 +1246,12 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac | ... | @@ -1246,14 +1246,12 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac |
| 1246 | try repository.checkout(out_dir, want_oid, &diagnostics); | 1246 | try repository.checkout(out_dir, want_oid, &diagnostics); |
| 1247 | | 1247 | |
| 1248 | if (diagnostics.errors.items.len > 0) { | 1248 | if (diagnostics.errors.items.len > 0) { |
| 1249 | defer res.deinit(); | 1249 | try res.rootErrorMessage("unable to unpack packfile"); |
| 1250 | | | |
| 1251 | for (diagnostics.errors.items) |item| { | 1250 | for (diagnostics.errors.items) |item| { |
| 1252 | switch (item) { | 1251 | switch (item) { |
| 1253 | .unable_to_create_sym_link => |i| try res.symLink(i.file_name, i.link_name, i.code), | 1252 | .unable_to_create_sym_link => |i| try res.unableToCreateSymLink(i.file_name, i.link_name, i.code), |
| 1254 | } | 1253 | } |
| 1255 | } | 1254 | } |
| 1256 | try res.rootErrorMessage("unable to unpack packfile"); | | |
| 1257 | } | 1255 | } |
| 1258 | } | 1256 | } |
| 1259 | } | 1257 | } |
| ... | @@ -1812,14 +1810,14 @@ const UnpackResult = struct { | ... | @@ -1812,14 +1810,14 @@ const UnpackResult = struct { |
| 1812 | return self.errors.items.len > 0; | 1810 | return self.errors.items.len > 0; |
| 1813 | } | 1811 | } |
| 1814 | | 1812 | |
| 1815 | fn createFile(self: *UnpackResult, file_name: []const u8, err: anyerror) !void { | 1813 | fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) !void { |
| 1816 | try self.errors.append(self.allocator, .{ .unable_to_create_file = .{ | 1814 | try self.errors.append(self.allocator, .{ .unable_to_create_file = .{ |
| 1817 | .code = err, | 1815 | .code = err, |
| 1818 | .file_name = try self.allocator.dupe(u8, file_name), | 1816 | .file_name = try self.allocator.dupe(u8, file_name), |
| 1819 | } }); | 1817 | } }); |
| 1820 | } | 1818 | } |
| 1821 | | 1819 | |
| 1822 | fn symLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) !void { | 1820 | fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) !void { |
| 1823 | try self.errors.append(self.allocator, .{ .unable_to_create_sym_link = .{ | 1821 | try self.errors.append(self.allocator, .{ .unable_to_create_sym_link = .{ |
| 1824 | .code = err, | 1822 | .code = err, |
| 1825 | .file_name = try self.allocator.dupe(u8, file_name), | 1823 | .file_name = try self.allocator.dupe(u8, file_name), |
| ... | @@ -1892,121 +1890,51 @@ const UnpackResult = struct { | ... | @@ -1892,121 +1890,51 @@ const UnpackResult = struct { |
| 1892 | }; | 1890 | }; |
| 1893 | | 1891 | |
| 1894 | test "fetch tarball: fail with unable to create file" { | 1892 | test "fetch tarball: fail with unable to create file" { |
| 1895 | const testing = std.testing; | | |
| 1896 | var buf: [4096]u8 = undefined; | | |
| 1897 | var buf_pos: usize = 0; | | |
| 1898 | | | |
| 1899 | // Create tmp dir | | |
| 1900 | var tmp = std.testing.tmpDir(.{}); | 1893 | var tmp = std.testing.tmpDir(.{}); |
| 1901 | defer tmp.cleanup(); | 1894 | defer tmp.cleanup(); |
| 1902 | const tmp_path = try tmp.dir.realpath(".", &buf); | | |
| 1903 | buf_pos += tmp_path.len; | | |
| 1904 | | 1895 | |
| 1905 | // Create tarball in tmp dir without build.zig.zon | | |
| 1906 | const tarball_name = "package.tar"; | 1896 | const tarball_name = "package.tar"; |
| 1907 | try createTestTarball(tmp.dir, tarball_name, false); | 1897 | try createTestTarball(tmp.dir, tarball_name, false); |
| 1908 | | 1898 | |
| 1909 | // Get path to the tarball | | |
| 1910 | const path_or_url = try std.fmt.bufPrint(buf[buf_pos..], "file://{s}/{s}", .{ tmp_path, tarball_name }); | | |
| 1911 | buf_pos += path_or_url.len; | | |
| 1912 | | | |
| 1913 | // Global cache directory in tmp | | |
| 1914 | const cache_path = try std.fmt.bufPrint(buf[buf_pos..], "{s}/{s}", .{ tmp_path, "global_cache" }); | | |
| 1915 | buf_pos += cache_path.len; | | |
| 1916 | | | |
| 1917 | // Run tarball fetch, expect to fail | 1899 | // Run tarball fetch, expect to fail |
| 1918 | var tf: TestFetch = undefined; | 1900 | var fb: TestFetchBuilder = undefined; |
| 1919 | try tf.init(testing.allocator, cache_path, path_or_url); | 1901 | var fetch = try fb.build(std.testing.allocator, tmp, tarball_name); |
| 1920 | defer tf.deinit(); | 1902 | defer fb.deinit(); |
| 1921 | try testing.expectError(error.FetchFailed, tf.fetch.run()); | 1903 | try std.testing.expectError(error.FetchFailed, fetch.run()); |
| 1922 | | 1904 | |
| 1923 | // Expect fetch errors | 1905 | try fb.expectFetchErrors(2, |
| 1924 | { | 1906 | \\error: unable to unpack tarball |
| 1925 | var errors = try tf.fetch.error_bundle.toOwnedBundle(""); | 1907 | \\ note: unable to create file 'dir/file': PathAlreadyExists |
| 1926 | defer errors.deinit(testing.allocator); | 1908 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| 1927 | | 1909 | \\ |
| 1928 | const em = errors.getErrorMessage(errors.getMessages()[0]); | 1910 | ); |
| 1929 | try testing.expectEqual(1, em.count); | | |
| 1930 | try testing.expectEqual(2, em.notes_len); | | |
| 1931 | | | |
| 1932 | var al = std.ArrayList(u8).init(testing.allocator); | | |
| 1933 | defer al.deinit(); | | |
| 1934 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); | | |
| 1935 | try testing.expectEqualStrings( | | |
| 1936 | \\error: unable to unpack tarball | | |
| 1937 | \\ note: unable to create file 'dir/file': PathAlreadyExists | | |
| 1938 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists | | |
| 1939 | \\ | | |
| 1940 | , al.items); | | |
| 1941 | } | | |
| 1942 | } | 1911 | } |
| 1943 | | 1912 | |
| 1944 | test "fetch tarball: error path are excluded" { | 1913 | test "fetch tarball: error path are excluded" { |
| 1945 | const testing = std.testing; | | |
| 1946 | var buf: [4096]u8 = undefined; | | |
| 1947 | var buf_pos: usize = 0; | | |
| 1948 | | | |
| 1949 | // Create tmp dir | | |
| 1950 | var tmp = std.testing.tmpDir(.{}); | 1914 | var tmp = std.testing.tmpDir(.{}); |
| 1951 | defer tmp.cleanup(); | 1915 | defer tmp.cleanup(); |
| 1952 | const tmp_path = try tmp.dir.realpath(".", &buf); | | |
| 1953 | buf_pos += tmp_path.len; | | |
| 1954 | | 1916 | |
| 1955 | // Create tarball in tmp dir | | |
| 1956 | const tarball_name = "package.tar"; | 1917 | const tarball_name = "package.tar"; |
| 1957 | try createTestTarball(tmp.dir, tarball_name, true); | 1918 | try createTestTarball(tmp.dir, tarball_name, true); |
| 1958 | | 1919 | |
| 1959 | // Get path to the tarball | 1920 | // Run tarball fetch, should succeed |
| 1960 | const path_or_url = try std.fmt.bufPrint(buf[buf_pos..], "file://{s}/{s}", .{ tmp_path, tarball_name }); | 1921 | var fb: TestFetchBuilder = undefined; |
| 1961 | buf_pos += path_or_url.len; | 1922 | var fetch = try fb.build(std.testing.allocator, tmp, tarball_name); |
| 1962 | | 1923 | defer fb.deinit(); |
| 1963 | // Global cache directory in tmp | 1924 | try fetch.run(); |
| 1964 | const cache_path = try std.fmt.bufPrint(buf[buf_pos..], "{s}/{s}", .{ tmp_path, "global_cache" }); | | |
| 1965 | buf_pos += cache_path.len; | | |
| 1966 | | 1925 | |
| 1967 | // Run tarball fetch | 1926 | const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); |
| 1968 | var tf: TestFetch = undefined; | 1927 | try std.testing.expectEqualStrings("122022afac878639d5ea6fcca14a123e21fd0395c1f2ef2c89017fa71390f73024af", &hex_digest); |
| 1969 | try tf.init(testing.allocator, cache_path, path_or_url); | | |
| 1970 | defer tf.deinit(); | | |
| 1971 | try tf.fetch.run(); | | |
| 1972 | | | |
| 1973 | const hex_digest = Package.Manifest.hexDigest(tf.fetch.actual_hash); | | |
| 1974 | try testing.expectEqualStrings("122022afac878639d5ea6fcca14a123e21fd0395c1f2ef2c89017fa71390f73024af", &hex_digest); | | |
| 1975 | | 1928 | |
| 1976 | const expected_files: []const []const u8 = &.{ | 1929 | const expected_files: []const []const u8 = &.{ |
| 1977 | "build.zig", | 1930 | "build.zig", |
| 1978 | "build.zig.zon", | 1931 | "build.zig.zon", |
| 1979 | "src/main.zig", | 1932 | "src/main.zig", |
| 1980 | }; | 1933 | }; |
| 1981 | // Unpacked package contains expected files | 1934 | try fb.expectPackageFiles(expected_files); |
| 1982 | { | | |
| 1983 | const package_path = try std.fmt.bufPrint(buf[buf_pos..], "global_cache/p/{s}", .{hex_digest}); | | |
| 1984 | buf_pos += package_path.len; | | |
| 1985 | var package_dir = try tmp.dir.openDir(package_path, .{ .iterate = true }); | | |
| 1986 | | | |
| 1987 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; | | |
| 1988 | defer actual_files.deinit(testing.allocator); | | |
| 1989 | defer for (actual_files.items) |file| testing.allocator.free(file); | | |
| 1990 | var walker = try package_dir.walk(testing.allocator); | | |
| 1991 | defer walker.deinit(); | | |
| 1992 | while (try walker.next()) |entry| { | | |
| 1993 | if (entry.kind != .file) continue; | | |
| 1994 | //std.debug.print("{s}\n", .{entry.path}); | | |
| 1995 | const path = try testing.allocator.dupe(u8, entry.path); | | |
| 1996 | errdefer testing.allocator.free(path); | | |
| 1997 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); | | |
| 1998 | try actual_files.append(testing.allocator, path); | | |
| 1999 | } | | |
| 2000 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { | | |
| 2001 | fn lessThan(_: void, a: []u8, b: []u8) bool { | | |
| 2002 | return std.mem.lessThan(u8, a, b); | | |
| 2003 | } | | |
| 2004 | }.lessThan); | | |
| 2005 | try testing.expectEqualDeep(expected_files, actual_files.items); | | |
| 2006 | } | | |
| 2007 | } | 1935 | } |
| 2008 | | 1936 | |
| 2009 | const TestFetch = struct { | 1937 | const TestFetchBuilder = struct { |
| 2010 | thread_pool: ThreadPool, | 1938 | thread_pool: ThreadPool, |
| 2011 | http_client: std.http.Client, | 1939 | http_client: std.http.Client, |
| 2012 | global_cache_directory: Cache.Directory, | 1940 | global_cache_directory: Cache.Directory, |
| ... | @@ -2014,36 +1942,30 @@ const TestFetch = struct { | ... | @@ -2014,36 +1942,30 @@ const TestFetch = struct { |
| 2014 | root_prog_node: *std.Progress.Node, | 1942 | root_prog_node: *std.Progress.Node, |
| 2015 | job_queue: Fetch.JobQueue, | 1943 | job_queue: Fetch.JobQueue, |
| 2016 | fetch: Fetch, | 1944 | fetch: Fetch, |
| 2017 | gpa: std.mem.Allocator, | | |
| 2018 | | 1945 | |
| 2019 | fn init( | 1946 | fn build(self: *TestFetchBuilder, allocator: std.mem.Allocator, tmp: std.testing.TmpDir, tarball_name: []const u8) !*Fetch { |
| 2020 | tf: *TestFetch, | 1947 | const cache_dir = try tmp.dir.makeOpenPath("zig-global-cache", .{}); |
| 2021 | gpa: std.mem.Allocator, | 1948 | const path_or_url = try std.fmt.allocPrint(allocator, "zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 2022 | global_cache_directory_path: []const u8, | 1949 | |
| 2023 | path_or_url: []const u8, | 1950 | try self.thread_pool.init(.{ .allocator = allocator }); |
| 2024 | ) !void { | 1951 | self.http_client = .{ .allocator = allocator }; |
| 2025 | try tf.thread_pool.init(.{ .allocator = gpa }); | 1952 | self.global_cache_directory = .{ .handle = cache_dir, .path = null }; |
| 2026 | tf.http_client = .{ .allocator = gpa }; | | |
| 2027 | tf.global_cache_directory = .{ | | |
| 2028 | .handle = try fs.cwd().makeOpenPath(global_cache_directory_path, .{}), | | |
| 2029 | .path = global_cache_directory_path, | | |
| 2030 | }; | | |
| 2031 | | 1953 | |
| 2032 | tf.progress = .{ .dont_print_on_dumb = true }; | 1954 | self.progress = .{ .dont_print_on_dumb = true }; |
| 2033 | tf.root_prog_node = tf.progress.start("Fetch", 0); | 1955 | self.root_prog_node = self.progress.start("Fetch", 0); |
| 2034 | | 1956 | |
| 2035 | tf.job_queue = .{ | 1957 | self.job_queue = .{ |
| 2036 | .http_client = &tf.http_client, | 1958 | .http_client = &self.http_client, |
| 2037 | .thread_pool = &tf.thread_pool, | 1959 | .thread_pool = &self.thread_pool, |
| 2038 | .global_cache = tf.global_cache_directory, | 1960 | .global_cache = self.global_cache_directory, |
| 2039 | .recursive = false, | 1961 | .recursive = false, |
| 2040 | .read_only = false, | 1962 | .read_only = false, |
| 2041 | .debug_hash = false, | 1963 | .debug_hash = false, |
| 2042 | .work_around_btrfs_bug = false, | 1964 | .work_around_btrfs_bug = false, |
| 2043 | }; | 1965 | }; |
| 2044 | | 1966 | |
| 2045 | tf.fetch = .{ | 1967 | self.fetch = .{ |
| 2046 | .arena = std.heap.ArenaAllocator.init(gpa), | 1968 | .arena = std.heap.ArenaAllocator.init(allocator), |
| 2047 | .location = .{ .path_or_url = path_or_url }, | 1969 | .location = .{ .path_or_url = path_or_url }, |
| 2048 | .location_tok = 0, | 1970 | .location_tok = 0, |
| 2049 | .hash_tok = 0, | 1971 | .hash_tok = 0, |
| ... | @@ -2051,8 +1973,8 @@ const TestFetch = struct { | ... | @@ -2051,8 +1973,8 @@ const TestFetch = struct { |
| 2051 | .lazy_status = .eager, | 1973 | .lazy_status = .eager, |
| 2052 | .parent_package_root = Cache.Path{ .root_dir = undefined }, | 1974 | .parent_package_root = Cache.Path{ .root_dir = undefined }, |
| 2053 | .parent_manifest_ast = null, | 1975 | .parent_manifest_ast = null, |
| 2054 | .prog_node = tf.root_prog_node, | 1976 | .prog_node = self.root_prog_node, |
| 2055 | .job_queue = &tf.job_queue, | 1977 | .job_queue = &self.job_queue, |
| 2056 | .omit_missing_hash_error = true, | 1978 | .omit_missing_hash_error = true, |
| 2057 | .allow_missing_paths_field = false, | 1979 | .allow_missing_paths_field = false, |
| 2058 | | 1980 | |
| ... | @@ -2066,9 +1988,11 @@ const TestFetch = struct { | ... | @@ -2066,9 +1988,11 @@ const TestFetch = struct { |
| 2066 | | 1988 | |
| 2067 | .module = null, | 1989 | .module = null, |
| 2068 | }; | 1990 | }; |
| | 1991 | return &self.fetch; |
| 2069 | } | 1992 | } |
| 2070 | | 1993 | |
| 2071 | fn deinit(self: *TestFetch) void { | 1994 | fn deinit(self: *TestFetchBuilder) void { |
| | 1995 | self.fetch.arena.child_allocator.free(self.fetch.location.path_or_url); |
| 2072 | self.fetch.deinit(); | 1996 | self.fetch.deinit(); |
| 2073 | self.job_queue.deinit(); | 1997 | self.job_queue.deinit(); |
| 2074 | self.root_prog_node.end(); | 1998 | self.root_prog_node.end(); |
| ... | @@ -2076,6 +2000,55 @@ const TestFetch = struct { | ... | @@ -2076,6 +2000,55 @@ const TestFetch = struct { |
| 2076 | self.http_client.deinit(); | 2000 | self.http_client.deinit(); |
| 2077 | self.thread_pool.deinit(); | 2001 | self.thread_pool.deinit(); |
| 2078 | } | 2002 | } |
| | 2003 | |
| | 2004 | fn packageDir(self: *TestFetchBuilder) !fs.Dir { |
| | 2005 | const root = self.fetch.package_root; |
| | 2006 | return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); |
| | 2007 | } |
| | 2008 | |
| | 2009 | fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { |
| | 2010 | var package_dir = try self.packageDir(); |
| | 2011 | defer package_dir.close(); |
| | 2012 | |
| | 2013 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; |
| | 2014 | defer actual_files.deinit(std.testing.allocator); |
| | 2015 | defer for (actual_files.items) |file| std.testing.allocator.free(file); |
| | 2016 | var walker = try package_dir.walk(std.testing.allocator); |
| | 2017 | defer walker.deinit(); |
| | 2018 | while (try walker.next()) |entry| { |
| | 2019 | if (entry.kind != .file) continue; |
| | 2020 | // std.debug.print("{s}\n", .{entry.path}); |
| | 2021 | const path = try std.testing.allocator.dupe(u8, entry.path); |
| | 2022 | errdefer std.testing.allocator.free(path); |
| | 2023 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); |
| | 2024 | try actual_files.append(std.testing.allocator, path); |
| | 2025 | } |
| | 2026 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { |
| | 2027 | fn lessThan(_: void, a: []u8, b: []u8) bool { |
| | 2028 | return std.mem.lessThan(u8, a, b); |
| | 2029 | } |
| | 2030 | }.lessThan); |
| | 2031 | |
| | 2032 | try std.testing.expectEqual(expected_files.len, actual_files.items.len); |
| | 2033 | for (expected_files, 0..) |file_name, i| { |
| | 2034 | try std.testing.expectEqualStrings(file_name, actual_files.items[i]); |
| | 2035 | } |
| | 2036 | try std.testing.expectEqualDeep(expected_files, actual_files.items); |
| | 2037 | } |
| | 2038 | |
| | 2039 | fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { |
| | 2040 | var errors = try self.fetch.error_bundle.toOwnedBundle(""); |
| | 2041 | defer errors.deinit(std.testing.allocator); |
| | 2042 | |
| | 2043 | const em = errors.getErrorMessage(errors.getMessages()[0]); |
| | 2044 | try std.testing.expectEqual(1, em.count); |
| | 2045 | try std.testing.expectEqual(notes_len, em.notes_len); |
| | 2046 | |
| | 2047 | var al = std.ArrayList(u8).init(std.testing.allocator); |
| | 2048 | defer al.deinit(); |
| | 2049 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); |
| | 2050 | try std.testing.expectEqualStrings(msg, al.items); |
| | 2051 | } |
| 2079 | }; | 2052 | }; |
| 2080 | | 2053 | |
| 2081 | fn createTestTarball(dir: fs.Dir, tarball_name: []const u8, with_manifest: bool) !void { | 2054 | fn createTestTarball(dir: fs.Dir, tarball_name: []const u8, with_manifest: bool) !void { |
Igor Anić <igor.anic@gmail.com> 2024-03-29 13:31:43+01:00
src