| ... | @@ -463,10 +463,6 @@ fn runResource( | ... | @@ -463,10 +463,6 @@ fn runResource( |
| 463 | | 463 | |
| 464 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); | 464 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); |
| 465 | defer unpack_result.deinit(); | 465 | defer unpack_result.deinit(); |
| 466 | if (unpack_result.hasErrors()) { | | |
| 467 | try unpack_result.bundleErrors(eb, try f.srcLoc(f.location_tok)); | | |
| 468 | return error.FetchFailed; | | |
| 469 | } | | |
| 470 | | 466 | |
| 471 | var pkg_path: Cache.Path = .{ | 467 | var pkg_path: Cache.Path = .{ |
| 472 | .root_dir = tmp_directory, | 468 | .root_dir = tmp_directory, |
| ... | @@ -491,10 +487,14 @@ fn runResource( | ... | @@ -491,10 +487,14 @@ fn runResource( |
| 491 | .include_paths = if (f.manifest) |m| m.paths else .{}, | 487 | .include_paths = if (f.manifest) |m| m.paths else .{}, |
| 492 | }; | 488 | }; |
| 493 | | 489 | |
| 494 | // TODO: | | |
| 495 | // If any error occurred for files that were ultimately excluded, those | 490 | // If any error occurred for files that were ultimately excluded, those |
| 496 | // errors should be ignored, such as failure to create symlinks that | 491 | // errors should be ignored, such as failure to create symlinks that |
| 497 | // weren't supposed to be included anyway. | 492 | // weren't supposed to be included anyway. |
| | 493 | try unpack_result.filterErrors(filter); |
| | 494 | if (unpack_result.hasErrors()) { |
| | 495 | try unpack_result.bundleErrors(eb, try f.srcLoc(f.location_tok)); |
| | 496 | return error.FetchFailed; |
| | 497 | } |
| 498 | | 498 | |
| 499 | // Apply the manifest's inclusion rules to the temporary directory by | 499 | // Apply the manifest's inclusion rules to the temporary directory by |
| 500 | // deleting excluded files. | 500 | // deleting excluded files. |
| ... | @@ -1701,7 +1701,7 @@ const ThreadPool = std.Thread.Pool; | ... | @@ -1701,7 +1701,7 @@ const ThreadPool = std.Thread.Pool; |
| 1701 | const WaitGroup = std.Thread.WaitGroup; | 1701 | const WaitGroup = std.Thread.WaitGroup; |
| 1702 | const Fetch = @This(); | 1702 | const Fetch = @This(); |
| 1703 | const git = @import("Fetch/git.zig"); | 1703 | const git = @import("Fetch/git.zig"); |
| 1704 | //const Package = @import("../Package.zig"); | 1704 | const Package = @import("../Package.zig"); |
| 1705 | const Manifest = Package.Manifest; | 1705 | const Manifest = Package.Manifest; |
| 1706 | const ErrorBundle = std.zig.ErrorBundle; | 1706 | const ErrorBundle = std.zig.ErrorBundle; |
| 1707 | const native_os = builtin.os.tag; | 1707 | const native_os = builtin.os.tag; |
| ... | @@ -1766,12 +1766,14 @@ const UnpackResult = struct { | ... | @@ -1766,12 +1766,14 @@ const UnpackResult = struct { |
| 1766 | file_type: u8, | 1766 | file_type: u8, |
| 1767 | }, | 1767 | }, |
| 1768 | | 1768 | |
| 1769 | fn excluded(self: Error, filter: Filter) bool { | 1769 | fn excluded(self: Error, filter: Filter, root_dir: []const u8) bool { |
| 1770 | switch (self) { | 1770 | const file_name = switch (self) { |
| 1771 | .unable_to_create_file => |info| return !filter.includePath(info.file_name), | 1771 | .unable_to_create_file => |info| info.file_name, |
| 1772 | .unable_to_create_sym_link => |info| return !filter.includePath(info.file_name), | 1772 | .unable_to_create_sym_link => |info| info.file_name, |
| 1773 | .unsupported_file_type => |info| return !filter.includePath(info.file_name), | 1773 | .unsupported_file_type => |info| info.file_name, |
| 1774 | } | 1774 | }; |
| | 1775 | |
| | 1776 | return !filter.includePath(stripRoot(file_name, root_dir)); |
| 1775 | } | 1777 | } |
| 1776 | | 1778 | |
| 1777 | fn free(self: Error, allocator: std.mem.Allocator) void { | 1779 | fn free(self: Error, allocator: std.mem.Allocator) void { |
| ... | @@ -1834,10 +1836,11 @@ const UnpackResult = struct { | ... | @@ -1834,10 +1836,11 @@ const UnpackResult = struct { |
| 1834 | | 1836 | |
| 1835 | fn filterErrors(self: *UnpackResult, filter: Filter) !void { | 1837 | fn filterErrors(self: *UnpackResult, filter: Filter) !void { |
| 1836 | var i = self.errors.items.len; | 1838 | var i = self.errors.items.len; |
| | 1839 | const root_dir: []const u8 = if (self.root_dir) |root_dir| root_dir else ""; |
| 1837 | while (i > 0) { | 1840 | while (i > 0) { |
| 1838 | i -= 1; | 1841 | i -= 1; |
| 1839 | const item = self.errors.items[i]; | 1842 | const item = self.errors.items[i]; |
| 1840 | if (item.excluded(filter)) { | 1843 | if (item.excluded(filter, root_dir)) { |
| 1841 | _ = self.errors.swapRemove(i); | 1844 | _ = self.errors.swapRemove(i); |
| 1842 | item.free(self.allocator); | 1845 | item.free(self.allocator); |
| 1843 | } | 1846 | } |
| ... | @@ -1888,132 +1891,236 @@ const UnpackResult = struct { | ... | @@ -1888,132 +1891,236 @@ const UnpackResult = struct { |
| 1888 | } | 1891 | } |
| 1889 | }; | 1892 | }; |
| 1890 | | 1893 | |
| 1891 | // Removing dependencies | 1894 | test "fetch tarball: fail with unable to create file" { |
| 1892 | const Package = struct { | | |
| 1893 | const build_zig_basename = "build.zig"; | | |
| 1894 | const Module = struct {}; | | |
| 1895 | const Manifest = @import("Manifest.zig"); | | |
| 1896 | }; | | |
| 1897 | | | |
| 1898 | test "fetch tarball" { | | |
| 1899 | const testing = std.testing; | 1895 | const testing = std.testing; |
| 1900 | const gpa = testing.allocator; | 1896 | var buf: [4096]u8 = undefined; |
| | 1897 | var buf_pos: usize = 0; |
| 1901 | | 1898 | |
| 1902 | var cache_tmp = std.testing.tmpDir(.{}); | 1899 | // Create tmp dir |
| 1903 | defer cache_tmp.cleanup(); | | |
| 1904 | const global_cache_directory_path = try cache_tmp.dir.realpathAlloc(gpa, "."); | | |
| 1905 | defer gpa.free(global_cache_directory_path); | | |
| 1906 | | | |
| 1907 | const paths: []const []const u8 = &.{ | | |
| 1908 | "main.zig", | | |
| 1909 | "src/root.zig", | | |
| 1910 | // duplicate file paths | | |
| 1911 | "dir/file", | | |
| 1912 | "dir1/file1", | | |
| 1913 | "dir/file", | | |
| 1914 | "dir1/file1", | | |
| 1915 | }; | | |
| 1916 | var tmp = std.testing.tmpDir(.{}); | 1900 | var tmp = std.testing.tmpDir(.{}); |
| 1917 | defer tmp.cleanup(); | 1901 | defer tmp.cleanup(); |
| 1918 | const file = try tmp.dir.createFile("package.tar", .{}); | 1902 | const tmp_path = try tmp.dir.realpath(".", &buf); |
| 1919 | try createTarball(file, "package", paths); | 1903 | buf_pos += tmp_path.len; |
| 1920 | file.close(); | | |
| 1921 | | | |
| 1922 | const tmp_path = try tmp.dir.realpathAlloc(gpa, "."); | | |
| 1923 | const path_or_url = try std.fmt.allocPrint( | | |
| 1924 | gpa, | | |
| 1925 | "file://{s}/package.tar", | | |
| 1926 | .{tmp_path}, | | |
| 1927 | ); | | |
| 1928 | gpa.free(tmp_path); | | |
| 1929 | defer gpa.free(path_or_url); | | |
| 1930 | | 1904 | |
| 1931 | var thread_pool: ThreadPool = undefined; | 1905 | // Create tarball in tmp dir without build.zig.zon |
| 1932 | try thread_pool.init(.{ .allocator = gpa }); | 1906 | const tarball_name = "package.tar"; |
| 1933 | defer thread_pool.deinit(); | 1907 | try createTestTarball(tmp.dir, tarball_name, false); |
| 1934 | | 1908 | |
| 1935 | var http_client: std.http.Client = .{ .allocator = gpa }; | 1909 | // Get path to the tarball |
| 1936 | defer http_client.deinit(); | 1910 | const path_or_url = try std.fmt.bufPrint(buf[buf_pos..], "file://{s}/{s}", .{ tmp_path, tarball_name }); |
| | 1911 | buf_pos += path_or_url.len; |
| 1937 | | 1912 | |
| 1938 | var global_cache_directory: Cache.Directory = .{ | 1913 | // Global cache directory in tmp |
| 1939 | .handle = try fs.cwd().makeOpenPath(global_cache_directory_path, .{}), | 1914 | const cache_path = try std.fmt.bufPrint(buf[buf_pos..], "{s}/{s}", .{ tmp_path, "global_cache" }); |
| 1940 | .path = global_cache_directory_path, | 1915 | buf_pos += cache_path.len; |
| 1941 | }; | 1916 | |
| 1942 | defer global_cache_directory.handle.close(); | 1917 | // Run tarball fetch, expect to fail |
| 1943 | | 1918 | var tf: TestFetch = undefined; |
| 1944 | var progress: std.Progress = .{ .dont_print_on_dumb = true }; | 1919 | try tf.init(testing.allocator, cache_path, path_or_url); |
| 1945 | const root_prog_node = progress.start("Fetch", 0); | 1920 | defer tf.deinit(); |
| 1946 | defer root_prog_node.end(); | 1921 | try testing.expectError(error.FetchFailed, tf.fetch.run()); |
| 1947 | | 1922 | |
| 1948 | var job_queue: Fetch.JobQueue = .{ | 1923 | // Expect fetch errors |
| 1949 | .http_client = &http_client, | 1924 | { |
| 1950 | .thread_pool = &thread_pool, | 1925 | var errors = try tf.fetch.error_bundle.toOwnedBundle(""); |
| 1951 | .global_cache = global_cache_directory, | 1926 | defer errors.deinit(testing.allocator); |
| 1952 | .recursive = false, | 1927 | |
| 1953 | .read_only = false, | 1928 | const em = errors.getErrorMessage(errors.getMessages()[0]); |
| 1954 | .debug_hash = true, | 1929 | try testing.expectEqual(1, em.count); |
| 1955 | .work_around_btrfs_bug = false, | 1930 | try testing.expectEqual(2, em.notes_len); |
| 1956 | }; | 1931 | |
| 1957 | defer job_queue.deinit(); | 1932 | var al = std.ArrayList(u8).init(testing.allocator); |
| 1958 | | 1933 | defer al.deinit(); |
| 1959 | var fetch: Fetch = .{ | 1934 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); |
| 1960 | .arena = std.heap.ArenaAllocator.init(gpa), | 1935 | try testing.expectEqualStrings( |
| 1961 | .location = .{ .path_or_url = path_or_url }, | 1936 | \\error: unable to unpack tarball |
| 1962 | .location_tok = 0, | 1937 | \\ note: unable to create file 'dir/file': PathAlreadyExists |
| 1963 | .hash_tok = 0, | 1938 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| 1964 | .name_tok = 0, | 1939 | \\ |
| 1965 | .lazy_status = .eager, | 1940 | , al.items); |
| 1966 | .parent_package_root = Cache.Path{ .root_dir = undefined }, | 1941 | } |
| 1967 | .parent_manifest_ast = null, | 1942 | } |
| 1968 | .prog_node = root_prog_node, | 1943 | |
| 1969 | .job_queue = &job_queue, | 1944 | test "fetch tarball: error path are excluded" { |
| 1970 | .omit_missing_hash_error = true, | 1945 | const testing = std.testing; |
| 1971 | .allow_missing_paths_field = false, | 1946 | var buf: [4096]u8 = undefined; |
| 1972 | | 1947 | var buf_pos: usize = 0; |
| 1973 | .package_root = undefined, | 1948 | |
| 1974 | .error_bundle = undefined, | 1949 | // Create tmp dir |
| 1975 | .manifest = null, | 1950 | var tmp = std.testing.tmpDir(.{}); |
| 1976 | .manifest_ast = undefined, | 1951 | defer tmp.cleanup(); |
| 1977 | .actual_hash = undefined, | 1952 | const tmp_path = try tmp.dir.realpath(".", &buf); |
| 1978 | .has_build_zig = false, | 1953 | buf_pos += tmp_path.len; |
| 1979 | .oom_flag = false, | 1954 | |
| 1980 | | 1955 | // Create tarball in tmp dir |
| 1981 | .module = null, | 1956 | const tarball_name = "package.tar"; |
| | 1957 | try createTestTarball(tmp.dir, tarball_name, true); |
| | 1958 | |
| | 1959 | // Get path to the tarball |
| | 1960 | const path_or_url = try std.fmt.bufPrint(buf[buf_pos..], "file://{s}/{s}", .{ tmp_path, tarball_name }); |
| | 1961 | buf_pos += path_or_url.len; |
| | 1962 | |
| | 1963 | // Global cache directory in tmp |
| | 1964 | const cache_path = try std.fmt.bufPrint(buf[buf_pos..], "{s}/{s}", .{ tmp_path, "global_cache" }); |
| | 1965 | buf_pos += cache_path.len; |
| | 1966 | |
| | 1967 | // Run tarball fetch |
| | 1968 | var tf: TestFetch = undefined; |
| | 1969 | try tf.init(testing.allocator, cache_path, path_or_url); |
| | 1970 | defer tf.deinit(); |
| | 1971 | try tf.fetch.run(); |
| | 1972 | |
| | 1973 | const hex_digest = Package.Manifest.hexDigest(tf.fetch.actual_hash); |
| | 1974 | try testing.expectEqualStrings("122022afac878639d5ea6fcca14a123e21fd0395c1f2ef2c89017fa71390f73024af", &hex_digest); |
| | 1975 | |
| | 1976 | const expected_files: []const []const u8 = &.{ |
| | 1977 | "build.zig", |
| | 1978 | "build.zig.zon", |
| | 1979 | "src/main.zig", |
| 1982 | }; | 1980 | }; |
| 1983 | defer fetch.deinit(); | 1981 | // Unpacked package contains expected files |
| 1984 | | 1982 | { |
| 1985 | try testing.expectError(error.FetchFailed, fetch.run()); | 1983 | const package_path = try std.fmt.bufPrint(buf[buf_pos..], "global_cache/p/{s}", .{hex_digest}); |
| 1986 | | 1984 | buf_pos += package_path.len; |
| 1987 | try testing.expectEqual(1, fetch.error_bundle.root_list.items.len); | 1985 | var package_dir = try tmp.dir.openDir(package_path, .{ .iterate = true }); |
| 1988 | var errors = try fetch.error_bundle.toOwnedBundle(""); | 1986 | |
| 1989 | defer errors.deinit(gpa); | 1987 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; |
| 1990 | | 1988 | defer actual_files.deinit(testing.allocator); |
| 1991 | const em = errors.getErrorMessage(errors.getMessages()[0]); | 1989 | defer for (actual_files.items) |file| testing.allocator.free(file); |
| 1992 | try testing.expectEqual(2, em.notes_len); | 1990 | var walker = try package_dir.walk(testing.allocator); |
| 1993 | | 1991 | defer walker.deinit(); |
| 1994 | var al = std.ArrayList(u8).init(gpa); | 1992 | while (try walker.next()) |entry| { |
| 1995 | defer al.deinit(); | 1993 | if (entry.kind != .file) continue; |
| 1996 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); | 1994 | //std.debug.print("{s}\n", .{entry.path}); |
| 1997 | try testing.expectEqualStrings( | 1995 | const path = try testing.allocator.dupe(u8, entry.path); |
| 1998 | \\error: unable to unpack tarball | 1996 | errdefer testing.allocator.free(path); |
| 1999 | \\ note: unable to create file 'dir/file': PathAlreadyExists | 1997 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); |
| 2000 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists | 1998 | try actual_files.append(testing.allocator, path); |
| 2001 | \\ | 1999 | } |
| 2002 | , al.items); | 2000 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { |
| | 2001 | fn lessThan(_: void, a: []u8, b: []u8) bool { |
| | 2002 | return std.mem.lessThan(u8, a, b); |
| | 2003 | } |
| | 2004 | }.lessThan); |
| | 2005 | try testing.expectEqualDeep(expected_files, actual_files.items); |
| | 2006 | } |
| 2003 | } | 2007 | } |
| 2004 | | 2008 | |
| 2005 | const TarHeader = std.tar.output.Header; | 2009 | const TestFetch = struct { |
| | 2010 | thread_pool: ThreadPool, |
| | 2011 | http_client: std.http.Client, |
| | 2012 | global_cache_directory: Cache.Directory, |
| | 2013 | progress: std.Progress, |
| | 2014 | root_prog_node: *std.Progress.Node, |
| | 2015 | job_queue: Fetch.JobQueue, |
| | 2016 | fetch: Fetch, |
| | 2017 | gpa: std.mem.Allocator, |
| | 2018 | |
| | 2019 | fn init( |
| | 2020 | tf: *TestFetch, |
| | 2021 | gpa: std.mem.Allocator, |
| | 2022 | global_cache_directory_path: []const u8, |
| | 2023 | path_or_url: []const u8, |
| | 2024 | ) !void { |
| | 2025 | try tf.thread_pool.init(.{ .allocator = gpa }); |
| | 2026 | tf.http_client = .{ .allocator = gpa }; |
| | 2027 | tf.global_cache_directory = .{ |
| | 2028 | .handle = try fs.cwd().makeOpenPath(global_cache_directory_path, .{}), |
| | 2029 | .path = global_cache_directory_path, |
| | 2030 | }; |
| | 2031 | |
| | 2032 | tf.progress = .{ .dont_print_on_dumb = true }; |
| | 2033 | tf.root_prog_node = tf.progress.start("Fetch", 0); |
| | 2034 | |
| | 2035 | tf.job_queue = .{ |
| | 2036 | .http_client = &tf.http_client, |
| | 2037 | .thread_pool = &tf.thread_pool, |
| | 2038 | .global_cache = tf.global_cache_directory, |
| | 2039 | .recursive = false, |
| | 2040 | .read_only = false, |
| | 2041 | .debug_hash = false, |
| | 2042 | .work_around_btrfs_bug = false, |
| | 2043 | }; |
| | 2044 | |
| | 2045 | tf.fetch = .{ |
| | 2046 | .arena = std.heap.ArenaAllocator.init(gpa), |
| | 2047 | .location = .{ .path_or_url = path_or_url }, |
| | 2048 | .location_tok = 0, |
| | 2049 | .hash_tok = 0, |
| | 2050 | .name_tok = 0, |
| | 2051 | .lazy_status = .eager, |
| | 2052 | .parent_package_root = Cache.Path{ .root_dir = undefined }, |
| | 2053 | .parent_manifest_ast = null, |
| | 2054 | .prog_node = tf.root_prog_node, |
| | 2055 | .job_queue = &tf.job_queue, |
| | 2056 | .omit_missing_hash_error = true, |
| | 2057 | .allow_missing_paths_field = false, |
| | 2058 | |
| | 2059 | .package_root = undefined, |
| | 2060 | .error_bundle = undefined, |
| | 2061 | .manifest = null, |
| | 2062 | .manifest_ast = undefined, |
| | 2063 | .actual_hash = undefined, |
| | 2064 | .has_build_zig = false, |
| | 2065 | .oom_flag = false, |
| | 2066 | |
| | 2067 | .module = null, |
| | 2068 | }; |
| | 2069 | } |
| 2006 | | 2070 | |
| 2007 | fn createTarball(file: fs.File, prefix: []const u8, paths: []const []const u8) !void { | 2071 | fn deinit(self: *TestFetch) void { |
| 2008 | for (paths) |path| { | 2072 | self.fetch.deinit(); |
| | 2073 | self.job_queue.deinit(); |
| | 2074 | self.root_prog_node.end(); |
| | 2075 | self.global_cache_directory.handle.close(); |
| | 2076 | self.http_client.deinit(); |
| | 2077 | self.thread_pool.deinit(); |
| | 2078 | } |
| | 2079 | }; |
| | 2080 | |
| | 2081 | fn createTestTarball(dir: fs.Dir, tarball_name: []const u8, with_manifest: bool) !void { |
| | 2082 | const file = try dir.createFile(tarball_name, .{}); |
| | 2083 | defer file.close(); |
| | 2084 | |
| | 2085 | const TarHeader = std.tar.output.Header; |
| | 2086 | const prefix = tarball_name; |
| | 2087 | |
| | 2088 | const files: []const []const u8 = &.{ |
| | 2089 | "build.zig", |
| | 2090 | "src/main.zig", |
| | 2091 | // duplicate file paths |
| | 2092 | "dir/file", |
| | 2093 | "dir1/file1", |
| | 2094 | "dir/file", |
| | 2095 | "dir1/file1", |
| | 2096 | }; |
| | 2097 | for (files) |path| { |
| 2009 | var hdr = TarHeader.init(); | 2098 | var hdr = TarHeader.init(); |
| 2010 | hdr.typeflag = .regular; | 2099 | hdr.typeflag = .regular; |
| 2011 | //if (prefix.len > 0) { | | |
| 2012 | try hdr.setPath(prefix, path); | 2100 | try hdr.setPath(prefix, path); |
| 2013 | // } else { | | |
| 2014 | // hdr.setName(path); | | |
| 2015 | // } | | |
| 2016 | try hdr.updateChecksum(); | 2101 | try hdr.updateChecksum(); |
| 2017 | try file.writeAll(std.mem.asBytes(&hdr)); | 2102 | try file.writeAll(std.mem.asBytes(&hdr)); |
| 2018 | } | 2103 | } |
| | 2104 | |
| | 2105 | if (with_manifest) { |
| | 2106 | const build_zig_zon = |
| | 2107 | \\ .{ |
| | 2108 | \\ .name = "fetch", |
| | 2109 | \\ .version = "0.0.0", |
| | 2110 | \\ .paths = .{ |
| | 2111 | \\ "src", |
| | 2112 | \\ "build.zig", |
| | 2113 | \\ "build.zig.zon" |
| | 2114 | \\ }, |
| | 2115 | \\ } |
| | 2116 | ; |
| | 2117 | var hdr = TarHeader.init(); |
| | 2118 | hdr.typeflag = .regular; |
| | 2119 | try hdr.setPath(prefix, "build.zig.zon"); |
| | 2120 | try hdr.setSize(build_zig_zon.len); |
| | 2121 | try hdr.updateChecksum(); |
| | 2122 | try file.writeAll(std.mem.asBytes(&hdr)); |
| | 2123 | try file.writeAll(build_zig_zon); |
| | 2124 | try file.writeAll(&[_]u8{0} ** (512 - build_zig_zon.len)); |
| | 2125 | } |
| 2019 | } | 2126 | } |
Igor Anić <igor.anic@gmail.com> 2024-03-29 01:25:57+01:00
src