diff --git a/build.zig b/build.zig index e97fac39188412a77f57f3559247cf07a40c25ed..19bf41dc607d3b93e96bf53598d9945f8bd5bcf4 100644 --- a/build.zig +++ b/build.zig @@ -175,6 +175,9 @@ pub fn build(b: *std.Build) !void { ".tar", // exclude files from lib/std/zip/testdata ".zip", + // exclude files from lib/compiler/Maker/Fetch/git/testdata + ".idx", + ".pack", // others "README.md", }, diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 4d4ceceea255b4c94681843e9e7f4c23c823c4e8..6fe474bee810bc95c0b6e6bcd6bfa063b60e8686 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -17,6 +17,10 @@ const log = std.log; const mem = std.mem; const process = std.process; const Color = std.zig.Color; +const EnvVar = std.zig.EnvVar; +const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename; +const allocPrint = std.fmt.allocPrint; +const stringToEnum = std.meta.stringToEnum; const Fuzz = @import("Maker/Fuzz.zig"); const Graph = @import("Maker/Graph.zig"); @@ -25,10 +29,11 @@ const Watch = @import("Maker/Watch.zig"); const WebServer = @import("Maker/WebServer.zig"); const ScannedConfig = @import("Maker/ScannedConfig.zig"); const PkgConfig = @import("Maker/PkgConfig.zig"); +const Fetch = @import("Maker/Fetch.zig"); +const Package = @import("Maker/Package.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, - .http_disable_tls = true, }; gpa: Allocator, @@ -100,6 +105,15 @@ const ErrorStyle = enum { const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; +/// Used to build the -M flags to pass to build-exe. +const CliModule = struct { + name: []const u8, + root_path: []const u8, + deps: Deps = .empty, + + const Deps = std.array_hash_map.String(*CliModule); +}; + pub fn main(init: process.Init.Minimal) !void { // The build runner is long-lived in the following use cases: // * `--watch` mode @@ -124,70 +138,54 @@ pub fn main(init: process.Init.Minimal) !void { const arena = arena_instance.allocator(); const args = try init.args.toSlice(arena); - - // skip my own exe name - var arg_idx: usize = 1; - - const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); - const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); - const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); - const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); - const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); - const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); + var arg_i: usize = 1; + const cmd_name = nextArgOrFatal(args, &arg_i); + const zig_lib_arg = prefixedArgOrFatal(args, &arg_i, "--zig-lib="); + const zig_exe_arg = prefixedArgOrFatal(args, &arg_i, "--zig="); + const global_cache_arg = prefixedArgOrFatal(args, &arg_i, "--global-cache="); + const seed_arg = prefixedArgOrFatal(args, &arg_i, "--seed="); const cwd: Dir = .cwd(); const zig_lib_directory: Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try cwd.openDir(io, zig_lib_dir, .{}), - }; - - const build_root_directory: Cache.Directory = .{ - .path = build_root, - .handle = try cwd.openDir(io, build_root, .{}), - }; - - const local_cache_directory: Cache.Directory = .{ - .path = local_cache_root, - .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), + .path = zig_lib_arg, + .handle = try cwd.openDir(io, zig_lib_arg, .{}), }; const global_cache_directory: Cache.Directory = .{ - .path = global_cache_root, - .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + .path = global_cache_arg, + .handle = try cwd.createDirPathOpen(io, global_cache_arg, .{}), }; var graph: Graph = .{ .io = io, .arena = arena, - .cache = .{ - .io = io, - .gpa = gpa, - .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), - .cwd = try process.currentPathAlloc(io, arena), - }, - .zig_exe = zig_exe, + .cache = undefined, + .zig_exe = zig_exe_arg, .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, - .local_cache_root = local_cache_directory, + .local_cache_root = undefined, .zig_lib_directory = zig_lib_directory, - .build_root_directory = build_root_directory, + .build_root_directory = undefined, + .random_seed = parseRandomSeed(seed_arg), }; - graph.cache.addPrefix(.{ .path = null, .handle = cwd }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); + const cmd = stringToEnum(enum { fetch, build }, cmd_name) orelse fatal("bad command name: {q}", .{ cmd_name }); + switch (cmd) { + .fetch => return cmdFetch( gpa, &graph, args[arg_i..]), + .build => {}, + } var step_names: std.ArrayList([]const u8) = .empty; var help_menu = false; var steps_menu = false; - var print_configuration = false; + var print_configuration: enum {none, zon, path} = .none; var override_install_prefix: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = null; var override_bin_dir: ?[]const u8 = null; var override_include_dir: ?[]const u8 = null; + var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(&graph.environ_map); + var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(&graph.environ_map); var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; var summary: ?Summary = null; @@ -201,39 +199,120 @@ pub fn main(init: process.Init.Minimal) !void { var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config = false; var run_args: ?[]const []const u8 = null; + var build_file: ?[]const u8 = null; - if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + var configure_argv: std.ArrayList([]const u8) = .empty; + var cached_passthru_configure: std.ArrayList(u32) = .empty; + var forks: std.ArrayList(Fork) = .empty; + var system_pkg_dir_path: ?[]const u8 = null; + var fetch_only = false; + var fetch_mode: Fetch.JobQueue.Mode = .needed; + var debug_target: ?[]const u8 = null; + var cache_poison: std.Build.Graph.CachePoison = .pure; + + if (EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (stringToEnum(ErrorStyle, str)) |style| { error_style = style; } } - if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + if (EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (stringToEnum(MultilineErrors, str)) |style| { multiline_errors = style; } } - while (nextArg(args, &arg_idx)) |arg| { + try configure_argv.ensureUnusedCapacity(arena, 16); + try cached_passthru_configure.ensureUnusedCapacity(arena, 16); + + _ = configure_argv.addOneAssumeCapacity(); // configurer executable + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", graph.zig_exe }; + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; + const conf_argv_index_build_root = configure_argv.items.len - 1; + + while (nextArg(args, &arg_i)) |arg| { if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try configure_argv.ensureUnusedCapacity(arena, 2); + if (mem.startsWith(u8, arg, "-D") or + mem.startsWith(u8, arg, "-fsys=") or + mem.startsWith(u8, arg, "-fno-sys=") or + mem.startsWith(u8, arg, "--release=") or + mem.eql(u8, arg, "--release")) + { + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--system")) { + system_pkg_dir_path = nextArgOrFatal(args, &arg_i); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. + continue; + } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { + color = stringToEnum(Color, rest) orelse + fatal("expected --color=[auto|on|off]; found {q}", .{arg}); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--cache-poison")) { + cache_poison = .poisoned; + configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); + continue; + } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { + // Allow the configurer process to report parse failure. + if (stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { + cache_poison = poison; + } + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--verbose")) { + // Intentionally is added both to make and configure but + // does not go into the cache hash. + configure_argv.appendAssumeCapacity(arg); + } else if (mem.eql(u8, arg, "--search-prefix")) { + const prefix = nextArgOrFatal(args, &arg_i); + // This argument is cache poisonous: it does not go into + // the cache and configurer must set the poison bit when + // choosing to observe it. + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, prefix }; + continue; + } else if (mem.eql(u8, arg, "--cache-dir")) { + override_local_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--pkg-dir")) { + override_pkg_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--fetch")) { + fetch_only = true; + } else if (mem.cutPrefix(u8, arg, "--fetch=")) |rest| { + fetch_only = true; + fetch_mode = stringToEnum(Fetch.JobQueue.Mode, rest) orelse + fatal("expected [needed|all] after \"--fetch=\", found {q}", .{rest}); + } else if (mem.cutPrefix(u8, arg, "--fork=")) |rest| { + try forks.append(arena, .init(rest)); + } else if (mem.eql(u8, arg, "--fork")) { + try forks.append(arena, .init(nextArgOrFatal(args, &arg_i))); + } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { help_menu = true; } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { steps_menu = true; } else if (mem.eql(u8, arg, "--print-configuration")) { - print_configuration = true; + print_configuration = .zon; + } else if (mem.eql(u8, arg, "--print-configuration-path")) { + print_configuration = .path; } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - override_install_prefix = nextArgOrFatal(args, &arg_idx); + override_install_prefix = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--build-file")) { + build_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - override_lib_dir = nextArgOrFatal(args, &arg_idx); + override_lib_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - override_bin_dir = nextArgOrFatal(args, &arg_idx); + override_bin_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - override_include_dir = nextArgOrFatal(args, &arg_idx); + override_include_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--sysroot")) { - graph.sysroot = nextArgOrFatal(args, &arg_idx); + graph.sysroot = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--maxrss")) { - const max_rss_text = nextArgOrFatal(args, &arg_idx); + const max_rss_text = nextArgOrFatal(args, &arg_i); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| fatal("invalid byte size {q}: {t}", .{ max_rss_text, err }); } else if (mem.eql(u8, arg, "--skip-oom-steps")) { @@ -253,7 +332,7 @@ pub fn main(init: process.Init.Minimal) !void { .{ "h", std.time.ns_per_hour }, .{ "hour", std.time.ns_per_hour }, }; - const timeout_str = nextArgOrFatal(args, &arg_idx); + const timeout_str = nextArgOrFatal(args, &arg_i); const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( "invalid timeout {q}: expected unit (ns, us, ms, s, m, h)", .{timeout_str}, @@ -274,50 +353,46 @@ pub fn main(init: process.Init.Minimal) !void { ); test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); } else if (mem.eql(u8, arg, "--search-prefix")) { - try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); + try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_i)); } else if (mem.eql(u8, arg, "--libc")) { - graph.libc_file = nextArgOrFatal(args, &arg_idx); + graph.libc_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected [auto|on|off] after {q}", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { + color = stringToEnum(Color, next_arg) orelse { fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ arg, next_arg, }); }; } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); - error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + error_style = stringToEnum(ErrorStyle, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); - multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + multiline_errors = stringToEnum(MultilineErrors, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected [all|new|failures|line|none] after {q}", .{arg}); - summary = std.meta.stringToEnum(Summary, next_arg) orelse { + summary = stringToEnum(Summary, next_arg) orelse { fatalWithHint("expected [all|new|failures|line|none] after {q}, found {q}", .{ arg, next_arg, }); }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after {q}", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed {q} as unsigned 32-bit integer: {t}", .{ next_arg, err }); - }; + } else if (mem.cutPrefix(u8, arg, "--seed=")) |rest| { + graph.random_seed = parseRandomSeed(rest); } else if (mem.eql(u8, arg, "--build-id")) { graph.build_id = .fast; } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { graph.build_id = std.zig.BuildId.parse(style) catch |err| fatal("unable to parse --build-id style {q}: {t}", .{ style, err }); } else if (mem.eql(u8, arg, "--debounce")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected u16 after {q}", .{arg}); debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { fatal("unable to parse debounce interval {q} as unsigned 16-bit integer: {t}", .{ @@ -333,8 +408,7 @@ pub fn main(init: process.Init.Minimal) !void { fatal("invalid web UI address {q}: {t}", .{ addr_str, err }); }; } else if (mem.eql(u8, arg, "--debug-log")) { - const next_arg = nextArgOrFatal(args, &arg_idx); - try graph.debug_log_scopes.append(arena, next_arg); + try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i)); } else if (mem.eql(u8, arg, "--debug-compile-errors")) { graph.debug_compile_errors = true; } else if (mem.eql(u8, arg, "--debug-incremental")) { @@ -344,19 +418,21 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = .Debug; } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + graph.debug_compiler_runtime_libs = stringToEnum(std.builtin.OptimizeMode, rest) orelse fatal("unrecognized optimization mode: {s}", .{rest}); } else if (is_debug_mode and mem.eql(u8, arg, "--debug-maker-leaks")) { debug_maker_leaks = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--verbose")) { graph.verbose = true; } else if (mem.eql(u8, arg, "--verbose-air")) { graph.verbose_air = true; } else if (mem.eql(u8, arg, "--verbose-cc")) { graph.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-link")) { + graph.verbose_link = true; } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { graph.verbose_llvm_ir = true; } else if (mem.eql(u8, arg, "--watch")) { @@ -439,7 +515,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-reference-trace")) { graph.reference_trace = null; } else if (mem.eql(u8, arg, "--error-limit")) { - const next_arg = nextArgOrFatal(args, &arg_idx); + const next_arg = nextArgOrFatal(args, &arg_i); graph.error_limit = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| fatal("unable to parse error limit {q}: {t}", .{ next_arg, err }); } else if (mem.cutPrefix(u8, arg, "-j")) |text| { @@ -449,7 +525,7 @@ pub fn main(init: process.Init.Minimal) !void { threaded.setAsyncLimit(.limited(n)); graph.max_jobs = n; } else if (mem.eql(u8, arg, "--")) { - run_args = argsRest(args, arg_idx); + run_args = argsRest(args, arg_i); break; } else { fatalWithHint("unrecognized argument: {s}", .{arg}); @@ -459,8 +535,40 @@ pub fn main(init: process.Init.Minimal) !void { } } - const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); - const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| + fatal("resolving current directory path failed: {t}", .{err}); + + const build_root = try findBuildRoot(arena, io, .{ + .cwd_path = cwd_path, + .build_file = build_file, + }); + + graph.build_root_directory = build_root.directory; + graph.local_cache_root = if (override_local_cache_dir) |unresolved_path| std.zig.Directories.openUnresolved( + arena, + io, + cwd_path, + unresolved_path, + .@"local_cache", + ) else .{ + .path = try Dir.path.join(arena, &.{build_root.directory.path orelse ".", default_local_zig_cache_basename}), + .handle = try build_root.directory.handle.createDirPathOpen(io, default_local_zig_cache_basename, .{}), + }; + graph.cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try graph.local_cache_root.handle.createDirPathOpen(io, "h", .{}), + .cwd = cwd_path, + }; + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(graph.build_root_directory); + graph.cache.addPrefix(zig_lib_directory); + graph.cache.addPrefix(graph.local_cache_root); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + const NO_COLOR = EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); graph.stderr_mode = switch (color) { .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), @@ -468,6 +576,555 @@ pub fn main(init: process.Init.Minimal) !void { .off => .no_color, }; + const main_progress_node = std.Progress.start(io, .{ + .disable_printing = (graph.stderr_mode.? == .no_color), + }); + defer main_progress_node.end(); + + { + // Cache lookup for configure options. If we get a match, we can skip + // execution of the configure script. If not, we get the file path to pass + // to the configure process. + var config_man = graph.cache.obtain(); + defer config_man.deinit(); + + for (cached_passthru_configure.items) |i| + config_man.hash.addBytes(configure_argv.items[i]); + + // Prevents a `zig build` from getting a false positive cache hit following + // a `zig build --cache-poison=ignored`. + config_man.hash.add(cache_poison == .ignored); + + // Normally the build runner is compiled for the host target but here is + // some code to help when debugging edits to the build runner so that you + // can make sure it compiles successfully on other targets. + const resolved_target: Package.Module.ResolvedTarget = t: { + if (debug_target) |triple| { + const target_query = try std.Target.Query.parse(.{ .arch_os_abi = triple }); + config_man.hash.addBytes(triple); + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, target_query), + .is_native_os = false, + .is_native_abi = false, + .is_explicit_dynamic_linker = false, + }; + } + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, .{}), + .is_native_os = true, + .is_native_abi = true, + .is_explicit_dynamic_linker = false, + }; + }; + + const pkg_root: Path = if (override_pkg_dir) |p| + .initCwd(p) + else if (system_pkg_dir_path) |p| + .initCwd(p) + else + .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }; + + configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; + + var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; + var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; + + { + // Populate fork_set. + var group: Io.Group = .init; + defer group.cancel(io); + + for (forks.items) |*fork| + group.async(io, Fork.load, .{ io, gpa, fork, color }); + + try group.await(io); + + for (forks.items) |*fork| { + if (fork.failed) process.exit(1); + try fork_set.put(arena, .{ + .path = fork.path, + .manifest_ast = fork.manifest_ast, + .manifest = fork.manifest, + .uses = 0, + }, {}); + } + } + defer Fork.deinitList(forks.items); + + var file_system_inputs: std.ArrayList(u8) = .empty; + defer file_system_inputs.deinit(gpa); + + var build_configurer_argv: std.ArrayList(u8) = .empty; + defer build_configurer_argv.deinit(gpa); + + var dependencies_source: std.ArrayList(u8) = .empty; + defer dependencies_source.deinit(gpa); + + const configurer_root_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "lib/compiler/configurer.zig", + }; + + const root_build_src_path: Cache.Path = .{ + .root_dir = build_root.directory, + .sub_path = build_root.build_zig_basename, + }; + + try build_configurer_argv.appendSlice(gpa, &.{ + graph.zig_exe, "build-exe", // + "--cache-dir", graph.local_cache_root.path orelse ".", // + "--global-cache-dir", graph.global_cache_root.path orelse ".", // + "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // + "--name", "configurer", // + "-fsingle-threaded", // + }); + if (graph.libc_file) |libc_file| { + try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file}); + } + if (graph.reference_trace) |n| { + try build_configurer_argv.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{n})); + } + if (graph.debug_compile_errors) { + try build_configurer_argv.append(gpa, "--debug-compile-errors"); + } + try build_configurer_argv.appendSlice(gpa, &.{ + "--dep", "@build", // + "--dep", "@dependencies", // + try allocPrint(arena, "-Mroot={f}", .{configurer_root_src_path}), // + try allocPrint(arena, "-M@build={f}", .{root_build_src_path}), // + }); + + // In the loop below, after doing the fetch operation, the argv will be + // truncated at this point, dependencies added, and then the + // "--listen=-" arg appended at the end. + const argv_deps_index = build_configurer_argv.items.len - 1; + + //const root_mod = try arena.create(CliModule); + //root_mod.* = .{ + // .name = "root", + // .root_path = try configurer_root_src_path.toString(arena), + //}; + + const build_mod = try arena.create(CliModule); + build_mod.* = .{ + .name = "@build", + .root_path = try root_build_src_path.toString(arena), + }; + defer build_mod.deps.deinit(gpa); + + const deps_mod = try arena.create(CliModule); + deps_mod.* = .{ + .name = "@dependencies", + .root_path = undefined, + }; + defer deps_mod.deps.deinit(gpa); + + // This loop is re-evaluated when the build script exits with an indication that it + // could not continue due to missing lazy dependencies. + const configuration_path: Path, const poisoned: bool = cp: while (true) { + //root_mod.deps.clearRetainingCapacity(); + build_mod.deps.clearRetainingCapacity(); + deps_mod.deps.clearRetainingCapacity(); + + // We want to release all the locks before executing the child process, so we make a nice + // big block here to ensure the cleanup gets run when we extract out our argv. + { + + + + { + const fetch_prog_node = main_progress_node.start("Fetch Packages", 0); + defer fetch_prog_node.end(); + + // Reset fork match counts. + for (fork_set.keys()) |*fork| fork.uses = 0; + + var job_queue: Package.Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = graph.global_cache_root, + .local_storage = &.{ + .cache_root = .{ .root_dir = graph.local_cache_root }, + .pkg_root = pkg_root, + }, + .recursive = true, + .debug_hash = false, + .unlazy_set = unlazy_set, + .fork_set = fork_set, + .mode = fetch_mode, + .prog_node = fetch_prog_node, + .read_only = system_pkg_dir_path != null, + }; + defer job_queue.deinit(); + + if (system_pkg_dir_path == null) { + try http_client.initDefaultProxies(arena, &graph.environ_map); + } + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); + + const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = phantom_package_root }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = phantom_package_root, + .parent_package_root = phantom_package_root, + .parent_manifest_ast = null, + .prog_node = fetch_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = true, + .oom_flag = false, + .latest_commit = null, + + .cli_module = build_mod, + }; + + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), + &fetch, + ); + + job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); + try job_queue.group.await(io); + + { + // Ensure that forks were actually used. This is done + // before printing manifest errors because using a fork can + // prevent them. + var any_unused = false; + for (fork_set.keys()) |*fork| { + if (fork.uses == 0) { + std.log.err("fork {f} matched no {s} packages", .{ + fork.path, fork.manifest.name, + }); + any_unused = true; + } else { + std.log.info("fork {f} matched {d} {s} packages", .{ + fork.path, fork.uses, fork.manifest.name, + }); + } + } + if (any_unused) process.exit(1); + } + + try job_queue.consolidateErrors(); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + // TODO when watching, watch and rebuild configure script rather than exit here + errors.renderToStderr(io, .{}, color) catch {}; + process.exit(1); + } + + if (fetch_only) return cleanExit(io); + + // Create the dependencies.zig file for configurer to + // obtain via `@import("@dependencies")`. + { + { + dependencies_source.clearRetainingCapacity(); + var source_writer: Io.Writer.Allocating = .fromArrayList(&dependencies_source); + defer dependencies_source = source_writer.toArrayList(); + job_queue.createDependenciesSource(&dependencies_source) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; + } + // Atomically create the file in a directory named after the hash of its contents. + var hh: Cache.HashHelper = .{}; + hh.addBytes(builtin.zig_version_string); + hh.addBytes(dependencies_source.items); + const hex_digest = hh.final(); + const dependencies_zig_path: Path = .{ + .root_dir = graph.local_cache_root, + .sub_path = try allocPrint(arena, "o/{s}/dependencies.zig", .{ &hex_digest }), + }; + var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic( + io, + dependencies_zig_path.sub_path, .{ .make_path = true, .replace = true }, + ); + defer atomic_file.deinit(io); + atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch |err| + fatal("writing dependencies.zig contents: {t}", .{err}); + atomic_file.replace(io) catch |err| + fatal("replacing {f}: {t}", .{dependencies_zig_path, err}); + + deps_mod.root_path = try dependencies_zig_path.toString(arena); + } + + { + // Add a CliModule for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(gpa, @intCast(hashes.len)); + for (hashes, fetches) |*hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + if (!f.has_build_zig) + continue; + const hash_slice = try arena.dupe(u8, hash.toSlice()); + + const m = try arena.create(CliModule); + m.* = .{ + .root_path = try f.package_root.toString(arena), + .name = hash_slice, + }; + deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m); + f.cli_module = m; + } + + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.cli_module orelse continue; + if (!f.have_manifest) continue; + const man = &f.manifest; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(gpa, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + global_cache_directory, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } + } + } + + // Lower module dependencies to CLI argv. + build_configurer_argv.shrinkRetainingCapacity(argv_deps_index); + for (deps_mod.deps.values()) |dep| { + try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1); + for (dep.deps.values()) |sub| { + build_configurer_argv.appendAssumeCapacity("--dep"); + build_configurer_argv.appendAssumeCapacity(sub.name); + } + build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}", .{ + dep.name, dep.root_path, + })); + } + try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * deps_mod.deps.count() + 1); + for (deps_mod.deps.values()) |dep| { + build_configurer_argv.appendAssumeCapacity("--dep"); + build_configurer_argv.appendAssumeCapacity(dep.name); + } + build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M@dependencies={s}", .{ + deps_mod.root_path, + })); + } + + const compile_prog_node = main_progress_node.start("Compile Configure Script", 0); + defer compile_prog_node.end(); + + try build_configurer_argv.append(gpa, "--listen=-"); + + file_system_inputs.clearRetainingCapacity(); + execute_child(build_configurer_argv, &file_system_inputs); + + const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); + const exe_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), + }; + _ = try config_man.addFilePath(exe_path, null); + configure_argv.items[0] = try exe_path.toString(arena); + + switch (cache_poison) { + .pure, .disallowed, .ignored => if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .{ + .root_dir = dirs.local_cache, + .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), + }, + false, + }; + }, + .poisoned => {}, // Don't bother checking for cache hit. + } + } + + if (!process.can_spawn) { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); + } + + const rand_int = randInt(io, u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + const config_tmp_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = tmp_dir_sub_path, + }; + const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( + io, + config_tmp_path.sub_path, + .{ .read = true, .exclusive = true }, + ); + defer config_tmp_file.close(io); + + const term = term: { + const child_node = main_progress_node.start("Run Configure Script", 0); + defer child_node.end(); + var child = std.process.spawn(io, .{ + .argv = configure_argv.items, + .stdout = .{ .file = config_tmp_file }, + .progress_node = child_node, + }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err }); + defer child.kill(io); + break :term child.wait(io) catch |err| + fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err }); + }; + if (!term.success()) { + // Failure to produce the configuration file. + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command {f}:\n{s}", .{ term, cmd }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); + + if (configuration.unlazy_deps.len != 0) { + if (!dev.env.supports(.fetch_command)) process.exit(1); + var any_errors = false; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); + any_errors = true; + continue; + } + try unlazy_set.put(arena, .fromSlice(hash), {}); + } + if (any_errors) process.exit(1); + if (system_pkg_dir_path) |p| { + // In this mode, the system needs to provide these packages; they + // cannot be fetched by Zig. + const s = fs.path.sep_str; + for (unlazy_set.keys()) |*hash| { + std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); + } + std.log.info("remote package fetching disabled due to --system mode", .{}); + std.log.info("dependencies might be avoidable depending on build configuration", .{}); + process.exit(1); + } + continue :cp; + } + + for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { + const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; + try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + } + + // We need to add to the configuration cache the source files of + // configurer itself, so that the maker process can watch the file system + // for those changes and restart itself. By doing this, we make it + // possible to bypass creating a Compilation for configurer on + // Configuration cache hit. + { + var it = mem.splitScalar(u8, file_system_inputs.items, 0); + while (it.next()) |input| { + _ = try config_man.addPrefixedPathPost(.{ + .prefix = input[0], + .sub_path = input[1..], + }); + } + } + + // If it is poisoned, there is no point in moving it to cached + // location. Just leave it in the tmp directory. + if (configuration.poisoned) { + break :cp .{ config_tmp_path, true }; + } else { + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), + }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| retry: { + const e = switch (err) { + error.FileNotFound => e: { + const dir_path = final_path.dirname().?; + dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| + fatal("failed to create directory {f}: {t}", .{ dir_path, e }); + if (Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + )) |_| break :retry else |e| break :e e; + }, + else => |e| e, + }; + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, e, + }); + }; + config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + break :cp .{ final_path, false }; + } + }; + + { + // Release all file system locks just before running the maker process. + var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; + defer if (configuration_lock) |*l| l.release(io); + + if (print_configuration_path) { + var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); + stdout_writer.interface.print("{f}\n", .{configuration_path}) catch + fatal("failed printing cache file path: {t}", .{stdout_writer.err.?}); + stdout_writer.flush() catch |err| + fatal("failed printing cache file path: {t}", .{err}); + return cleanExit(io); + } + const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); + + make_argv.items[0] = try make_runner.exe_path.toString(arena); + make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); + } + } + const scanned_config: ScannedConfig = sc: { const configuration = c: { var file = cwd.openFile(io, configure_path, .{}) catch |err| @@ -505,7 +1162,7 @@ pub fn main(init: process.Init.Minimal) !void { }; if (help_menu) { - var w = initStdoutWriter(io); + const w = initStdoutWriter(io); scanned_config.printUsage(&graph, w) catch |err| switch (err) { error.WriteFailed => return stdout_writer_allocation.err.?, else => |e| return e, @@ -513,18 +1170,24 @@ pub fn main(init: process.Init.Minimal) !void { w.flush() catch return stdout_writer_allocation.err.?; return cleanExit(io, &scanned_config); } else if (steps_menu) { - var w = initStdoutWriter(io); + const w = initStdoutWriter(io); scanned_config.printSteps(&graph, w) catch |err| switch (err) { error.WriteFailed => return stdout_writer_allocation.err.?, else => |e| return e, }; w.flush() catch return stdout_writer_allocation.err.?; return cleanExit(io, &scanned_config); - } else if (print_configuration) { - var w = initStdoutWriter(io); - scanned_config.print(w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return cleanExit(io, &scanned_config); + } else switch (print_configuration) { + .none => {}, + .zon => { + const w = initStdoutWriter(io); + scanned_config.print(w) catch return stdout_writer_allocation.err.?; + w.flush() catch return stdout_writer_allocation.err.?; + return cleanExit(io, &scanned_config); + }, + .path => { + @panic("TODO"); + }, } if (webui_listen != null) { @@ -532,11 +1195,6 @@ pub fn main(init: process.Init.Minimal) !void { if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); } - const main_progress_node = std.Progress.start(io, .{ - .disable_printing = (graph.stderr_mode.? == .no_color), - }); - defer main_progress_node.end(); - const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ .root_dir = .cwd(), .sub_path = try Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), @@ -706,6 +1364,338 @@ pub fn main(init: process.Init.Minimal) !void { } } +fn cmdFetch( + gpa: Allocator, + graph: *Graph, + args: []const []const u8 +) !void { + const environ_map = &graph.environ_map; + const io = graph.io; + const arena = graph.arena; + + const color: Color = Color.settingFromEnvironment(environ_map); + var opt_path_or_url: ?[]const u8 = null; + var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); + var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); + var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); + var debug_hash: bool = false; + var save: union(enum) { + no, + yes: ?[]const u8, + exact: ?[]const u8, + } = .no; + + var arg_i: usize = 0; + while (nextArg(args, &arg_i)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try Io.File.stdout().writeStreamingAll(io, usage_fetch); + return cleanExit(io); + } else if (mem.eql(u8, arg, "--global-cache-dir")) { + override_global_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--cache-dir")) { + override_local_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--pkg-dir")) { + override_pkg_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--debug-hash")) { + debug_hash = true; + } else if (mem.eql(u8, arg, "--debug-log")) { + try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i)); + } else if (mem.eql(u8, arg, "--save")) { + save = .{ .yes = null }; + } else if (mem.cutPrefix(u8, arg, "--save=")) |rest| { + save = .{ .yes = rest }; + } else if (mem.eql(u8, arg, "--save-exact")) { + save = .{ .exact = null }; + } else if (mem.cutPrefix(u8, arg, "--save-exact=")) |rest| { + save = .{ .exact = rest }; + } else { + fatal("unrecognized parameter: {q}", .{arg}); + } + } else if (opt_path_or_url != null) { + fatal("unexpected extra parameter: {q}", .{arg}); + } else { + opt_path_or_url = arg; + } + } + + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); + + var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + try http_client.initDefaultProxies(arena, environ_map); + + var root_prog_node = std.Progress.start(io, .{ + .root_name = "Fetch", + }); + defer root_prog_node.end(); + + var local_storage: Fetch.LocalStorage = undefined; + var build_root: BuildRoot = undefined; + var build_root_initialized = false; + defer if (build_root_initialized) build_root.deinit(io); + + const cwd_path = try std.zig.getResolvedCwd(io, arena); + + const local_storage_ptr = switch (save) { + .no => null, + .yes, .exact => ls: { + build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path }); + build_root_initialized = true; + + local_storage = .{ + .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{ + .root_dir = build_root.directory, + .sub_path = ".zig-cache", + }, + .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }, + }; + + break :ls &local_storage; + }, + }; + + var job_queue: Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = graph.global_cache_root, + .local_storage = local_storage_ptr, + .recursive = false, + .read_only = false, + .debug_hash = debug_hash, + .mode = .all, + .prog_node = root_prog_node, + }; + defer job_queue.deinit(); + + var fetch: Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = undefined, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + + .module = null, + }; + defer fetch.deinit(); + + fetch.run() catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.FetchFailed => {}, // error bundle checked below + }; + + try job_queue.group.await(io); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + process.exit(1); + } + + const package_hash = fetch.computedPackageHash(); + const package_hash_slice = package_hash.toSlice(); + + root_prog_node.end(); + root_prog_node = .{ .index = .none }; + + const name = switch (save) { + .no => { + var data: [2][]const u8 = .{ package_hash_slice, "\n" }; + const w = initStdoutWriter(); + try w.writeVecAll(&data); + try w.flush(); + return cleanExit(io); + }, + .yes, .exact => |name| name: { + if (name) |n| break :name n; + if (!fetch.have_manifest) + fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); + break :name fetch.manifest.name; + }, + }; + + // The name to use in case the manifest file needs to be created now. + const init_root_name = Dir.path.basename(build_root.directory.path orelse cwd_path); + var manifest, var ast = try loadManifest(gpa, arena, io, .{ + .root_name = try sanitizeExampleName(arena, init_root_name), + .dir = build_root.directory.handle, + .color = color, + }); + defer { + manifest.deinit(gpa); + ast.deinit(gpa); + } + + var fixups: Ast.Render.Fixups = .{}; + defer fixups.deinit(gpa); + + var saved_path_or_url = path_or_url; + + if (fetch.latest_commit) |latest_commit| resolved: { + const latest_commit_hex = try allocPrint(arena, "{f}", .{latest_commit}); + + var uri = try std.Uri.parse(path_or_url); + + if (uri.fragment) |fragment| { + const target_ref = try fragment.toRawMaybeAlloc(arena); + + // the refspec may already be fully resolved + if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; + + std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex }); + + // include the original refspec in a query parameter, could be used to check for updates + uri.query = .{ .percent_encoded = try allocPrint(arena, "ref={f}", .{ + std.fmt.alt(fragment, .formatEscaped), + }) }; + } else { + std.log.info("resolved to commit {s}", .{latest_commit_hex}); + } + + // replace the refspec with the resolved commit SHA + uri.fragment = .{ .raw = latest_commit_hex }; + + switch (save) { + .yes => saved_path_or_url = try allocPrint(arena, "{f}", .{uri}), + .no, .exact => {}, // keep the original URL + } + } + + const new_node_init = try allocPrint(arena, + \\.{{ + \\ .url = "{f}", + \\ .hash = "{f}", + \\ }} + , .{ + std.zig.fmtString(saved_path_or_url), + std.zig.fmtString(package_hash_slice), + }); + + const new_node_text = try allocPrint(arena, ".{f} = {s},\n", .{ + std.zig.fmtIdPU(name), new_node_init, + }); + + const dependencies_init = try allocPrint(arena, ".{{\n {s} }}", .{ + new_node_text, + }); + + const dependencies_text = try allocPrint(arena, ".dependencies = {s},\n", .{ + dependencies_init, + }); + + if (manifest.dependencies.get(name)) |dep| { + if (dep.hash) |h| { + switch (dep.location) { + .url => |u| { + if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { + std.log.info("existing dependency named {q} is up-to-date", .{name}); + process.exit(0); + } + }, + .path => {}, + } + } + + const location_replace = try allocPrint( + arena, + "\"{f}\"", + .{std.zig.fmtString(saved_path_or_url)}, + ); + const hash_replace = try allocPrint( + arena, + "\"{f}\"", + .{std.zig.fmtString(package_hash_slice)}, + ); + + warn("overwriting existing dependency named {q}", .{name}); + try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); + if (dep.hash_node.unwrap()) |hash_node| { + try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); + } else { + // https://github.com/ziglang/zig/issues/21690 + } + } else if (manifest.dependencies.count() > 0) { + // Add fixup for adding another dependency. + const deps = manifest.dependencies.values(); + const last_dep_node = deps[deps.len - 1].node; + try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); + } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { + // Add fixup for replacing the entire dependencies struct. + try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); + } else { + // Add fixup for adding dependencies struct. + try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); + } + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + try ast.render(gpa, &aw.writer, fixups); + const rendered = aw.written(); + + build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch |err| { + fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err }); + }; + + return cleanExit(io); +} + +const usage_fetch = + \\Usage: zig fetch [options] + \\Usage: zig fetch [options] + \\ + \\ Copy a package into the global cache and print its hash. + \\ must point to one of the following: + \\ - A git+http / git+https server for the package + \\ - A tarball file (with or without compression) containing + \\ package source + \\ - A git bundle file containing package source + \\ + \\Examples: + \\ + \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git + \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --cache-dir [path] Override path to local cache directory + \\ --pkg-dir [path] Override path to local package directory + \\ --debug-hash Print verbose hash information to stdout + \\ --debug-log [scope] Enable printing debug/info log messages for scope + \\ --save Add the fetched package to build.zig.zon + \\ --save=[name] Add the fetched package to build.zig.zon as name + \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim + \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim + \\ +; + +fn cmdBuild() !void { + +} + fn markFailedStepsDirty(maker: *Maker) void { const all_steps = maker.step_stack.keys(); @@ -1677,16 +2667,13 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { } fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); - }; + return nextArg(args, idx) orelse fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); } -fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { - const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); - if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); - const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); - return arg; +fn prefixedArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, prefix: []const u8) []const u8 { + const arg = args[index_ptr.*]; + if (mem.cutPrefix(u8, arg, prefix)) |rest| return rest; + fatal("expected {q} to begin with {q}", .{arg, prefix}); } fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { @@ -2006,11 +2993,11 @@ pub fn installSymLinks( const name = conf_comp.root_name.slice(c); const filename_major_only, const filename_name_only = if (os_tag.isDarwin()) .{ - try std.fmt.allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }), - try std.fmt.allocPrint(arena, "lib{s}.dylib", .{name}), + try allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }), + try allocPrint(arena, "lib{s}.dylib", .{name}), } else .{ - try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }), - try std.fmt.allocPrint(arena, "lib{s}.so", .{name}), + try allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }), + try allocPrint(arena, "lib{s}.so", .{name}), }; return installSymLinksInner(maker, arena, output_path, asking_step_index, filename_major_only, filename_name_only); @@ -2059,3 +3046,229 @@ inline fn debugMakerLeaks() bool { if (!is_debug_mode) return false; return debug_maker_leaks; } + +const BuildRoot = struct { + directory: Cache.Directory, + build_zig_basename: []const u8, + cleanup_build_dir: ?Io.Dir, + + fn deinit(br: *BuildRoot, io: Io) void { + if (br.cleanup_build_dir) |*dir| dir.close(io); + br.* = undefined; + } +}; + +const FindBuildRootOptions = struct { + build_file: ?[]const u8 = null, + cwd_path: ?[]const u8 = null, +}; + +fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot { + const cwd_path = options.cwd_path orelse try std.zig.getResolvedCwd(io, arena); + const build_zig_basename = if (options.build_file) |bf| + Dir.path.basename(bf) + else + std.zig.build_zig_basename; + + if (options.build_file) |bf| { + if (Dir.path.dirname(bf)) |dirname| { + const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { + fatal("failed opening directory containing {q}: {t}", .{ bf, err }); + }; + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ .path = dirname, .handle = dir }, + .cleanup_build_dir = dir, + }; + } + + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ .path = null, .handle = Io.Dir.cwd() }, + .cleanup_build_dir = null, + }; + } + // Search up parent directories until we find build.zig. + var dirname: []const u8 = cwd_path; + while (true) { + const joined_path = try Dir.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); + if (Io.Dir.cwd().access(io, joined_path, .{})) |_| { + const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { + fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err }); + }; + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ + .path = dirname, + .handle = dir, + }, + .cleanup_build_dir = dir, + }; + } else |err| switch (err) { + error.FileNotFound => { + dirname = Dir.path.dirname(dirname) orelse { + std.log.info("initialize {s} template file with \"zig init\"", .{ std.zig.build_zig_basename }); + std.log.info("see \"zig --help\" for more options", .{}); + fatal("no build.zig file found, in the current directory or any parent directories", .{}); + }; + continue; + }, + else => |e| return e, + } + } +} + +const Fork = struct { + path: Path, + manifest_ast: std.zig.Ast, + manifest: Package.Manifest, + error_bundle: std.zig.ErrorBundle.Wip, + failed: bool, + arena_allocator: std.heap.ArenaAllocator, + + fn init(cwd_relative_path: []const u8) Fork { + return .{ + .manifest_ast = undefined, + .manifest = undefined, + .error_bundle = undefined, + .arena_allocator = undefined, + .path = .{ + .root_dir = .cwd(), + .sub_path = cwd_relative_path, + }, + .failed = false, + }; + } + + fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { + loadFallible(io, gpa, fork, color) catch |err| switch (err) { + error.Canceled => |e| return e, + error.AlreadyReported => fork.failed = true, + else => |e| { + std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); + fork.failed = true; + }, + }; + } + + fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void { + fork.arena_allocator = .init(gpa); + const arena = fork.arena_allocator.allocator(); + + var error_bundle: std.zig.ErrorBundle.Wip = undefined; + try error_bundle.init(gpa); + defer error_bundle.deinit(); + + const manifest_path = try fork.path.join(arena, Package.Manifest.basename); + + Package.Manifest.load( + io, + arena, + manifest_path, + &fork.manifest_ast, + &error_bundle, + &fork.manifest, + true, + ) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ErrorsBundled => { + assert(error_bundle.root_list.items.len > 0); + var errors = try error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + return error.AlreadyReported; + }, + else => |e| { + std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); + return error.AlreadyReported; + }, + }; + } + + fn deinitList(forks: []Fork) void { + for (forks) |*fork| fork.arena_allocator.deinit(); + } +}; + +fn parseRandomSeed(arg: []const u8) u32 { + return std.fmt.parseUnsigned(u32, arg, 0) catch |err| + fatal("failed parsing random seed {q} as unsigned 32-bit integer: {t}", .{ arg, err }); +} + +fn randInt(io: Io, comptime T: type) T { + var x: T = undefined; + io.random(@ptrCast(&x)); + return x; +} + +const LoadManifestOptions = struct { + root_name: []const u8, + dir: Io.Dir, + color: Color, +}; + +fn loadManifest( + gpa: Allocator, + arena: Allocator, + io: Io, + options: LoadManifestOptions, +) !struct { Package.Manifest, std.zig.Ast } { + const rng: std.Random.IoSource = .{ .io = io }; + + const manifest_bytes = while (true) { + break options.dir.readFileAllocOptions( + io, + Package.Manifest.basename, + arena, + .limited(Package.Manifest.max_bytes), + .@"1", + 0, + ) catch |err| switch (err) { + error.FileNotFound => { + writeSimpleTemplateFile(io, Package.Manifest.basename, + \\.{{ + \\ .name = .{s}, + \\ .version = "{s}", + \\ .paths = .{{""}}, + \\ .fingerprint = 0x{x}, + \\}} + \\ + , .{ + options.root_name, + build_options.version, + Package.Fingerprint.generate(rng.interface(), options.root_name).int(), + }) catch |e| { + fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e }); + }; + continue; + }, + else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }), + }; + }; + var ast = try Ast.parse(gpa, manifest_bytes, .zon); + errdefer ast.deinit(gpa); + + if (ast.errors.len > 0) { + try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color); + process.exit(2); + } + + var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); + errdefer manifest.deinit(gpa); + + if (manifest.errors.len > 0) { + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + + const src_path = try wip_errors.addString(Package.Manifest.basename); + try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); + + var error_bundle = try wip_errors.toOwnedBundle(""); + defer error_bundle.deinit(gpa); + error_bundle.renderToStderr(io, .{}, options.color) catch {}; + + process.exit(2); + } + return .{ manifest, ast }; +} + diff --git a/lib/compiler/Maker/Fetch.zig b/lib/compiler/Maker/Fetch.zig new file mode 100644 index 0000000000000000000000000000000000000000..b6093f59b6992efba93f8327348b1145c905c41b --- /dev/null +++ b/lib/compiler/Maker/Fetch.zig @@ -0,0 +1,2286 @@ +//! Represents one independent job whose responsibility is to: +//! +//! 1. Check the local zig package directory to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 9. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. +//! 2. Check the global package cache for a compressed tarball matching the +//! hash. If it is found, unpack the contents into a temporary directory inside +//! project local zig cache. Rename this directory into the local zig package +//! directory and goto step 9, skipping step 10. +//! 3. Fetch and unpack a URL into a temporary directory. +//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed +//! for the file to be missing, in which case this fetched package is considered +//! to be a "naked" package. +//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by +//! deleting excluded files. If any files had errors for files that were +//! ultimately excluded, those errors should be ignored, such as failure to +//! create symlinks that weren't supposed to be included anyway. +//! 6. Compute the package hash based on the remaining files in the temporary +//! directory. +//! 7. Rename the temporary directory into the local zig package directory. If +//! the hash already exists, delete the temporary directory and leave the zig +//! package directory untouched as it may be in use. This is done even if +//! the hash is invalid, in case the package with the different hash is used +//! in the future. +//! 8. Validate the computed hash against the expected hash. If invalid, +//! this job is done. +//! 9. Spawn a new fetch job for each dependency in the manifest file. Use +//! a mutex and a hash map so that redundant jobs do not get queued up. +//! 10.Compress the package directory and store it into the global package +//! cache. +//! +//! All of this must be done with only referring to the state inside this struct +//! because this work will be done in a dedicated thread. +const Fetch = @This(); + +const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("std"); +const Io = std.Io; +const fs = std.fs; +const log = std.log.scoped(.fetch); +const assert = std.debug.assert; +const ascii = std.ascii; +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const git = @import("Fetch/git.zig"); +const Package = @import("../Package.zig"); +const Manifest = Package.Manifest; +const ErrorBundle = std.zig.ErrorBundle; + +arena: std.heap.ArenaAllocator, +location: Location, +location_tok: std.zig.Ast.TokenIndex, +hash_tok: std.zig.Ast.OptionalTokenIndex, +name_tok: std.zig.Ast.TokenIndex, +lazy_status: LazyStatus, +/// Same as `parent_packge_root` except it is unchanged when recursing into +/// relative file paths (as opposed to URL). +remote_package_root: Cache.Path, +parent_package_root: Cache.Path, +parent_manifest_ast: ?*const std.zig.Ast, +prog_node: std.Progress.Node, +job_queue: *JobQueue, +/// If true, don't add an error for a missing hash. This flag is not passed +/// down to recursive dependencies. It's intended to be used only be the CLI. +omit_missing_hash_error: bool, +/// If true, don't fail when a manifest file is missing the `paths` field, +/// which specifies inclusion rules. This is intended to be true for the first +/// fetch task and false for the recursive dependencies. +allow_missing_paths_field: bool, +/// If true and URL points to a Git repository, will use the latest commit. +use_latest_commit: bool, + +// Above this are fields provided as inputs to `run`. +// Below this are fields populated by `run`. + +/// Relative to the build root of the root package. +package_root: Cache.Path, +error_bundle: ErrorBundle.Wip, +manifest: Manifest, +manifest_ast: std.zig.Ast, +have_manifest: bool, +computed_hash: ComputedHash, +/// Fetch logic notices whether a package has a build.zig file and sets this flag. +has_build_zig: bool, +/// Indicates whether the task aborted due to an out-of-memory condition. +oom_flag: bool, +/// If `use_latest_commit` was true, this will be set to the commit that was used. +/// If the resource pointed to by the location is not a Git-repository, this +/// will be left unchanged. +latest_commit: ?git.Oid, + +// This field is used by the CLI only, untouched by this file. + +/// The module for this `Fetch` tasks's package, which exposes `build.zig` as +/// the root source file. +/// +/// This could be an opaque "userdata" field because this code does not observe +/// this data in any way but let's have some type safety because we can. +cli_module: ?*@import("../Maker.zig").CliModule, + +pub const LazyStatus = enum { + /// Not lazy. + eager, + /// Lazy, found. + available, + /// Lazy, not found. + unavailable, +}; + +pub const LocalStorage = struct { + cache_root: Cache.Path, + /// Path to "zig-pkg" inside the package in which the user ran `zig build`. + pkg_root: Cache.Path, +}; + +/// Contains shared state among all `Fetch` tasks. +pub const JobQueue = struct { + io: Io, + mutex: Io.Mutex = .init, + /// It's an array hash map so that it can be sorted before rendering the + /// dependencies.zig source file. + /// Protected by `mutex`. + table: Table = .{}, + /// `table` may be missing some tasks such as ones that failed, so this + /// field contains references to all of them. + /// Protected by `mutex`. + all_fetches: std.ArrayList(*Fetch) = .empty, + prog_node: std.Progress.Node, + + http_client: *std.http.Client, + /// This tracks `Fetch` tasks as well as recompression tasks. + group: Io.Group = .init, + global_cache: Cache.Directory, + /// If `null`, indicates fetch globally only. + local_storage: ?*const LocalStorage, + /// If true then, no fetching occurs, and: + /// * The `global_cache` directory is assumed to be the direct parent + /// directory of on-disk packages rather than having the "p/" directory + /// prefix inside of it. + /// * An error occurs if any non-lazy packages are not already present in + /// the package cache directory. + /// * Missing hash field causes an error, and no fetching occurs so it does + /// not print the correct hash like usual. + read_only: bool, + recursive: bool, + /// Dumps hash information to stdout which can be used to troubleshoot why + /// two hashes of the same package do not match. + /// If this is true, `recursive` must be false. + debug_hash: bool, + mode: Mode, + /// Set of hashes that will be additionally fetched even if they are marked + /// as lazy. + unlazy_set: UnlazySet = .{}, + /// Identifies paths that override all packages in the tree with matching + /// project ids. + fork_set: ForkSet = .{}, + + pub const Mode = enum { + /// Non-lazy dependencies are always fetched. + /// Lazy dependencies are fetched only when needed. + needed, + /// Both non-lazy and lazy dependencies are always fetched. + all, + }; + pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch); + pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void); + pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false); + + pub const Fork = struct { + path: Cache.Path, + manifest_ast: std.zig.Ast, + manifest: Package.Manifest, + uses: usize, + + pub const Context = struct { + pub fn hash(_: @This(), a: Fork) u32 { + const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); + return @truncate(project_id.hash()); + } + + pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool { + const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); + const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); + return a_project_id.eql(&b_project_id); + } + }; + + pub const Adapter = struct { + pub fn hash(_: @This(), a: Package.ProjectId) u32 { + return @truncate(a.hash()); + } + + pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool { + const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); + return a_project_id.eql(&b_project_id); + } + }; + }; + + pub fn deinit(jq: *JobQueue) void { + const io = jq.io; + jq.group.cancel(io); + if (jq.all_fetches.items.len == 0) return; + const gpa = jq.all_fetches.items[0].arena.child_allocator; + jq.table.deinit(gpa); + // These must be deinitialized in reverse order because subsequent + // `Fetch` instances are allocated in prior ones' arenas. + // Sorry, I know it's a bit weird, but it slightly simplifies the + // critical section. + while (jq.all_fetches.pop()) |f| f.deinit(); + jq.all_fetches.deinit(gpa); + jq.* = undefined; + } + + /// Dumps all subsequent error bundles into the first one. + pub fn consolidateErrors(jq: *JobQueue) !void { + const root = &jq.all_fetches.items[0].error_bundle; + const gpa = root.gpa; + for (jq.all_fetches.items[1..]) |fetch| { + if (fetch.error_bundle.root_list.items.len > 0) { + var bundle = try fetch.error_bundle.toOwnedBundle(""); + defer bundle.deinit(gpa); + try root.addBundleAsRoots(bundle); + } + } + } + + /// Creates the dependencies.zig source code for the build runner to obtain + /// via `@import("@dependencies")`. + pub fn createDependenciesSource(jq: *JobQueue, w: *Io.Writer) Io.Writer.Error!void { + const keys = jq.table.keys(); + + assert(keys.len != 0); // caller should have added the first one + if (keys.len == 1) { + // This is the first one. It must have no dependencies. + return createEmptyDependenciesSource(w); + } + + try w.writeAll("pub const packages = struct {\n"); + + // Ensure the generated .zig file is deterministic. + jq.table.sortUnstable(@as(struct { + keys: []const Package.Hash, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); + } + }, .{ .keys = keys })); + + for (keys, jq.table.values()) |*hash, fetch| { + if (fetch == jq.all_fetches.items[0]) { + // The first one is a dummy package for the current project. + continue; + } + + const hash_slice = hash.toSlice(); + + try w.print( + \\ pub const {f} = struct {{ + \\ + , .{std.zig.fmtId(hash_slice)}); + + lazy: { + switch (fetch.lazy_status) { + .eager => break :lazy, + .available => { + try w.writeAll( + \\ pub const available = true; + \\ + ); + break :lazy; + }, + .unavailable => { + try w.writeAll( + \\ pub const available = false; + \\ }; + \\ + ); + continue; + }, + } + } + + try w.print( + \\ pub const build_root = "{f}"; + \\ + , .{std.fmt.alt(fetch.package_root, .formatEscapeString)}); + + if (fetch.has_build_zig) { + try w.print( + \\ pub const build_zig = @import("{f}"); + \\ + , .{std.zig.fmtString(hash_slice)}); + } + + if (fetch.have_manifest) { + const manifest = &fetch.manifest; + try w.writeAll( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; + try w.print( + " .{{ \"{f}\", \"{f}\" }},\n", + .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, + ); + } + + try w.writeAll( + \\ }; + \\ }; + \\ + ); + } else { + try w.writeAll( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; + \\ }; + \\ + ); + } + } + + try w.writeAll( + \\}; + \\ + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + + const root_fetch = jq.all_fetches.items[0]; + assert(root_fetch.have_manifest); + const root_manifest = &root_fetch.manifest; + + for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { + const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; + try w.print( + " .{{ \"{f}\", \"{f}\" }},\n", + .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, + ); + } + try w.appendSlice("};\n"); + } + + pub fn createEmptyDependenciesSource(w: *Io.Writer) Io.Writer!void { + try w.writeAll( + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + } + + fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void { + const pkg_hash_slice = package_hash.toSlice(); + + const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice}); + defer prog_node.end(); + + var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; + const dest_path: Cache.Path = .{ + .root_dir = jq.global_cache, + .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable, + }; + + const gpa = jq.http_client.allocator; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ReadFailed => comptime unreachable, + error.WriteFailed => comptime unreachable, + else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), + }; + } + + fn recompressFallible( + jq: *JobQueue, + arena: Allocator, + dest_path: Cache.Path, + pkg_hash_slice: []const u8, + package_root: Cache.Path, + prog_node: std.Progress.Node, + ) !void { + const gpa = jq.http_client.allocator; + const io = jq.io; + + // We have to walk the file system up front in order to sort the file + // list for determinism purposes. The hash of the recompressed file is + // not critical because the true hash is based on the content alone. + // However, if we want Zig users to be able to share cached package + // data with each other via peer-to-peer protocols, we benefit greatly + // from the data being identical on everyone's computers. + var scanned_files: std.ArrayList(ScannedFile) = .empty; + defer scanned_files.deinit(gpa); + + var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true }); + defer pkg_dir.close(io); + + { + var walker = try pkg_dir.walk(gpa); + defer walker.deinit(); + + while (try walker.next(io)) |entry| { + const symlink = switch (entry.kind) { + .directory => continue, + .file => false, + .sym_link => true, + else => return error.IllegalFileType, + }; + const entry_path = try arena.dupe(u8, entry.path); + // If necessary, normalize path separators to POSIX-style since the tar format requires that. + if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) { + std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix); + } + try scanned_files.append(gpa, .{ + .ptr = entry_path.ptr, + .len = @intCast(entry_path.len), + .symlink = symlink, + }); + } + + std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp); + } + + prog_node.setEstimatedTotalItems(scanned_files.items.len); + + var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ + .make_path = true, + .replace = true, + }); + defer atomic_file.deinit(io); + + var file_write_buffer: [4096]u8 = undefined; + var file_writer = atomic_file.file.writer(io, &file_write_buffer); + + var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + }; + + var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; + archiver.prefix = pkg_hash_slice; + + var file_read_buffer: [4096]u8 = undefined; + var link_buf: [fs.max_path_bytes]u8 = undefined; + + for (scanned_files.items) |scanned_file| { + const entry_path = scanned_file.ptr[0..scanned_file.len]; + if (scanned_file.symlink) { + const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)]; + archiver.writeLink(entry_path, link_name, .{}) catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + else => |e| return e, + }; + } else { + var file = try pkg_dir.openFile(io, entry_path, .{}); + defer file.close(io); + var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer); + archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + error.WriteFailed => return file_writer.err.?, + else => |e| return e, + }; + } + prog_node.completeOne(); + } + + // intentionally omitting the pointless trailer + //try archiver.finish(); + compress.finish() catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + }; + try file_writer.flush(); + try atomic_file.replace(io); + } +}; + +const ScannedFile = struct { + ptr: [*]const u8, + len: u32, + symlink: bool, +}; + +fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool { + return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]); +} + +pub const Location = union(enum) { + remote: Remote, + /// A directory found inside the parent package. + relative_path: Cache.Path, + /// Recursive Fetch tasks will never use this Location, but it may be + /// passed in by the CLI. Indicates the file contents here should be copied + /// into the global package cache. It may be a file relative to the cwd or + /// absolute, in which case it should be treated exactly like a `file://` + /// URL, or a directory, in which case it should be treated as an + /// already-unpacked directory (but still needs to be copied into the + /// global package cache and have inclusion rules applied). + path_or_url: []const u8, + + pub const Remote = struct { + url: []const u8, + /// If this is null it means the user omitted the hash field from a dependency. + /// It will be an error but the logic should still fetch and print the discovered hash. + hash: ?Package.Hash, + }; +}; + +pub const RunError = error{ + OutOfMemory, + Canceled, + /// This error code is intended to be handled by inspecting the + /// `error_bundle` field. + FetchFailed, +}; + +pub fn run(f: *Fetch) RunError!void { + const job_queue = f.job_queue; + const io = job_queue.io; + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + + try eb.init(gpa); + + // Check the global zig package cache to see if the hash already exists. If + // so, load, parse, and validate the build.zig.zon file therein, and skip + // ahead to queuing up jobs for dependencies. Likewise if the location is a + // relative path, treat this the same as a cache hit. Otherwise, proceed. + + const remote = switch (f.location) { + .relative_path => |pkg_root| { + if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( + f.location_tok, + try eb.addString("expected path relative to build root; found absolute path"), + ); + if (f.hash_tok.unwrap()) |hash_tok| return f.fail( + hash_tok, + try eb.addString("path-based dependencies are not hashed"), + ); + // Packages fetched by URL may not use relative paths to escape outside the + // fetched package directory from within the package cache. + + // This code path is only reachable recursively and the sub_path + // will already have been resolved to no longer have extra ".." or + // "." components. + assert(job_queue.local_storage != null); + log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{ + pkg_root.sub_path, f.remote_package_root.sub_path, + }); + assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir)); + if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail( + f.location_tok, + try eb.printString("dependency path outside project: '{f}'", .{pkg_root}), + ); + f.package_root = pkg_root; + try loadManifest(f, pkg_root); + if (!f.has_build_zig) try checkBuildFileExistence(f); + if (!job_queue.recursive) return; + return queueJobsForDeps(f); + }, + .remote => |remote| remote, + .path_or_url => |path_or_url| { + if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| { + var resource: Resource = .{ .dir = dir }; + return f.runResource(path_or_url, &resource, null, false); + } else |dir_err| { + var server_header_buffer: [init_resource_buffer_size]u8 = undefined; + + const file_err = if (dir_err == error.NotDir) e: { + if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| { + var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) }; + return f.runResource(path_or_url, &resource, null, false); + } else |err| break :e err; + } else dir_err; + + const uri = std.Uri.parse(path_or_url) catch |uri_err| { + return f.fail(0, try eb.printString( + "'{s}' could not be recognized as a file path ({t}) or an URL ({t})", + .{ path_or_url, file_err, uri_err }, + )); + }; + var resource: Resource = undefined; + try f.initResource(uri, &resource, &server_header_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false); + } + }, + }; + + var resource_buffer: [init_resource_buffer_size]u8 = undefined; + + if (remote.hash) |expected_hash| { + const expected_project_id: Package.ProjectId = expected_hash.projectId(); + if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) |fork| { + log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name }); + fork.uses += 1; + f.package_root = fork.path; + f.remote_package_root = f.package_root; + f.manifest_ast = fork.manifest_ast; + f.manifest = fork.manifest; + f.have_manifest = true; + try checkBuildFileExistence(f); + if (!job_queue.recursive) return; + return queueJobsForDeps(f); + } + + if (job_queue.local_storage) |ls| { + const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice()); + if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { + assert(f.lazy_status != .unavailable); + f.package_root = package_root; + f.remote_package_root = f.package_root; + try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); + if (!job_queue.recursive) return; + return queueJobsForDeps(f); + } else |err| switch (err) { + error.FileNotFound => { + log.debug("FileNotFound: {f}", .{package_root}); + if (job_queue.read_only and f.lazy_status == .eager) return f.fail( + f.name_tok, + try eb.printString("package not found at '{f}'", .{package_root}), + ); + }, + error.Canceled => |e| return e, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ + package_root, e, + }), + }); + return error.FetchFailed; + }, + } + } + + // Check global cache before remote fetch. + const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()}); + const cached_tarball_path: Cache.Path = .{ + .root_dir = job_queue.global_cache, + .sub_path = cached_tarball_sub_path, + }; + if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| { + log.debug("found global cached tarball {f}", .{cached_tarball_path}); + var resource: Resource = .{ .file = file.reader(io, &resource_buffer) }; + return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true); + } else |err| switch (err) { + error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}), + error.Canceled => |e| return e, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{ + cached_tarball_path, e, + }), + }); + return error.FetchFailed; + }, + } + + switch (f.lazy_status) { + .eager => {}, + .available => if (!job_queue.unlazy_set.contains(expected_hash)) { + f.lazy_status = .unavailable; + return; + }, + .unavailable => unreachable, + } + } else if (job_queue.read_only) { + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + }); + return error.FetchFailed; + } + + // Fetch and unpack the remote into a temporary directory. + const uri = std.Uri.parse(remote.url) catch |err| return f.fail( + f.location_tok, + try eb.printString("invalid URI: {t}", .{err}), + ); + var resource: Resource = undefined; + try f.initResource(uri, &resource, &resource_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false); +} + +pub fn deinit(f: *Fetch) void { + f.error_bundle.deinit(); + f.arena.deinit(); +} + +/// Consumes `resource`, even if an error is returned. +fn runResource( + f: *Fetch, + uri_path: []const u8, + resource: *Resource, + remote_hash: ?Package.Hash, + disable_recompress: bool, +) RunError!void { + const job_queue = f.job_queue; + assert(!job_queue.read_only); + + const io = job_queue.io; + defer resource.deinit(io); + + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + const rand_int = r: { + var x: u64 = undefined; + io.random(@ptrCast(&x)); + break :r x; + }; + const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int); + const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path; + const tmp_directory_path: Cache.Path = if (job_queue.local_storage) |ls| + try ls.pkg_root.join(arena, tmp_dir_sub_path) + else + .{ + .root_dir = job_queue.global_cache, + .sub_path = tmp_tmp_dir_sub_path, + }; + + const package_sub_path = blk: { + var tmp_directory: Cache.Directory = .{ + .path = tmp_directory_path.sub_path, + .handle = handle: { + const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{ + .open_options = .{ .iterate = true }, + }) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{ + tmp_directory_path, err, + }), + }); + return error.FetchFailed; + }; + break :handle dir; + }, + }; + defer tmp_directory.handle.close(io); + + // Fetch and unpack a resource into a temporary directory. + var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); + + const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; + + // Load, parse, and validate the unpacked build.zig.zon file. It is allowed + // for the file to be missing, in which case this fetched package is + // considered to be a "naked" package. + try loadManifest(f, pkg_path); + + const filter: Filter = .{ + .include_paths = if (f.have_manifest) f.manifest.paths else .{}, + }; + + // Ignore errors that were excluded by manifest, such as failure to + // create symlinks that weren't supposed to be included anyway. + try unpack_result.validate(f, filter); + + // Apply the manifest's inclusion rules to the temporary directory by + // deleting excluded files. + // Empty directories have already been omitted by `unpackResource`. + // Compute the package hash based on the remaining files in the temporary + // directory. + f.computed_hash = try computeHash(f, pkg_path, filter); + + if (unpack_result.root_dir.len > 0) + break :blk try tmp_directory_path.join(arena, unpack_result.root_dir); + + break :blk tmp_directory_path; + }; + + const computed_package_hash = computedPackageHash(f); + + // Rename the temporary directory into the local zig package directory. If + // the hash already exists, delete the temporary directory and leave the + // zig package directory untouched as it may be in use. This is done even + // if the hash is invalid, in case the package with the different hash is + // used in the future. + if (job_queue.local_storage) |ls| { + f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice()); + renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "failed renaming temporary directory {f} into package cache directory {f}: {t}", + .{ package_sub_path, f.package_root, err }, + ) }); + return error.FetchFailed; + }; + } else { + f.package_root = tmp_directory_path; + } + f.remote_package_root = f.package_root; + + if (!disable_recompress) { + // Spin off a task to recompress the tarball, with filtered files deleted, into + // the global cache. + job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root }); + } + + // Remove temporary directory root if not already renamed to global cache. + if (!package_sub_path.eql(tmp_directory_path)) { + tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }), + }; + } + + // Validate the computed hash against the expected hash. If invalid, this + // job is done. + + if (remote_hash) |declared_hash| { + const hash_tok = f.hash_tok.unwrap().?; + if (!computed_package_hash.eql(&declared_hash)) { + return f.fail(hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, + )); + } + } else if (!f.omit_missing_hash_error) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), + })); + return error.FetchFailed; + } + + // Spawn a new fetch job for each dependency in the manifest file. Use + // a mutex and a hash map so that redundant jobs do not get queued up. + if (!job_queue.recursive) return; + return queueJobsForDeps(f); +} + +pub fn computedPackageHash(f: *const Fetch) Package.Hash { + const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); + if (f.have_manifest) { + const man = &f.manifest; + var version_buffer: [32]u8 = undefined; + const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer; + return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); + } + // In the future build.zig.zon fields will be added to allow overriding these values + // for naked tarballs. + return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); +} + +/// `computeHash` gets a free check for the existence of `build.zig`, but when +/// not computing a hash, we need to do a syscall to check for it. +fn checkBuildFileExistence(f: *Fetch) RunError!void { + const io = f.job_queue.io; + const eb = &f.error_bundle; + if (f.package_root.access(io, Package.build_zig_basename, .{})) |_| { + f.has_build_zig = true; + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to access '{f}{s}': {t}", .{ + f.package_root, Package.build_zig_basename, e, + }), + }); + return error.FetchFailed; + }, + } +} + +/// This function populates `f.manifest` or leaves it `null`. +fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { + const io = f.job_queue.io; + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const manifest_path = try pkg_root.join(arena, Manifest.basename); + + Manifest.load( + io, + arena, + manifest_path, + &f.manifest_ast, + eb, + &f.manifest, + f.allow_missing_paths_field, + ) catch |err| switch (err) { + error.FileNotFound => return, + error.Canceled => |e| return e, + error.ErrorsBundled => return error.FetchFailed, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }), + }); + return error.FetchFailed; + }, + }; + f.have_manifest = true; +} + +fn queueJobsForDeps(f: *Fetch) RunError!void { + const io = f.job_queue.io; + + assert(f.job_queue.recursive); + + // If the package does not have a build.zig.zon file then there are no dependencies. + if (!f.have_manifest) return; + const manifest = &f.manifest; + + const new_fetches, const prog_names = nf: { + const parent_arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + const dep_names = manifest.dependencies.keys(); + const deps = manifest.dependencies.values(); + // Grab the new tasks into a temporary buffer so we can unlock that mutex + // as fast as possible. + // This overallocates any fetches that get skipped by the `continue` in the + // loop below. + const new_fetches = try parent_arena.alloc(Fetch, deps.len); + const prog_names = try parent_arena.alloc([]const u8, deps.len); + var new_fetch_index: usize = 0; + + try f.job_queue.mutex.lock(io); + defer f.job_queue.mutex.unlock(io); + + try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); + + // There are four cases here: + // * Correct hash is provided by manifest. + // - Hash map already has the entry, no need to add it again. + // * Incorrect hash is provided by manifest. + // - Hash mismatch error emitted; `queueJobsForDeps` is not called. + // * Hash is not provided by manifest. + // - Hash missing error emitted; `queueJobsForDeps` is not called. + // * path-based location is used without a hash. + // - Hash is added to the table based on the path alone before + // calling run(); no need to add it again. + // + // If we add a dep as lazy and then later try to add the same dep as eager, + // eagerness takes precedence and the existing entry is updated and re-scheduled + // for fetching. + + for (dep_names, deps) |dep_name, dep| { + var promoted_existing_to_eager = false; + const new_fetch = &new_fetches[new_fetch_index]; + const location: Location = switch (dep.location) { + .url => |url| .{ + .remote = .{ + .url = url, + .hash = h: { + const h = dep.hash orelse break :h null; + const pkg_hash: Package.Hash = .fromSlice(h); + if (h.len == 0) break :h pkg_hash; + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); + if (gop.found_existing) { + if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { + gop.value_ptr.*.lazy_status = .eager; + promoted_existing_to_eager = true; + } else { + continue; + } + } + gop.value_ptr.* = new_fetch; + break :h pkg_hash; + }, + }, + }, + .path => |rel_path| l: { + // This might produce an invalid path, which is checked for + // at the beginning of run(). + const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); + const pkg_hash = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); + if (gop.found_existing) { + if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { + gop.value_ptr.*.lazy_status = .eager; + promoted_existing_to_eager = true; + } else { + continue; + } + } + gop.value_ptr.* = new_fetch; + break :l .{ .relative_path = new_root }; + }, + }; + prog_names[new_fetch_index] = dep_name; + new_fetch_index += 1; + if (!promoted_existing_to_eager) { + f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); + } + new_fetch.* = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = location, + .location_tok = dep.location_tok, + .hash_tok = dep.hash_tok, + .name_tok = dep.name_tok, + .lazy_status = switch (f.job_queue.mode) { + .needed => if (dep.lazy) .available else .eager, + .all => .eager, + }, + .parent_package_root = f.package_root, + .remote_package_root = f.remote_package_root, + .parent_manifest_ast = &f.manifest_ast, + .prog_node = f.prog_node, + .job_queue = f.job_queue, + .omit_missing_hash_error = false, + .allow_missing_paths_field = true, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + + .cli_module = null, + }; + } + + f.prog_node.increaseEstimatedTotalItems(new_fetch_index); + + break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; + }; + + // Now it's time to dispatch tasks. + for (new_fetches, prog_names) |*new_fetch, prog_name| { + f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name }); + } +} + +pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { + return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); +} + +pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void { + const prog_node = f.prog_node.start(prog_name, 0); + defer prog_node.end(); + + run(f) catch |err| switch (err) { + error.OutOfMemory => f.oom_flag = true, + error.Canceled => |e| return e, + error.FetchFailed => { + // Nothing to do because the errors are already reported in `error_bundle`, + // and a reference is kept to the `Fetch` task inside `all_fetches`. + }, + }; +} + +fn srcLoc( + f: *Fetch, + tok: std.zig.Ast.TokenIndex, +) Allocator.Error!ErrorBundle.SourceLocationIndex { + const ast = f.parent_manifest_ast orelse return .none; + const eb = &f.error_bundle; + const start_loc = ast.tokenLocation(0, tok); + const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); + const msg_off = 0; + return eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = ast.tokenStart(tok), + .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), + .span_main = ast.tokenStart(tok) + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }); +} + +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = msg_str, + .src_loc = try f.srcLoc(msg_tok), + }); + return error.FetchFailed; +} + +const Resource = union(enum) { + file: Io.File.Reader, + http_request: HttpRequest, + git: Git, + dir: Io.Dir, + + const Git = struct { + session: git.Session, + fetch_stream: git.Session.FetchStream, + want_oid: git.Oid, + }; + + const HttpRequest = struct { + request: std.http.Client.Request, + response: std.http.Client.Response, + transfer_buffer: []u8, + decompress: std.http.Decompress, + decompress_buffer: []u8, + }; + + fn deinit(resource: *Resource, io: Io) void { + switch (resource.*) { + .file => |*file_reader| file_reader.file.close(io), + .http_request => |*http_request| http_request.request.deinit(), + .git => |*git_resource| { + git_resource.fetch_stream.deinit(); + }, + .dir => |*dir| dir.close(io), + } + resource.* = undefined; + } + + fn reader(resource: *Resource) *Io.Reader { + return switch (resource.*) { + .file => |*file_reader| return &file_reader.interface, + .http_request => |*http_request| return http_request.response.readerDecompressing( + http_request.transfer_buffer, + &http_request.decompress, + http_request.decompress_buffer, + ), + .git => |*g| return &g.fetch_stream.reader, + .dir => unreachable, + }; + } +}; + +const FileType = enum { + tar, + @"tar.gz", + @"tar.xz", + @"tar.zst", + git_pack, + zip, + + fn fromPath(file_path: []const u8) ?FileType { + if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; + if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; + if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; + if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip; + return null; + } + + /// Parameter is a content-disposition header value. + fn fromContentDisposition(cd_header: []const u8) ?FileType { + const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse + return null; + + var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse + return null; + value_start += "filename".len; + if (cd_header[value_start] == '*') { + value_start += 1; + } + if (cd_header[value_start] != '=') return null; + value_start += 1; + + var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; + if (cd_header[value_end - 1] == '\"') { + value_end -= 1; + } + return fromPath(cd_header[value_start..value_end]); + } + + test fromContentDisposition { + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); + try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); + + try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; size=42") == null); + try std.testing.expect(fromContentDisposition("inline; size=42") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); + } +}; + +const init_resource_buffer_size = git.Packet.max_data_length; + +fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void { + const io = f.job_queue.io; + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + + if (ascii.eqlIgnoreCase(uri.scheme, "file")) { + const path = try uri.path.toRawMaybeAlloc(arena); + const file = f.parent_package_root.openFile(io, path, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{ + f.parent_package_root, path, err, + })); + }; + resource.* = .{ .file = file.reader(io, reader_buffer) }; + return; + } + + const http_client = f.job_queue.http_client; + + if (ascii.eqlIgnoreCase(uri.scheme, "http") or + ascii.eqlIgnoreCase(uri.scheme, "https")) + { + resource.* = .{ .http_request = .{ + .request = http_client.request(.GET, uri, .{}) catch |err| + return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})), + .response = undefined, + .transfer_buffer = reader_buffer, + .decompress_buffer = &.{}, + .decompress = undefined, + } }; + const request = &resource.http_request.request; + errdefer request.deinit(); + + request.sendBodiless() catch |err| + return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err})); + + var redirect_buffer: [8000]u8 = undefined; + const response = &resource.http_request.response; + response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) { + error.ReadFailed => { + return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{ + request.connection.?.getReadError().?, + })); + }, + else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})), + }; + + if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString( + "bad HTTP response code: '{d} {s}'", + .{ response.head.status, response.head.status.phrase() orelse "" }, + )); + + resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + return; + } + + if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or + ascii.eqlIgnoreCase(uri.scheme, "git+https")) + { + var transport_uri = uri; + transport_uri.scheme = uri.scheme["git+".len..]; + var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| { + return f.fail( + f.location_tok, + try eb.printString("unable to discover remote git server capabilities: {t}", .{err}), + ); + }; + + const want_oid = want_oid: { + const want_ref = + if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; + if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} + + const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); + const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); + + var ref_iterator: git.Session.RefIterator = undefined; + session.listRefs(&ref_iterator, .{ + .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, + .include_peeled = true, + .buffer = reader_buffer, + }) catch |err| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err})); + defer ref_iterator.deinit(); + while (ref_iterator.next() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to iterate refs: {s}", + .{@errorName(err)}, + )); + }) |ref| { + if (std.mem.eql(u8, ref.name, want_ref) or + std.mem.eql(u8, ref.name, want_ref_head) or + std.mem.eql(u8, ref.name, want_ref_tag)) + { + break :want_oid ref.peeled orelse ref.oid; + } + } + return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); + }; + if (f.use_latest_commit) { + f.latest_commit = want_oid; + } else if (uri.fragment == null) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("url field is missing an explicit ref"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("try .url = \"{f}#{f}\",", .{ + uri.fmt(.{ .scheme = true, .authority = true, .path = true }), + want_oid, + }), + })); + return error.FetchFailed; + } + + var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; + _ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable; + resource.* = .{ .git = .{ + .session = session, + .fetch_stream = undefined, + .want_oid = want_oid, + } }; + const fetch_stream = &resource.git.fetch_stream; + session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err})); + }; + errdefer fetch_stream.deinit(fetch_stream); + + return; + } + + return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme})); +} + +fn unpackResource( + f: *Fetch, + resource: *Resource, + uri_path: []const u8, + tmp_directory: Cache.Directory, +) RunError!UnpackResult { + const eb = &f.error_bundle; + const file_type = switch (resource.*) { + .file => FileType.fromPath(uri_path) orelse + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), + + .http_request => |*http_request| ft: { + const head = &http_request.response.head; + + // Content-Type takes first precedence. + const content_type = head.content_type orelse + return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); + + // Extract the MIME type, ignoring charset and boundary directives + const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; + const mime_type = content_type[0..mime_type_end]; + + if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) + break :ft .tar; + + if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or + ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or + ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or + ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or + ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) + { + break :ft .@"tar.gz"; + } + + if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) + break :ft .@"tar.xz"; + + if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) + break :ft .@"tar.zst"; + + if (ascii.eqlIgnoreCase(mime_type, "application/zip") or + ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or + ascii.eqlIgnoreCase(mime_type, "application/java-archive")) + { + break :ft .zip; + } + + if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and + !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) + { + return f.fail(f.location_tok, try eb.printString( + "unrecognized 'Content-Type' header: '{s}'", + .{content_type}, + )); + } + + // Next, the filename from 'content-disposition: attachment' takes precedence. + if (head.content_disposition) |cd_header| { + break :ft FileType.fromContentDisposition(cd_header) orelse { + return f.fail(f.location_tok, try eb.printString( + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + )); + }; + } + + // Finally, the path from the URI is used. + break :ft FileType.fromPath(uri_path) orelse { + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})); + }; + }, + + .git => .git_pack, + + .dir => |dir| { + f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{ + uri_path, err, + })); + }; + return .{}; + }, + }; + + switch (file_type) { + .tar => { + return unpackTarball(f, tmp_directory.handle, resource.reader()); + }, + .@"tar.gz" => { + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); + }, + .@"tar.xz" => { + const gpa = f.arena.child_allocator; + var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err| + return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err})); + defer decompress.deinit(); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); + }, + .@"tar.zst" => { + const window_len = std.compress.zstd.default_window_len; + const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max); + var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{ + .verify_checksum = false, + .window_len = window_len, + }); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); + }, + .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { + error.FetchFailed, error.OutOfMemory => |e| return e, + else => |e| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})), + }, + .zip => return unzip(f, tmp_directory.handle, resource.reader()) catch |err| switch (err) { + error.ReadFailed => return f.fail(f.location_tok, try eb.printString( + "failed reading resource: {t}", + .{err}, + )), + else => |e| return e, + }, + } +} + +fn unpackTarball(f: *Fetch, out_dir: Io.Dir, reader: *Io.Reader) RunError!UnpackResult { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const io = f.job_queue.io; + + var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; + + std.tar.pipeToFileSystem(io, out_dir, reader, .{ + .diagnostics = &diagnostics, + .strip_components = 0, + .mode_mode = .ignore, + .exclude_empty_directories = true, + }) catch |err| return f.fail( + f.location_tok, + try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}), + ); + + var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; + if (diagnostics.errors.items.len > 0) { + try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); + for (diagnostics.errors.items) |item| { + switch (item) { + .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), + .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), + .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), + .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 + } + } + } + return res; +} + +fn unzip( + f: *Fetch, + out_dir: Io.Dir, + reader: *Io.Reader, +) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult { + // We write the entire contents to a file first because zip files + // must be processed back to front and they could be too large to + // load into memory. + + const io = f.job_queue.io; + const cache_root = f.job_queue.global_cache; + const prefix = "tmp/"; + const suffix = ".zip"; + const eb = &f.error_bundle; + const random_len = @sizeOf(u64) * 2; + + var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined; + zip_path[0..prefix.len].* = prefix.*; + zip_path[prefix.len + random_len ..].* = suffix.*; + + var zip_file = while (true) { + const random_integer = r: { + var x: u64 = undefined; + io.random(@ptrCast(&x)); + break :r x; + }; + zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer); + + break cache_root.handle.createFile(io, &zip_path, .{ + .exclusive = true, + .read = true, + }) catch |err| switch (err) { + error.PathAlreadyExists => continue, + error.FileNotFound => { + cache_root.handle.createDir(io, prefix, .default_dir) catch |dir_err| switch (dir_err) { + error.Canceled => |e| return e, + // error.PathAlreadyExists is considered a failure here because + // it implies that the prefix is not a directory. + else => |e| return f.fail( + f.location_tok, + try eb.printString("failed to create temporary directory: {t}", .{e}), + ), + }; + continue; + }, + error.Canceled => |e| return e, + else => |e| return f.fail( + f.location_tok, + try eb.printString("failed to create temporary zip file: {t}", .{e}), + ), + }; + }; + defer zip_file.close(io); + var zip_file_buffer: [4096]u8 = undefined; + var zip_file_reader = b: { + var zip_file_writer = zip_file.writer(io, &zip_file_buffer); + + _ = reader.streamRemaining(&zip_file_writer.interface) catch |err| switch (err) { + error.ReadFailed => |e| return e, + error.WriteFailed => return f.fail( + f.location_tok, + try eb.printString("failed writing temporary zip file: {t}", .{err}), + ), + }; + zip_file_writer.interface.flush() catch |err| return f.fail( + f.location_tok, + try eb.printString("failed writing temporary zip file: {t}", .{err}), + ); + break :b zip_file_writer.moveToReader(); + }; + + var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; + // no need to deinit since we are using an arena allocator + + zip_file_reader.seekTo(0) catch |err| + return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err})); + std.zip.extract(out_dir, &zip_file_reader, .{ + .allow_backslashes = true, + .diagnostics = &diagnostics, + }) catch |err| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err})); + + cache_root.handle.deleteFile(io, &zip_path) catch |err| + return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err})); + + return .{ .root_dir = diagnostics.root_dir }; +} + +fn unpackGitPack(f: *Fetch, out_dir: Io.Dir, resource: *Resource.Git) anyerror!UnpackResult { + const io = f.job_queue.io; + const arena = f.arena.allocator(); + // TODO don't try to get a gpa from an arena. expose this dependency higher up + // because the backing of arena could be page allocator + const gpa = f.arena.child_allocator; + const object_format: git.Oid.Format = resource.want_oid; + + var res: UnpackResult = .{}; + // The .git directory is used to store the packfile and associated index, but + // we do not attempt to replicate the exact structure of a real .git + // directory, since that isn't relevant for fetching a package. + { + var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{}); + defer pack_dir.close(io); + var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true }); + defer pack_file.close(io); + var pack_file_buffer: [4096]u8 = undefined; + var pack_file_reader = b: { + var pack_file_writer = pack_file.writer(io, &pack_file_buffer); + const fetch_reader = &resource.fetch_stream.reader; + _ = try fetch_reader.streamRemaining(&pack_file_writer.interface); + try pack_file_writer.interface.flush(); + break :b pack_file_writer.moveToReader(); + }; + + var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true }); + defer index_file.close(io); + var index_file_buffer: [2000]u8 = undefined; + var index_file_writer = index_file.writer(io, &index_file_buffer); + { + const index_prog_node = f.prog_node.start("Index pack", 0); + defer index_prog_node.end(); + try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer); + } + + { + var index_file_reader = index_file.reader(io, &index_file_buffer); + const checkout_prog_node = f.prog_node.start("Checkout", 0); + defer checkout_prog_node.end(); + var repository: git.Repository = undefined; + try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader); + defer repository.deinit(); + var diagnostics: git.Diagnostics = .{ .allocator = arena }; + try repository.checkout(io, out_dir, resource.want_oid, &diagnostics); + + if (diagnostics.errors.items.len > 0) { + try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); + for (diagnostics.errors.items) |item| { + switch (item) { + .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), + .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), + } + } + } + } + } + + try out_dir.deleteTree(io, ".git"); + return res; +} + +fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void { + const gpa = f.arena.child_allocator; + const io = f.job_queue.io; + // Recursive directory copy. + var it = try dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + switch (entry.kind) { + .directory => {}, // omit empty directories + .file => { + dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); + try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}); + }, + else => |e| return e, + }; + }, + .sym_link => { + var buf: [fs.max_path_bytes]u8 = undefined; + const link_name = buf[0..try dir.readLink(io, entry.path, &buf)]; + // TODO: if this would create a symlink to outside + // the destination directory, fail with an error instead. + tmp_dir.symLink(io, link_name, entry.path, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); + try tmp_dir.symLink(io, link_name, entry.path, .{}); + }, + else => |e| return e, + }; + }, + else => return error.IllegalFileTypeInPackage, + } + } +} + +pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void { + var handled_missing_dir = false; + while (true) { + Io.Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dest_path.root_dir.handle, + dest_path.sub_path, + io, + ) catch |err| switch (err) { + error.FileNotFound => { + if (handled_missing_dir) return err; + const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?; + dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) { + error.PathAlreadyExists => handled_missing_dir = true, + else => |e| return e, + }; + continue; + }, + error.DirNotEmpty, error.AccessDenied => { + // Package has been already downloaded and may already be in use on the system. + tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) { + error.Canceled => |e| return e, + // Garbage files leftover in zig-cache/tmp/ is, as they say + // on Star Trek, "operating within normal parameters". + else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), + }; + }, + else => |e| return e, + }; + break; + } +} + +const ComputedHash = struct { + digest: Package.Hash.Digest, + total_size: u64, +}; + +/// Assumes that files not included in the package have already been filtered +/// prior to calling this function. This ensures that files not protected by +/// the hash are not present on the file system. Empty directories are *not +/// hashed* and must not be present on the file system when calling this +/// function. +fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { + const io = f.job_queue.io; + // All the path name strings need to be in memory for sorting. + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const root_dir = pkg_path.root_dir.handle; + + // Collect all files, recursively, then sort. + var all_files = std.array_list.Managed(*HashedFile).init(gpa); + defer all_files.deinit(); + + var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa); + defer deleted_files.deinit(); + + // Track directories which had any files deleted from them so that empty directories + // can be deleted. + var sus_dirs: std.array_hash_map.String(void) = .empty; + defer sus_dirs.deinit(gpa); + + var walker = try root_dir.walk(gpa); + defer walker.deinit(); + + // Total number of bytes of file contents included in the package. + var total_size: u64 = 0; + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var group: Io.Group = .init; + defer group.cancel(io); + + while (walker.next(io) catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to walk temporary directory '{f}': {t}", + .{ pkg_path, err }, + ) }); + return error.FetchFailed; + }) |entry| { + if (entry.kind == .directory) continue; + + const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); + if (!filter.includePath(entry_pkg_path)) { + // Delete instead of including in hash calculation. + const fs_path = try arena.dupe(u8, entry.path); + + // Also track the parent directory in case it becomes empty. + if (fs.path.dirname(fs_path)) |parent| + try sus_dirs.put(gpa, parent, {}); + + const deleted_file = try arena.create(DeletedFile); + deleted_file.* = .{ + .fs_path = fs_path, + .failure = undefined, // to be populated by the worker + }; + group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file }); + try deleted_files.append(deleted_file); + continue; + } + + const kind: HashedFile.Kind = switch (entry.kind) { + .directory => unreachable, + .file => .file, + .sym_link => .link, + else => return f.fail(f.location_tok, try eb.printString( + "package contains '{s}' which has illegal file type '{t}'", + .{ entry.path, entry.kind }, + )), + }; + + if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) + f.has_build_zig = true; + + const fs_path = try arena.dupe(u8, entry.path); + const hashed_file = try arena.create(HashedFile); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), + .kind = kind, + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + .size = undefined, // to be populated by the worker + }; + group.async(io, workerHashFile, .{ io, root_dir, hashed_file }); + try all_files.append(hashed_file); + } + + try group.await(io); + } + + { + // Sort by length, descending, so that child directories get removed first. + sus_dirs.sortUnstable(@as(struct { + keys: []const []const u8, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.keys[b_index].len < ctx.keys[a_index].len; + } + }, .{ .keys = sus_dirs.keys() })); + + // During this loop, more entries will be added, so we must loop by index. + var i: usize = 0; + while (i < sus_dirs.count()) : (i += 1) { + const sus_dir = sus_dirs.keys()[i]; + root_dir.deleteDir(io, sus_dir) catch |err| switch (err) { + error.DirNotEmpty => continue, + error.FileNotFound => continue, + else => |e| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to delete empty directory '{s}': {s}", + .{ sus_dir, @errorName(e) }, + ) }); + return error.FetchFailed; + }, + }; + if (fs.path.dirname(sus_dir)) |parent| { + try sus_dirs.put(gpa, parent, {}); + } + } + } + + std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Package.Hash.Algo.init(.{}); + var any_failures = false; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to hash '{s}': {s}", .{ + hashed_file.fs_path, @errorName(err), + }), + }); + }; + hasher.update(&hashed_file.hash); + total_size += hashed_file.size; + } + for (deleted_files.items) |deleted_file| { + deleted_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ + deleted_file.fs_path, @errorName(err), + }), + }); + }; + } + + if (any_failures) return error.FetchFailed; + + if (f.job_queue.debug_hash) { + assert(!f.job_queue.recursive); + // Print something to stdout that can be text diffed to figure out why + // the package hash is different. + dumpHashInfo(io, all_files.items) catch |err| + std.process.fatal("unable to write to stdout: {t}", .{err}); + } + + return .{ + .digest = hasher.finalResult(), + .total_size = total_size, + }; +} + +fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void { + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer); + dumpHashInfoWriter(&stdout_writer.interface, all_files) catch |err| switch (err) { + error.WriteFailed => return stdout_writer.err.?, + }; + try stdout_writer.flush(); +} + +fn dumpHashInfoWriter(w: *Io.Writer, all_files: []const *const HashedFile) Io.Writer.Error!void { + for (all_files) |hashed_file| { + try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path }); + } +} + +fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void { + hashed_file.failure = hashFileFallible(io, dir, hashed_file); +} + +fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void { + deleted_file.failure = deleteFileFallible(io, dir, deleted_file); +} + +fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void { + var buf: [8000]u8 = undefined; + var hasher = Package.Hash.Algo.init(.{}); + hasher.update(hashed_file.normalized_path); + var file_size: u64 = 0; + + switch (hashed_file.kind) { + .file => { + var file = try dir.openFile(io, hashed_file.fs_path, .{}); + defer file.close(io); + // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 + hasher.update(&.{ 0, 0 }); + var file_header: FileHeader = .{}; + while (true) { + const bytes_read = try file.readPositional(io, &.{&buf}, file_size); + if (bytes_read == 0) break; + file_size += bytes_read; + hasher.update(buf[0..bytes_read]); + file_header.update(buf[0..bytes_read]); + } + if (file_header.isExecutable()) { + try setExecutable(io, file); + } + }, + .link => { + const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)]; + if (fs.path.sep != canonical_sep) { + // Package hashes are intended to be consistent across + // platforms which means we must normalize path separators + // inside symlinks. + normalizePath(link_name); + } + hasher.update(link_name); + }, + } + hasher.final(&hashed_file.hash); + hashed_file.size = file_size; +} + +fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { + try dir.deleteFile(io, deleted_file.fs_path); +} + +fn setExecutable(io: Io, file: Io.File) !void { + if (!Io.File.Permissions.has_executable_bit) return; + try file.setPermissions(io, .executable_file); +} + +const DeletedFile = struct { + fs_path: []const u8, + failure: Error!void, + + const Error = + Io.Dir.DeleteFileError || + Io.Dir.DeleteDirError; +}; + +const HashedFile = struct { + fs_path: []const u8, + normalized_path: []const u8, + hash: Package.Hash.Digest, + failure: Error!void, + kind: Kind, + size: u64, + + const Error = + Io.File.OpenError || + Io.File.ReadPositionalError || + Io.File.StatError || + Io.File.SetPermissionsError || + Io.Dir.ReadLinkError; + + const Kind = enum { file, link }; + + fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { + _ = context; + return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); + } +}; + +/// Strips root directory name from file system path. +fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { + if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; + + if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { + return fs_path[root_dir.len + 1 ..]; + } + + return fs_path; +} + +/// Make a file system path identical independently of operating system path inconsistencies. +/// This converts backslashes into forward slashes. +fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { + const normalized = try arena.dupe(u8, pkg_path); + if (fs.path.sep == canonical_sep) return normalized; + normalizePath(normalized); + return normalized; +} + +const canonical_sep = fs.path.sep_posix; + +fn normalizePath(bytes: []u8) void { + assert(fs.path.sep != canonical_sep); + std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); +} + +const Filter = struct { + include_paths: std.array_hash_map.String(void) = .empty, + + /// sub_path is relative to the package root. + pub fn includePath(self: *const Filter, sub_path: []const u8) bool { + if (self.include_paths.count() == 0) return true; + if (self.include_paths.contains("")) return true; + if (self.include_paths.contains(".")) return true; + if (self.include_paths.contains(sub_path)) return true; + + // Check if any included paths are parent directories of sub_path. + var dirname = sub_path; + while (std.fs.path.dirname(dirname)) |next_dirname| { + if (self.include_paths.contains(next_dirname)) return true; + dirname = next_dirname; + } + + return false; + } + + test includePath { + const gpa = std.testing.allocator; + var filter: Filter = .{}; + defer filter.include_paths.deinit(gpa); + + try filter.include_paths.put(gpa, "src", {}); + try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); + try std.testing.expect(!filter.includePath(".gitignore")); + } +}; + +pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { + if (dep.hash) |h| return .fromSlice(h); + + switch (dep.location) { + .url => return null, + .path => |rel_path| { + var buf: [fs.max_path_bytes]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch + return null; + return relativePathDigest(new_root, cache_root); + }, + } +} + +// Detects executable header: ELF or Macho-O magic header or shebang line. +const FileHeader = struct { + header: [4]u8 = undefined, + bytes_read: usize = 0, + + pub fn update(self: *FileHeader, buf: []const u8) void { + if (self.bytes_read >= self.header.len) return; + const n = @min(self.header.len - self.bytes_read, buf.len); + @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); + self.bytes_read += n; + } + + fn isScript(self: *FileHeader) bool { + const shebang = "#!"; + return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); + } + + fn isElf(self: *FileHeader) bool { + const elf_magic = std.elf.MAGIC; + return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); + } + + fn isMachO(self: *FileHeader) bool { + if (self.bytes_read < 4) return false; + const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); + return magic_number == std.macho.MH_MAGIC or + magic_number == std.macho.MH_MAGIC_64 or + magic_number == std.macho.FAT_MAGIC or + magic_number == std.macho.FAT_MAGIC_64 or + magic_number == std.macho.MH_CIGAM or + magic_number == std.macho.MH_CIGAM_64 or + magic_number == std.macho.FAT_CIGAM or + magic_number == std.macho.FAT_CIGAM_64; + } + + pub fn isExecutable(self: *FileHeader) bool { + return self.isScript() or self.isElf() or self.isMachO(); + } +}; + +test FileHeader { + var h: FileHeader = .{}; + try std.testing.expect(!h.isExecutable()); + + const elf_magic = std.elf.MAGIC; + h.update(elf_magic[0..2]); + try std.testing.expect(!h.isExecutable()); + h.update(elf_magic[2..4]); + try std.testing.expect(h.isExecutable()); + + h.update(elf_magic[2..4]); + try std.testing.expect(h.isExecutable()); + + const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; + h.bytes_read = 0; + h.update(&macho64_magic_bytes); + try std.testing.expect(h.isExecutable()); + + const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; + h.bytes_read = 0; + h.update(&macho64_cigam_bytes); + try std.testing.expect(h.isExecutable()); +} + +// Result of the `unpackResource` operation. Enables collecting errors from +// tar/git diagnostic, filtering that errors by manifest inclusion rules and +// emitting remaining errors to an `ErrorBundle`. +const UnpackResult = struct { + errors: []Error = undefined, + errors_count: usize = 0, + root_error_message: []const u8 = "", + + // A non empty value means that the package contents are inside a + // sub-directory indicated by the named path. + root_dir: []const u8 = "", + + const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, + unsupported_file_type: struct { + file_name: []const u8, + file_type: u8, + }, + + fn excluded(self: Error, filter: Filter) bool { + const file_name = switch (self) { + .unable_to_create_file => |info| info.file_name, + .unable_to_create_sym_link => |info| info.file_name, + .unsupported_file_type => |info| info.file_name, + }; + return !filter.includePath(file_name); + } + }; + + fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { + self.root_error_message = try arena.dupe(u8, root_error_message); + self.errors = try arena.alloc(UnpackResult.Error, n); + } + + fn hasErrors(self: *UnpackResult) bool { + return self.errors_count > 0; + } + + fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { + self.errors[self.errors_count] = .{ .unable_to_create_file = .{ + .code = err, + .file_name = file_name, + } }; + self.errors_count += 1; + } + + fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { + self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ + .code = err, + .file_name = file_name, + .link_name = link_name, + } }; + self.errors_count += 1; + } + + fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { + self.errors[self.errors_count] = .{ .unsupported_file_type = .{ + .file_name = file_name, + .file_type = file_type, + } }; + self.errors_count += 1; + } + + fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { + if (self.errors_count == 0) return; + + var unfiltered_errors: u32 = 0; + for (self.errors) |item| { + if (item.excluded(filter)) continue; + unfiltered_errors += 1; + } + if (unfiltered_errors == 0) return; + + // Emmit errors to an `ErrorBundle`. + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(self.root_error_message), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = unfiltered_errors, + }); + var note_i: u32 = try eb.reserveNotes(unfiltered_errors); + for (self.errors) |item| { + if (item.excluded(filter)) continue; + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + .unable_to_create_file => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create file '{s}': {s}", .{ + info.file_name, @errorName(info.code), + }), + })); + }, + .unsupported_file_type => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ + info.file_name, info.file_type, + }), + })); + }, + } + note_i += 1; + } + + return error.FetchFailed; + } + + test validate { + const gpa = std.testing.allocator; + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + // fill UnpackResult with errors + var res: UnpackResult = .{}; + try res.allocErrors(arena, 4, "unable to unpack"); + try std.testing.expectEqual(0, res.errors_count); + res.unableToCreateFile("dir1/file1", error.File1); + res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); + res.unableToCreateFile("dir1/file3", error.File3); + res.unsupportedFileType("dir2/file4", 'x'); + try std.testing.expectEqual(4, res.errors_count); + + // create filter, includes dir2, excludes dir1 + var filter: Filter = .{}; + try filter.include_paths.put(arena, "dir2", {}); + + // init Fetch + var fetch: Fetch = undefined; + fetch.parent_manifest_ast = null; + fetch.location_tok = 0; + try fetch.error_bundle.init(gpa); + defer fetch.error_bundle.deinit(); + + // validate errors with filter + try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); + + // output errors to string + var errors = try fetch.error_bundle.toOwnedBundle(""); + defer errors.deinit(gpa); + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + try errors.renderToWriter(.{}, &aw.writer); + try std.testing.expectEqualStrings( + \\error: unable to unpack + \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError + \\ note: file 'dir2/file4' has unsupported type 'x' + \\ + , aw.written()); + } +}; + +test { + _ = Filter; + _ = FileType; + _ = UnpackResult; +} diff --git a/lib/compiler/Maker/Fetch/git.zig b/lib/compiler/Maker/Fetch/git.zig new file mode 100644 index 0000000000000000000000000000000000000000..d3bd1d701a618281355dba5e585d286cb3f9107f --- /dev/null +++ b/lib/compiler/Maker/Fetch/git.zig @@ -0,0 +1,1750 @@ +//! Git support for package fetching. +//! +//! This is not intended to support all features of Git: it is limited to the +//! basic functionality needed to clone a repository for the purpose of fetching +//! a package. + +const std = @import("std"); +const Io = std.Io; +const mem = std.mem; +const testing = std.testing; +const Allocator = mem.Allocator; +const Sha1 = std.crypto.hash.Sha1; +const Sha256 = std.crypto.hash.sha2.Sha256; +const assert = std.debug.assert; + +/// The ID of a Git object. +pub const Oid = union(Format) { + sha1: [Sha1.digest_length]u8, + sha256: [Sha256.digest_length]u8, + + pub const max_formatted_length = len: { + var max: usize = 0; + for (std.enums.values(Format)) |f| { + max = @max(max, f.formattedLength()); + } + break :len max; + }; + + pub const Format = enum { + sha1, + sha256, + + pub fn byteLength(f: Format) usize { + return switch (f) { + .sha1 => Sha1.digest_length, + .sha256 => Sha256.digest_length, + }; + } + + pub fn formattedLength(f: Format) usize { + return 2 * f.byteLength(); + } + }; + + const Hasher = union(Format) { + sha1: Sha1, + sha256: Sha256, + + fn init(oid_format: Format) Hasher { + return switch (oid_format) { + .sha1 => .{ .sha1 = Sha1.init(.{}) }, + .sha256 => .{ .sha256 = Sha256.init(.{}) }, + }; + } + + // Must be public for use from HashedReader and HashedWriter. + pub fn update(hasher: *Hasher, b: []const u8) void { + switch (hasher.*) { + inline else => |*inner| inner.update(b), + } + } + + fn finalResult(hasher: *Hasher) Oid { + return switch (hasher.*) { + inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), + }; + } + }; + + const Hashing = union(Format) { + sha1: Io.Writer.Hashing(Sha1), + sha256: Io.Writer.Hashing(Sha256), + + fn init(oid_format: Format, buffer: []u8) Hashing { + return switch (oid_format) { + .sha1 => .{ .sha1 = .init(buffer) }, + .sha256 => .{ .sha256 = .init(buffer) }, + }; + } + + fn writer(h: *@This()) *Io.Writer { + return switch (h.*) { + inline else => |*inner| &inner.writer, + }; + } + + fn final(h: *@This()) Oid { + switch (h.*) { + inline else => |*inner, tag| { + inner.writer.flush() catch unreachable; // hashers cannot fail + return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult()); + }, + } + } + }; + + pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { + assert(bytes.len == oid_format.byteLength()); + return switch (oid_format) { + inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), + }; + } + + pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid { + return switch (oid_format) { + inline else => |tag| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*), + }; + } + + pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { + switch (oid_format) { + inline else => |tag| { + if (s.len != tag.formattedLength()) return error.InvalidOid; + var bytes: [tag.byteLength()]u8 = undefined; + for (&bytes, 0..) |*b, i| { + b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; + } + return @unionInit(Oid, @tagName(tag), bytes); + }, + } + } + + test parse { + try testing.expectEqualSlices( + u8, + &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, + &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, + ); + try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); + try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); + try testing.expectEqualSlices( + u8, + &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, + &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, + ); + try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); + try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); + try testing.expectError(error.InvalidOid, parse(.sha1, "master")); + try testing.expectError(error.InvalidOid, parse(.sha256, "master")); + try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); + try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); + } + + pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { + return for (std.enums.values(Format)) |f| { + if (s.len == f.formattedLength()) break parse(f, s); + } else error.InvalidOid; + } + + pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void { + try writer.print("{x}", .{oid.slice()}); + } + + pub fn slice(oid: *const Oid) []const u8 { + return switch (oid.*) { + inline else => |*bytes| bytes, + }; + } +}; + +pub const Diagnostics = struct { + allocator: Allocator, + errors: std.ArrayList(Error) = .empty, + + pub const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, + }; + + pub fn deinit(d: *Diagnostics) void { + for (d.errors.items) |item| { + switch (item) { + .unable_to_create_sym_link => |info| { + d.allocator.free(info.file_name); + d.allocator.free(info.link_name); + }, + .unable_to_create_file => |info| { + d.allocator.free(info.file_name); + }, + } + } + d.errors.deinit(d.allocator); + d.* = undefined; + } +}; + +pub const Repository = struct { + odb: Odb, + + pub fn init( + repo: *Repository, + allocator: Allocator, + format: Oid.Format, + pack_file: *Io.File.Reader, + index_file: *Io.File.Reader, + ) !void { + repo.* = .{ .odb = undefined }; + try repo.odb.init(allocator, format, pack_file, index_file); + } + + pub fn deinit(repository: *Repository) void { + repository.odb.deinit(); + repository.* = undefined; + } + + /// Checks out the repository at `commit_oid` to `worktree`. + pub fn checkout( + repository: *Repository, + io: Io, + worktree: Io.Dir, + commit_oid: Oid, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(commit_oid); + const tree_oid = tree_oid: { + const commit_object = try repository.odb.readObject(); + if (commit_object.type != .commit) return error.NotACommit; + break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); + }; + try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics); + } + + /// Checks out the tree at `tree_oid` to `worktree`. + fn checkoutTree( + repository: *Repository, + io: Io, + dir: Io.Dir, + tree_oid: Oid, + current_path: []const u8, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(tree_oid); + const tree_object = try repository.odb.readObject(); + if (tree_object.type != .tree) return error.NotATree; + // The tree object may be evicted from the object cache while we're + // iterating over it, so we can make a defensive copy here to make sure + // it remains valid until we're done with it + const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); + defer repository.odb.allocator.free(tree_data); + + var tree_iter: TreeIterator = .{ + .format = repository.odb.format, + .data = tree_data, + .pos = 0, + }; + while (try tree_iter.next()) |entry| { + switch (entry.type) { + .directory => { + try dir.createDir(io, entry.name, .default_dir); + var subdir = try dir.openDir(io, entry.name, .{}); + defer subdir.close(io); + const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); + defer repository.odb.allocator.free(sub_path); + try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics); + }, + .file => { + try repository.odb.seekOid(entry.oid); + const file_object = try repository.odb.readObject(); + if (file_object.type != .blob) return error.InvalidFile; + var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ + .code = e, + .file_name = file_name, + } }); + continue; + }; + defer file.close(io); + try file.writePositionalAll(io, file_object.data, 0); + }, + .symlink => { + try repository.odb.seekOid(entry.oid); + const symlink_object = try repository.odb.readObject(); + if (symlink_object.type != .blob) return error.InvalidFile; + const link_name = symlink_object.data; + dir.symLink(io, link_name, entry.name, .{}) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); + errdefer diagnostics.allocator.free(link_name_dup); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ + .code = e, + .file_name = file_name, + .link_name = link_name_dup, + } }); + }; + }, + .gitlink => { + // Consistent with git archive behavior, create the directory but + // do nothing else + try dir.createDir(io, entry.name, .default_dir); + }, + } + } + } + + /// Returns the ID of the tree associated with the given commit (provided as + /// raw object data). + fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { + if (!mem.startsWith(u8, commit_data, "tree ") or + commit_data.len < "tree ".len + format.formattedLength() + "\n".len or + commit_data["tree ".len + format.formattedLength()] != '\n') + { + return error.InvalidCommit; + } + return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); + } + + const TreeIterator = struct { + format: Oid.Format, + data: []const u8, + pos: usize, + + const Entry = struct { + type: Type, + executable: bool, + name: [:0]const u8, + oid: Oid, + + const Type = enum(u4) { + directory = 0o4, + file = 0o10, + symlink = 0o12, + gitlink = 0o16, + }; + }; + + fn next(iterator: *TreeIterator) !?Entry { + if (iterator.pos == iterator.data.len) return null; + + const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; + const mode: packed struct { + permission: u9, + unused: u3, + type: u4, + } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); + const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree; + const executable = switch (mode.permission) { + 0 => if (@"type" == .file) return error.InvalidTree else false, + 0o644 => if (@"type" != .file) return error.InvalidTree else false, + 0o755 => if (@"type" != .file) return error.InvalidTree else true, + else => return error.InvalidTree, + }; + iterator.pos = mode_end + 1; + + const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; + const name = iterator.data[iterator.pos..name_end :0]; + iterator.pos = name_end + 1; + + const oid_length = iterator.format.byteLength(); + if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; + const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); + iterator.pos += oid_length; + + return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; + } + }; +}; + +/// A Git object database backed by a packfile. A packfile index is also used +/// for efficient access to objects in the packfile. +/// +/// The format of the packfile and its associated index are documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +const Odb = struct { + format: Oid.Format, + pack_file: *Io.File.Reader, + index_header: IndexHeader, + index_file: *Io.File.Reader, + cache: ObjectCache = .{}, + allocator: Allocator, + + /// Initializes the database from open pack and index files. + fn init( + odb: *Odb, + allocator: Allocator, + format: Oid.Format, + pack_file: *Io.File.Reader, + index_file: *Io.File.Reader, + ) !void { + try pack_file.seekTo(0); + try index_file.seekTo(0); + odb.* = .{ + .format = format, + .pack_file = pack_file, + .index_header = undefined, + .index_file = index_file, + .allocator = allocator, + }; + try odb.index_header.read(&index_file.interface); + } + + fn deinit(odb: *Odb) void { + odb.cache.deinit(odb.allocator); + odb.* = undefined; + } + + /// Reads the object at the current position in the database. + fn readObject(odb: *Odb) !Object { + var base_offset = odb.pack_file.logicalPos(); + var base_header: EntryHeader = undefined; + var delta_offsets: std.ArrayList(u64) = .empty; + defer delta_offsets.deinit(odb.allocator); + const base_object = while (true) { + if (odb.cache.get(base_offset)) |base_object| break base_object; + + base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(odb.allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; + try odb.pack_file.seekTo(base_offset); + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(odb.allocator, base_offset); + try odb.seekOid(ref_delta.base_object); + base_offset = odb.pack_file.logicalPos(); + }, + else => { + const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength()); + errdefer odb.allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try odb.cache.put(odb.allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain( + odb.allocator, + odb.format, + odb.pack_file, + base_object, + delta_offsets.items, + &odb.cache, + ); + + return .{ .type = base_object.type, .data = base_data }; + } + + /// Seeks to the beginning of the object with the given ID. + fn seekOid(odb: *Odb, oid: Oid) !void { + const oid_length = odb.format.byteLength(); + const key = oid.slice()[0]; + var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; + var end_index = odb.index_header.fan_out_table[key]; + const found_index = while (start_index < end_index) { + const mid_index = start_index + (end_index - start_index) / 2; + try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); + const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface); + switch (mem.order(u8, mid_oid.slice(), oid.slice())) { + .lt => start_index = mid_index + 1, + .gt => end_index = mid_index, + .eq => break mid_index, + } + } else return error.ObjectNotFound; + + const n_objects = odb.index_header.fan_out_table[255]; + const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); + try odb.index_file.seekTo(offset_values_start + found_index * 4); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big)); + const pack_offset = pack_offset: { + if (l1_offset.big) { + const l2_offset_values_start = offset_values_start + n_objects * 4; + try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); + break :pack_offset try odb.index_file.interface.takeInt(u64, .big); + } else { + break :pack_offset l1_offset.value; + } + }; + + try odb.pack_file.seekTo(pack_offset); + } +}; + +const Object = struct { + type: Type, + data: []const u8, + + const Type = enum { + commit, + tree, + blob, + tag, + }; +}; + +/// A cache for object data. +/// +/// The purpose of this cache is to speed up resolution of deltas by caching the +/// results of resolving delta objects, while maintaining a maximum cache size +/// to avoid excessive memory usage. If the total size of the objects in the +/// cache exceeds the maximum, the cache will begin evicting the least recently +/// used objects: when resolving delta chains, the most recently used objects +/// will likely be more helpful as they will be further along in the chain +/// (skipping earlier reconstruction steps). +/// +/// Object data stored in the cache is managed by the cache. It should not be +/// freed by the caller at any point after inserting it into the cache. Any +/// objects remaining in the cache will be freed when the cache itself is freed. +const ObjectCache = struct { + objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, + lru_nodes: std.DoublyLinkedList = .{}, + lru_nodes_len: usize = 0, + byte_size: usize = 0, + + const max_byte_size = 128 * 1024 * 1024; // 128MiB + /// A list of offsets stored in the cache, with the most recently used + /// entries at the end. + const LruListNode = struct { + data: u64, + node: std.DoublyLinkedList.Node, + }; + const CacheEntry = struct { object: Object, lru_node: *LruListNode }; + + fn deinit(cache: *ObjectCache, allocator: Allocator) void { + var object_iterator = cache.objects.iterator(); + while (object_iterator.next()) |object| { + allocator.free(object.value_ptr.object.data); + allocator.destroy(object.value_ptr.lru_node); + } + cache.objects.deinit(allocator); + cache.* = undefined; + } + + /// Gets an object from the cache, moving it to the most recently used + /// position if it is present. + fn get(cache: *ObjectCache, offset: u64) ?Object { + if (cache.objects.get(offset)) |entry| { + cache.lru_nodes.remove(&entry.lru_node.node); + cache.lru_nodes.append(&entry.lru_node.node); + return entry.object; + } else { + return null; + } + } + + /// Puts an object in the cache, possibly evicting older entries if the + /// cache exceeds its maximum size. Note that, although old objects may + /// be evicted, the object just added to the cache with this function + /// will not be evicted before the next call to `put` or `deinit` even if + /// it exceeds the maximum cache size. + fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { + const lru_node = try allocator.create(LruListNode); + errdefer allocator.destroy(lru_node); + lru_node.data = offset; + + const gop = try cache.objects.getOrPut(allocator, offset); + if (gop.found_existing) { + cache.byte_size -= gop.value_ptr.object.data.len; + cache.lru_nodes.remove(&gop.value_ptr.lru_node.node); + cache.lru_nodes_len -= 1; + allocator.destroy(gop.value_ptr.lru_node); + allocator.free(gop.value_ptr.object.data); + } + gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; + cache.byte_size += object.data.len; + cache.lru_nodes.append(&lru_node.node); + cache.lru_nodes_len += 1; + + while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) { + // The > 1 check is to make sure that we don't evict the most + // recently added node, even if it by itself happens to exceed the + // maximum size of the cache. + const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?)); + cache.lru_nodes_len -= 1; + const evict_offset = evict_node.data; + allocator.destroy(evict_node); + const evict_object = cache.objects.get(evict_offset).?.object; + cache.byte_size -= evict_object.data.len; + allocator.free(evict_object.data); + _ = cache.objects.remove(evict_offset); + } + } +}; + +/// A single pkt-line in the Git protocol. +/// +/// The format of a pkt-line is documented in +/// [protocol-common](https://git-scm.com/docs/protocol-common). The special +/// meanings of the delimiter and response-end packets are documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +pub const Packet = union(enum) { + flush, + delimiter, + response_end, + data: []const u8, + + pub const max_data_length = 65516; + + /// Reads a packet in pkt-line format. + fn read(reader: *Io.Reader) !Packet { + const packet: Packet = try .peek(reader); + switch (packet) { + .data => |data| reader.toss(data.len), + else => {}, + } + return packet; + } + + /// Consumes the header of a pkt-line packet and reads any associated data + /// into the reader's buffer, but does not consume the data. + fn peek(reader: *Io.Reader) !Packet { + const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket; + switch (length) { + 0 => return .flush, + 1 => return .delimiter, + 2 => return .response_end, + 3 => return error.InvalidPacket, + else => if (length - 4 > max_data_length) return error.InvalidPacket, + } + return .{ .data = try reader.peek(length - 4) }; + } + + /// Writes a packet in pkt-line format. + fn write(packet: Packet, writer: *Io.Writer) !void { + switch (packet) { + .flush => try writer.writeAll("0000"), + .delimiter => try writer.writeAll("0001"), + .response_end => try writer.writeAll("0002"), + .data => |data| { + assert(data.len <= max_data_length); + try writer.print("{x:0>4}", .{data.len + 4}); + try writer.writeAll(data); + }, + } + } + + /// Returns the normalized form of textual packet data, stripping any + /// trailing '\n'. + /// + /// As documented in + /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), + /// non-binary (textual) pkt-line data should contain a trailing '\n', but + /// is not required to do so (implementations must support both forms). + fn normalizeText(data: []const u8) []const u8 { + return if (mem.endsWith(u8, data, "\n")) + data[0 .. data.len - 1] + else + data; + } +}; + +/// A client session for the Git protocol, currently limited to an HTTP(S) +/// transport. Only protocol version 2 is supported, as documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +pub const Session = struct { + transport: *std.http.Client, + location: Location, + supports_agent: bool, + supports_shallow: bool, + object_format: Oid.Format, + arena: Allocator, + + const agent = "zig/" ++ @import("builtin").zig_version_string; + const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); + + /// Initializes a client session and discovers the capabilities of the + /// server for optimal transport. + pub fn init( + arena: Allocator, + transport: *std.http.Client, + uri: std.Uri, + /// Asserted to be at least `Packet.max_data_length` + response_buffer: []u8, + ) !Session { + assert(response_buffer.len >= Packet.max_data_length); + var session: Session = .{ + .transport = transport, + .location = try .init(arena, uri), + .supports_agent = false, + .supports_shallow = false, + .object_format = .sha1, + .arena = arena, + }; + var capability_iterator: CapabilityIterator = undefined; + try session.getCapabilities(&capability_iterator, response_buffer); + defer capability_iterator.deinit(); + while (try capability_iterator.next()) |capability| { + if (mem.eql(u8, capability.key, "agent")) { + session.supports_agent = true; + } else if (mem.eql(u8, capability.key, "fetch")) { + var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); + while (feature_iterator.next()) |feature| { + if (mem.eql(u8, feature, "shallow")) { + session.supports_shallow = true; + } + } + } else if (mem.eql(u8, capability.key, "object-format")) { + if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { + session.object_format = format; + } + } + } + return session; + } + + /// An owned `std.Uri` representing the location of the server (base URI). + const Location = struct { + uri: std.Uri, + + fn init(arena: Allocator, uri: std.Uri) !Location { + const scheme = try arena.dupe(u8, uri.scheme); + const user = if (uri.user) |user| try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(user, .formatUser), + }) else null; + const password = if (uri.password) |password| try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(password, .formatPassword), + }) else null; + const host = if (uri.host) |host| try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(host, .formatHost), + }) else null; + const path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(uri.path, .formatPath), + }); + // The query and fragment are not used as part of the base server URI. + return .{ + .uri = .{ + .scheme = scheme, + .user = if (user) |s| .{ .percent_encoded = s } else null, + .password = if (password) |s| .{ .percent_encoded = s } else null, + .host = if (host) |s| .{ .percent_encoded = s } else null, + .port = uri.port, + .path = .{ .percent_encoded = path }, + }, + }; + } + }; + + /// Returns an iterator over capabilities supported by the server. + /// + /// The `session.location` is updated if the server returns a redirect, so + /// that subsequent session functions do not need to handle redirects. + fn getCapabilities(session: *Session, it: *CapabilityIterator, response_buffer: []u8) !void { + const arena = session.arena; + assert(response_buffer.len >= Packet.max_data_length); + var info_refs_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(session.location.uri.path, .formatPath), + }); + info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ + "/", session_uri_path, "info/refs", + }) }; + } + info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; + info_refs_uri.fragment = null; + + const max_redirects = 3; + it.* = .{ + .request = try session.transport.request(.GET, info_refs_uri, .{ + .redirect_behavior = .init(max_redirects), + .extra_headers = &.{ + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }), + .reader = undefined, + .decompress = undefined, + }; + errdefer it.deinit(); + const request = &it.request; + try request.sendBodiless(); + + var redirect_buffer: [1024]u8 = undefined; + var response = try request.receiveHead(&redirect_buffer); + if (response.head.status != .ok) return error.ProtocolError; + const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; + if (any_redirects_occurred) { + const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(request.uri.path, .formatPath), + }); + if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; + var new_uri = request.uri; + new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; + session.location = try .init(arena, new_uri); + } + + const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer); + var state: enum { response_start, response_content } = .response_start; + while (true) { + // Some Git servers (at least GitHub) include an additional + // '# service=git-upload-pack' informative response before sending + // the expected 'version 2' packet and capability information. + // This is not universal: SourceHut, for example, does not do this. + // Thus, we need to skip any such useless additional responses + // before we get the one we're actually looking for. The responses + // will be delimited by flush packets. + const packet = Packet.read(it.reader) catch |err| switch (err) { + error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found + else => |e| return e, + }; + switch (packet) { + .flush => state = .response_start, + .data => |data| switch (state) { + .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { + return; + } else { + state = .response_content; + }, + else => {}, + }, + else => return error.UnexpectedPacket, + } + } + } + + const CapabilityIterator = struct { + request: std.http.Client.Request, + reader: *Io.Reader, + decompress: std.http.Decompress, + + const Capability = struct { + key: []const u8, + value: ?[]const u8 = null, + + fn parse(data: []const u8) Capability { + return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| + .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } + else + .{ .key = data }; + } + }; + + fn deinit(it: *CapabilityIterator) void { + it.request.deinit(); + it.* = undefined; + } + + fn next(it: *CapabilityIterator) !?Capability { + switch (try Packet.read(it.reader)) { + .flush => return null, + .data => |data| return Capability.parse(Packet.normalizeText(data)), + else => return error.UnexpectedPacket, + } + } + }; + + const ListRefsOptions = struct { + /// The ref prefixes (if any) to use to filter the refs available on the + /// server. Note that the client must still check the returned refs + /// against its desired filters itself: the server is not required to + /// respect these prefix filters and may return other refs as well. + ref_prefixes: []const []const u8 = &.{}, + /// Whether to include symref targets for returned symbolic refs. + include_symrefs: bool = false, + /// Whether to include the peeled object ID for returned tag refs. + include_peeled: bool = false, + /// Asserted to be at least `Packet.max_data_length`. + buffer: []u8, + }; + + /// Returns an iterator over refs known to the server. + pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void { + const arena = session.arena; + assert(options.buffer.len >= Packet.max_data_length); + var upload_pack_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(session.location.uri.path, .formatPath), + }); + upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; + } + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var body: Io.Writer = .fixed(options.buffer); + try Packet.write(.{ .data = "command=ls-refs\n" }, &body); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, &body); + } + { + const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{ + session.object_format, + }); + try Packet.write(.{ .data = object_format_packet }, &body); + } + try Packet.write(.delimiter, &body); + for (options.ref_prefixes) |ref_prefix| { + const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix}); + try Packet.write(.{ .data = ref_prefix_packet }, &body); + } + if (options.include_symrefs) { + try Packet.write(.{ .data = "symrefs\n" }, &body); + } + if (options.include_peeled) { + try Packet.write(.{ .data = "peel\n" }, &body); + } + try Packet.write(.flush, &body); + + it.* = .{ + .request = try session.transport.request(.POST, upload_pack_uri, .{ + .redirect_behavior = .unhandled, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }), + .reader = undefined, + .format = session.object_format, + .decompress = undefined, + }; + const request = &it.request; + errdefer request.deinit(); + try request.sendBodyComplete(body.buffered()); + + var response = try request.receiveHead(options.buffer); + if (response.head.status != .ok) return error.ProtocolError; + const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer); + } + + pub const RefIterator = struct { + format: Oid.Format, + request: std.http.Client.Request, + reader: *Io.Reader, + decompress: std.http.Decompress, + + pub const Ref = struct { + oid: Oid, + name: []const u8, + symref_target: ?[]const u8, + peeled: ?Oid, + }; + + pub fn deinit(iterator: *RefIterator) void { + iterator.request.deinit(); + iterator.* = undefined; + } + + pub fn next(it: *RefIterator) !?Ref { + switch (try Packet.read(it.reader)) { + .flush => return null, + .data => |data| { + const ref_data = Packet.normalizeText(data); + const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; + const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; + + const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; + const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; + + var symref_target: ?[]const u8 = null; + var peeled: ?Oid = null; + var last_sep_pos = name_sep_pos; + while (last_sep_pos < ref_data.len) { + const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; + const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; + if (mem.startsWith(u8, attribute, "symref-target:")) { + symref_target = attribute["symref-target:".len..]; + } else if (mem.startsWith(u8, attribute, "peeled:")) { + peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; + } + last_sep_pos = next_sep_pos; + } + + return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; + }, + else => return error.UnexpectedPacket, + } + } + }; + + /// Fetches the given refs from the server. A shallow fetch (depth 1) is + /// performed if the server supports it. + pub fn fetch( + session: Session, + fs: *FetchStream, + wants: []const []const u8, + /// Asserted to be at least `Packet.max_data_length`. + response_buffer: []u8, + ) !void { + const arena = session.arena; + assert(response_buffer.len >= Packet.max_data_length); + var upload_pack_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(session.location.uri.path, .formatPath), + }); + upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; + } + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var body: Io.Writer = .fixed(response_buffer); + try Packet.write(.{ .data = "command=fetch\n" }, &body); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, &body); + } + { + const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)}); + try Packet.write(.{ .data = object_format_packet }, &body); + } + try Packet.write(.delimiter, &body); + // Our packfile parser supports the OFS_DELTA object type + try Packet.write(.{ .data = "ofs-delta\n" }, &body); + // We do not currently convey server progress information to the user + try Packet.write(.{ .data = "no-progress\n" }, &body); + if (session.supports_shallow) { + try Packet.write(.{ .data = "deepen 1\n" }, &body); + } + for (wants) |want| { + var buf: [Packet.max_data_length]u8 = undefined; + const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; + try Packet.write(.{ .data = arg }, &body); + } + try Packet.write(.{ .data = "done\n" }, &body); + try Packet.write(.flush, &body); + + fs.* = .{ + .request = try session.transport.request(.POST, upload_pack_uri, .{ + .redirect_behavior = .not_allowed, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }), + .input = undefined, + .reader = undefined, + .remaining_len = undefined, + .decompress = undefined, + }; + const request = &fs.request; + errdefer request.deinit(); + + try request.sendBodyComplete(body.buffered()); + + var response = try request.receiveHead(&.{}); + if (response.head.status != .ok) return error.ProtocolError; + + const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer); + // We are not interested in any of the sections of the returned fetch + // data other than the packfile section, since we aren't doing anything + // complex like ref negotiation (this is a fresh clone). + var state: enum { section_start, section_content } = .section_start; + while (true) { + const packet = try Packet.read(reader); + switch (state) { + .section_start => switch (packet) { + .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { + fs.input = reader; + fs.reader = .{ + .buffer = &.{}, + .vtable = &.{ .stream = FetchStream.stream }, + .seek = 0, + .end = 0, + }; + fs.remaining_len = 0; + return; + } else { + state = .section_content; + }, + else => return error.UnexpectedPacket, + }, + .section_content => switch (packet) { + .delimiter => state = .section_start, + .data => {}, + else => return error.UnexpectedPacket, + }, + } + } + } + + pub const FetchStream = struct { + request: std.http.Client.Request, + input: *Io.Reader, + reader: Io.Reader, + err: ?Error = null, + remaining_len: usize, + decompress: std.http.Decompress, + + pub fn deinit(fs: *FetchStream) void { + fs.request.deinit(); + } + + pub const Error = error{ + InvalidPacket, + ProtocolError, + UnexpectedPacket, + WriteFailed, + ReadFailed, + EndOfStream, + }; + + const StreamCode = enum(u8) { + pack_data = 1, + progress = 2, + fatal_error = 3, + _, + }; + + pub fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { + const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r)); + const input = fs.input; + if (fs.remaining_len == 0) { + while (true) { + switch (Packet.peek(input) catch |err| { + fs.err = err; + return error.ReadFailed; + }) { + .flush => return error.EndOfStream, + .data => |data| switch (@as(StreamCode, @enumFromInt(data[0]))) { + .pack_data => { + input.toss(1); + fs.remaining_len = data.len - 1; + break; + }, + .fatal_error => { + fs.err = error.ProtocolError; + return error.ReadFailed; + }, + else => { + input.toss(data.len); + }, + }, + else => { + fs.err = error.UnexpectedPacket; + return error.ReadFailed; + }, + } + } + } + const buf = limit.slice(try w.writableSliceGreedy(1)); + const n = @min(buf.len, fs.remaining_len); + try input.readSliceAll(buf[0..n]); + w.advance(n); + fs.remaining_len -= n; + return n; + } + }; +}; + +const PackHeader = struct { + total_objects: u32, + + const signature = "PACK"; + const supported_version = 2; + + fn read(reader: *Io.Reader) !PackHeader { + const actual_signature = reader.take(4) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; + const version = reader.takeInt(u32, .big) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (version != supported_version) return error.UnsupportedVersion; + const total_objects = reader.takeInt(u32, .big) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + return .{ .total_objects = total_objects }; + } +}; + +const EntryHeader = union(Type) { + commit: Undeltified, + tree: Undeltified, + blob: Undeltified, + tag: Undeltified, + ofs_delta: OfsDelta, + ref_delta: RefDelta, + + const Type = enum(u3) { + commit = 1, + tree = 2, + blob = 3, + tag = 4, + ofs_delta = 6, + ref_delta = 7, + }; + + const Undeltified = struct { + uncompressed_length: u64, + }; + + const OfsDelta = struct { + offset: u64, + uncompressed_length: u64, + }; + + const RefDelta = struct { + base_object: Oid, + uncompressed_length: u64, + }; + + fn objectType(header: EntryHeader) Object.Type { + return switch (header) { + inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), + else => unreachable, + }; + } + + fn uncompressedLength(header: EntryHeader) u64 { + return switch (header) { + inline else => |entry| entry.uncompressed_length, + }; + } + + fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader { + const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; + const initial: InitialByte = @bitCast(reader.takeByte() catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }); + const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0; + var uncompressed_length: u64 = initial.len; + uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; + const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; + return switch (@"type") { + inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ + .uncompressed_length = uncompressed_length, + }), + .ofs_delta => .{ .ofs_delta = .{ + .offset = try readOffsetVarInt(reader), + .uncompressed_length = uncompressed_length, + } }, + .ref_delta => .{ .ref_delta = .{ + .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }, + .uncompressed_length = uncompressed_length, + } }, + }; + } +}; + +fn readOffsetVarInt(r: *Io.Reader) !u64 { + const Byte = packed struct { value: u7, has_next: bool }; + var b: Byte = @bitCast(try r.takeByte()); + var value: u64 = b.value; + while (b.has_next) { + b = @bitCast(try r.takeByte()); + value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; + value |= b.value; + } + return value; +} + +const IndexHeader = struct { + fan_out_table: [256]u32, + + const signature = "\xFFtOc"; + const supported_version = 2; + const size = 4 + 4 + @sizeOf([256]u32); + + fn read(index_header: *IndexHeader, reader: *Io.Reader) !void { + const sig = try reader.take(4); + if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; + const version = try reader.takeInt(u32, .big); + if (version != supported_version) return error.UnsupportedVersion; + try reader.readSliceEndian(u32, &index_header.fan_out_table, .big); + } +}; + +const IndexEntry = struct { + offset: u64, + crc32: u32, +}; + +/// Writes out a version 2 index for the given packfile, as documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +pub fn indexPack( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + index_writer: *Io.File.Writer, +) !void { + try pack.seekTo(0); + + var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; + defer index_entries.deinit(allocator); + var pending_deltas: std.ArrayList(IndexEntry) = .empty; + defer pending_deltas.deinit(allocator); + + const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); + + var cache: ObjectCache = .{}; + defer cache.deinit(allocator); + var remaining_deltas = pending_deltas.items.len; + while (remaining_deltas > 0) { + var i: usize = remaining_deltas; + while (i > 0) { + i -= 1; + const delta = pending_deltas.items[i]; + if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { + try index_entries.put(allocator, oid, delta); + _ = pending_deltas.swapRemove(i); + } + } + if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; + remaining_deltas = pending_deltas.items.len; + } + + var oids: std.ArrayList(Oid) = .empty; + defer oids.deinit(allocator); + try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); + var index_entries_iter = index_entries.iterator(); + while (index_entries_iter.next()) |entry| { + oids.appendAssumeCapacity(entry.key_ptr.*); + } + mem.sortUnstable(Oid, oids.items, {}, struct { + fn lessThan(_: void, o1: Oid, o2: Oid) bool { + return mem.lessThan(u8, o1.slice(), o2.slice()); + } + }.lessThan); + + var fan_out_table: [256]u32 = undefined; + var count: u32 = 0; + var fan_out_index: u8 = 0; + for (oids.items) |oid| { + const key = oid.slice()[0]; + if (key > fan_out_index) { + @memset(fan_out_table[fan_out_index..key], count); + fan_out_index = key; + } + count += 1; + } + @memset(fan_out_table[fan_out_index..], count); + + var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{}); + const writer = &index_hashed_writer.writer; + try writer.writeAll(IndexHeader.signature); + try writer.writeInt(u32, IndexHeader.supported_version, .big); + for (fan_out_table) |fan_out_entry| { + try writer.writeInt(u32, fan_out_entry, .big); + } + + for (oids.items) |oid| { + try writer.writeAll(oid.slice()); + } + + for (oids.items) |oid| { + try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); + } + + var big_offsets: std.ArrayList(u64) = .empty; + defer big_offsets.deinit(allocator); + for (oids.items) |oid| { + const offset = index_entries.get(oid).?.offset; + if (offset <= std.math.maxInt(u31)) { + try writer.writeInt(u32, @intCast(offset), .big); + } else { + const index = big_offsets.items.len; + try big_offsets.append(allocator, offset); + try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); + } + } + for (big_offsets.items) |offset| { + try writer.writeInt(u64, offset, .big); + } + + try writer.writeAll(pack_checksum.slice()); + const index_checksum = index_hashed_writer.hasher.finalResult(); + try index_writer.interface.writeAll(index_checksum.slice()); + try index_writer.end(); +} + +/// Performs the first pass over the packfile data for index construction. +/// This will index all non-delta objects, queue delta objects for further +/// processing, and return the pack checksum (which is part of the index +/// format). +fn indexPackFirstPass( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), + pending_deltas: *std.ArrayList(IndexEntry), +) !Oid { + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. + var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); + + const pack_header = try PackHeader.read(&pack_hashed.reader); + + for (0..pack_header.total_objects) |_| { + const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen(); + const entry_header = try EntryHeader.read(format, &pack_hashed.reader); + switch (entry_header) { + .commit, .tree, .blob, .tag => |object| { + var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{}); + var oid_hasher: Oid.Hashing = .init(format, &flate_buffer); + const oid_hasher_w = oid_hasher.writer(); + // The object header is not included in the pack data but is + // part of the object's ID + try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length }); + const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); + if (n != object.uncompressed_length) return error.InvalidObject; + const oid = oid_hasher.final(); + if (!skip_checksums) @compileError("TODO"); + try index_entries.put(allocator, oid, .{ + .offset = entry_offset, + .crc32 = 0, + }); + }, + inline .ofs_delta, .ref_delta => |delta| { + var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer); + const n = try entry_decompress.reader.discardRemaining(); + if (n != delta.uncompressed_length) return error.InvalidObject; + if (!skip_checksums) @compileError("TODO"); + try pending_deltas.append(allocator, .{ + .offset = entry_offset, + .crc32 = 0, + }); + }, + } + } + + if (!skip_checksums) @compileError("TODO"); + return pack_hashed.hasher.finalResult(); +} + +/// Attempts to determine the final object ID of the given deltified object. +/// May return null if this is not yet possible (if the delta is a ref-based +/// delta and we do not yet know the offset of the base object). +fn indexPackHashDelta( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + delta: IndexEntry, + index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), + cache: *ObjectCache, +) !?Oid { + // Figure out the chain of deltas to resolve + var base_offset = delta.offset; + var base_header: EntryHeader = undefined; + var delta_offsets: std.ArrayList(u64) = .empty; + defer delta_offsets.deinit(allocator); + const base_object = while (true) { + if (cache.get(base_offset)) |base_object| break base_object; + + try pack.seekTo(base_offset); + base_header = try EntryHeader.read(format, &pack.interface); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; + }, + else => { + const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength()); + errdefer allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try cache.put(allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); + + var entry_hasher_buffer: [64]u8 = undefined; + var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer); + const entry_hasher_w = entry_hasher.writer(); + // Writes to hashers cannot fail. + entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable; + entry_hasher_w.writeAll(base_data) catch unreachable; + return entry_hasher.final(); +} + +/// Resolves a chain of deltas, returning the final base object data. `pack` is +/// assumed to be looking at the start of the object data for the base object of +/// the chain, and will then apply the deltas in `delta_offsets` in reverse order +/// to obtain the final object. +fn resolveDeltaChain( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + base_object: Object, + delta_offsets: []const u64, + cache: *ObjectCache, +) ![]const u8 { + var base_data = base_object.data; + var i: usize = delta_offsets.len; + while (i > 0) { + i -= 1; + + const delta_offset = delta_offsets[i]; + try pack.seekTo(delta_offset); + const delta_header = try EntryHeader.read(format, &pack.interface); + const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); + defer allocator.free(delta_data); + var delta_reader: Io.Reader = .fixed(delta_data); + _ = try delta_reader.takeLeb128(u64); // base object size + const expanded_size = try delta_reader.takeLeb128(u64); + + const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; + const expanded_data = try allocator.alloc(u8, expanded_alloc_size); + errdefer allocator.free(expanded_data); + var expanded_delta_stream: Io.Writer = .fixed(expanded_data); + try expandDelta(base_data, &delta_reader, &expanded_delta_stream); + if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; + + try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); + base_data = expanded_data; + } + return base_data; +} + +/// Reads the complete contents of an object from `reader`. This function may +/// read more bytes than required from `reader`, so the reader position after +/// returning is not reliable. +fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 { + const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; + var aw: Io.Writer.Allocating = .init(allocator); + try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len); + defer aw.deinit(); + var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); + try decompress.reader.streamExact(&aw.writer, alloc_size); + return aw.toOwnedSlice(); +} + +/// Expands delta data from `delta_reader` to `writer`. +/// +/// The format of the delta data is documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +fn expandDelta(base_object: []const u8, delta_reader: *Io.Reader, writer: *Io.Writer) !void { + while (true) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { + error.EndOfStream => return, + else => |other| return other, + }); + if (inst.copy) { + const available: packed struct { + offset1: bool, + offset2: bool, + offset3: bool, + offset4: bool, + size1: bool, + size2: bool, + size3: bool, + } = @bitCast(inst.value); + const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ + .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, + .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, + .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, + .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, + }; + const base_offset: u32 = @bitCast(offset_parts); + const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ + .size1 = if (available.size1) try delta_reader.takeByte() else 0, + .size2 = if (available.size2) try delta_reader.takeByte() else 0, + .size3 = if (available.size3) try delta_reader.takeByte() else 0, + }; + var size: u24 = @bitCast(size_parts); + if (size == 0) size = 0x10000; + try writer.writeAll(base_object[base_offset..][0..size]); + } else if (inst.value != 0) { + try delta_reader.streamExact(writer, inst.value); + } else { + return error.InvalidDeltaInstruction; + } + } +} + +/// Runs the packfile indexing and checkout test. +/// +/// The two testrepo repositories under testdata contain identical commit +/// histories and contents. +/// +/// To verify the contents of the packfiles using Git alone, run the +/// following commands in an empty directory: +/// +/// 1. `git init --object-format=(sha1|sha256)` +/// 2. `git unpack-objects return false, + else => return std.hash.Crc32.hash(name) == n.checksum, + } + } + + pub fn int(n: Fingerprint) u64 { + return @bitCast(n); + } +}; + +/// A user-readable, file system safe hash that identifies an exact package +/// snapshot, including file contents. +/// +/// The hash is not only to prevent collisions but must resist attacks where +/// the adversary fully controls the contents being hashed. Thus, it contains +/// a full SHA-256 digest. +/// +/// This data structure can be used to store the legacy hash format too. Legacy +/// hash format is scheduled to be removed after 0.14.0 is tagged. +/// +/// There's also a third way this structure is used. When using path rather than +/// hash, a unique hash is still needed, so one is computed based on the path. +pub const Hash = struct { + /// Maximum size of a package hash. Unused bytes at the end are + /// filled with zeroes. + /// + /// Assumed to be already validated. + bytes: [max_len]u8, + + pub const Algo = std.crypto.hash.sha2.Sha256; + pub const Digest = [Algo.digest_length]u8; + + /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" + pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; + + /// Asserts `s` is valid. + pub fn fromSlice(s: []const u8) Hash { + assert(validate(s) == .ok); + var result: Hash = undefined; + @memcpy(result.bytes[0..s.len], s); + @memset(result.bytes[s.len..], 0); + return result; + } + + pub const Validation = enum { ok, short, long, incomplete }; + + pub fn validate(s: []const u8) Validation { + if (s.len > max_len) return .long; + if (s.len < 44) return .short; + const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-'); + if (n_dashes < 2) return .incomplete; + return .ok; + } + + test validate { + try std.testing.expectEqual(.short, validate("")); + } + + pub fn toSlice(ph: *const Hash) []const u8 { + var end: usize = ph.bytes.len; + while (true) { + end -= 1; + if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; + } + } + + pub fn eql(a: *const Hash, b: *const Hash) bool { + return std.mem.eql(u8, &a.bytes, &b.bytes); + } + + /// Produces "$name-$semver-$hashplus". + /// * name is the name field from build.zig.zon, asserted to be at most 32 + /// bytes and assumed be a valid zig identifier + /// * semver is the version field from build.zig.zon, asserted to be at + /// most 32 bytes + /// * hashplus is the following 33-byte array, base64 encoded using -_ to make + /// it filesystem safe: + /// - (4 bytes) LE u32 Package ID + /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated + /// - (25 bytes) truncated SHA-256 digest of hashed files of the package + pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { + assert(name.len <= 32); + assert(ver.len <= 32); + var result: Hash = undefined; + var buf: std.ArrayList(u8) = .initBuffer(&result.bytes); + buf.appendSliceAssumeCapacity(name); + buf.appendAssumeCapacity('-'); + buf.appendSliceAssumeCapacity(ver); + buf.appendAssumeCapacity('-'); + var hashplus: [33]u8 = undefined; + std.mem.writeInt(u32, hashplus[0..4], id, .little); + std.mem.writeInt(u32, hashplus[4..8], size, .little); + hashplus[8..].* = digest[0..25].*; + _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); + @memset(buf.unusedCapacitySlice(), 0); + return result; + } + + /// Produces a unique hash based on the path provided. The result should + /// not be user-visible. + pub fn initPath(sub_path: []const u8, is_global: bool) Hash { + var result: Hash = .{ .bytes = @splat(0) }; + var i: usize = 0; + if (is_global) { + result.bytes[0] = '/'; + i += 1; + } + if (i + sub_path.len <= result.bytes.len) { + @memcpy(result.bytes[i..][0..sub_path.len], sub_path); + return result; + } + var bin_digest: [Algo.digest_length]u8 = undefined; + Algo.hash(sub_path, &bin_digest, .{}); + _ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable; + return result; + } + + pub fn projectId(hash: *const Hash) ProjectId { + const bytes = hash.toSlice(); + const name = std.mem.sliceTo(bytes, '-'); + const encoded_hashplus = bytes[bytes.len - 44 ..]; + var hashplus: [33]u8 = undefined; + std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable; + const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little); + return .init(name, fingerprint_id); + } + + test projectId { + const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw"); + const project_id = hash.projectId(); + + var expected_name: [32]u8 = @splat(0); + expected_name[0.."pulseaudio".len].* = "pulseaudio".*; + try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); + + try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id); + } + + test "projectId with dashes in the base64" { + const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC"); + const project_id = hash.projectId(); + + var expected_name: [32]u8 = @splat(0); + expected_name[0.."dvui".len].* = "dvui".*; + try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); + + try std.testing.expectEqual(0x99490101, project_id.fingerprint_id); + } +}; + +/// Minimum information required to identify whether a package is an artifact +/// of a given project. +pub const ProjectId = struct { + /// Bytes after name.len are set to zero. + padded_name: [32]u8, + fingerprint_id: u32, + + pub fn init(name: []const u8, fingerprint_id: u32) ProjectId { + var padded_name: [32]u8 = @splat(0); + @memcpy(padded_name[0..name.len], name); + return .{ + .padded_name = padded_name, + .fingerprint_id = fingerprint_id, + }; + } + + pub fn eql(a: *const ProjectId, b: *const ProjectId) bool { + return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name); + } + + pub fn hash(a: *const ProjectId) u64 { + const x: u64 = @bitCast(a.padded_name[0..8].*); + return std.hash.int(x | a.fingerprint_id); + } +}; + +test Hash { + const example_digest: Hash.Digest = .{ + 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, + 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, + }; + const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); + try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); +} + +test { + _ = Fetch; +} diff --git a/lib/compiler/Maker/Package/Manifest.zig b/lib/compiler/Maker/Package/Manifest.zig new file mode 100644 index 0000000000000000000000000000000000000000..849fc742ec5c7b5be02a2895f54f99fece02b2ff --- /dev/null +++ b/lib/compiler/Maker/Package/Manifest.zig @@ -0,0 +1,734 @@ +const Manifest = @This(); + +const std = @import("std"); +const Io = std.Io; +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; + +const Package = @import("../Package.zig"); + +pub const max_bytes = 10 * 1024 * 1024; +pub const basename = "build.zig.zon"; +pub const max_name_len = 32; +pub const max_version_len = 32; + +pub const Dependency = struct { + location: Location, + location_tok: Ast.TokenIndex, + location_node: Ast.Node.Index, + hash: ?[]const u8, + hash_tok: Ast.OptionalTokenIndex, + hash_node: Ast.Node.OptionalIndex, + node: Ast.Node.Index, + name_tok: Ast.TokenIndex, + lazy: bool, + + pub const Location = union(enum) { + url: []const u8, + path: []const u8, + }; +}; + +pub const ErrorMessage = struct { + msg: []const u8, + tok: Ast.TokenIndex, + off: u32, +}; + +name: []const u8, +id: u32, +version: std.SemanticVersion, +version_node: Ast.Node.Index, +dependencies: std.array_hash_map.String(Dependency), +dependencies_node: Ast.Node.OptionalIndex, +paths: std.array_hash_map.String(void), +minimum_zig_version: ?std.SemanticVersion, + +errors: []ErrorMessage, +arena_state: std.heap.ArenaAllocator.State, + +pub const ParseOptions = struct { + allow_missing_paths_field: bool = false, +}; + +pub const Error = Allocator.Error; + +pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest { + const main_node_index = ast.nodeData(.root).node; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + errdefer arena_instance.deinit(); + + var p: Parse = .{ + .gpa = gpa, + .ast = ast.*, + .arena = arena_instance.allocator(), + .errors = .empty, + + .name = undefined, + .id = 0, + .version = undefined, + .version_node = undefined, + .dependencies = .{}, + .dependencies_node = .none, + .paths = .empty, + .allow_missing_paths_field = options.allow_missing_paths_field, + .minimum_zig_version = null, + .buf = .empty, + }; + defer p.buf.deinit(gpa); + defer p.errors.deinit(gpa); + defer p.dependencies.deinit(gpa); + defer p.paths.deinit(gpa); + + p.parseRoot(main_node_index, rng) catch |err| switch (err) { + error.ParseFailure => assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .id = p.id, + .version = p.version, + .version_node = p.version_node, + .dependencies = try p.dependencies.clone(p.arena), + .dependencies_node = p.dependencies_node, + .paths = try p.paths.clone(p.arena), + .minimum_zig_version = p.minimum_zig_version, + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +pub fn copyErrorsIntoBundle( + man: Manifest, + ast: Ast, + /// ErrorBundle null-terminated string index + src_path: u32, + eb: *std.zig.ErrorBundle.Wip, +) Allocator.Error!void { + for (man.errors) |msg| { + const start_loc = ast.tokenLocation(0, msg.tok); + + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = ast.tokenStart(msg.tok), + .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), + .span_main = ast.tokenStart(msg.tok) + msg.off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }), + }); + } +} + +const Parse = struct { + gpa: Allocator, + ast: Ast, + arena: Allocator, + buf: std.ArrayList(u8), + errors: std.ArrayList(ErrorMessage), + + name: []const u8, + id: u32, + version: std.SemanticVersion, + version_node: Ast.Node.Index, + dependencies: std.array_hash_map.String(Dependency), + dependencies_node: Ast.Node.OptionalIndex, + paths: std.array_hash_map.String(void), + allow_missing_paths_field: bool, + minimum_zig_version: ?std.SemanticVersion, + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + var have_included_paths = false; + var fingerprint: ?Package.Fingerprint = null; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + p.dependencies_node = field_init.toOptional(); + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "paths")) { + have_included_paths = true; + try parseIncludedPaths(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseName(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "fingerprint")) { + fingerprint = try parseFingerprint(p, field_init); + } else if (mem.eql(u8, field_name, "version")) { + p.version_node = field_init; + const version_text = try parseString(p, field_init); + if (version_text.len > max_version_len) { + try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); + } + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else if (mem.eql(u8, field_name, "minimum_zig_version")) { + const version_text = try parseString(p, field_init); + p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v null; + }; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } else { + if (fingerprint) |n| { + if (!n.validate(p.name)) { + return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ + n.int(), Package.Fingerprint.generate(rng, p.name).int(), + }); + } + p.id = n.id; + } else { + try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ + Package.Fingerprint.generate(rng, p.name).int(), + }); + } + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + + if (!have_included_paths) { + if (p.allow_missing_paths_field) { + try p.paths.put(p.gpa, "", {}); + } else { + try appendError(p, main_token, "missing top-level 'paths' field", .{}); + } + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .location = undefined, + .location_tok = undefined, + .location_node = undefined, + .hash = null, + .hash_tok = .none, + .hash_node = .none, + .node = node, + .name_tok = undefined, + .lazy = false, + }; + var has_location = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + dep.name_tok = name_token; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + if (has_location) { + return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = ast.nodeMainToken(field_init); + dep.location_node = field_init; + } else if (mem.eql(u8, field_name, "path")) { + if (has_location) { + return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .path = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = ast.nodeMainToken(field_init); + dep.location_node = field_init; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); + dep.hash_node = field_init.toOptional(); + } else if (mem.eql(u8, field_name, "lazy")) { + dep.lazy = parseBool(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!has_location) { + try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); + } + + return dep; + } + + fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected paths expression to be a list of strings", .{}); + }; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parseString(p, elem_node); + // This is normalized so that it can be used in string comparisons + // against file system paths. + const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); + try p.paths.put(p.gpa, normalized, {}); + } + } + + fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { + const ast = p.ast; + if (ast.nodeTag(node) != .identifier) { + return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); + } + const ident_token = ast.nodeMainToken(node); + const token_bytes = ast.tokenSlice(ident_token); + if (mem.eql(u8, token_bytes, "true")) { + return true; + } else if (mem.eql(u8, token_bytes, "false")) { + return false; + } else { + return fail(p, ident_token, "expected boolean", .{}); + } + } + + fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + if (ast.nodeTag(node) != .number_literal) { + return fail(p, main_token, "expected integer literal", .{}); + } + const token_bytes = ast.tokenSlice(main_token); + const parsed = std.zig.parseNumberLiteral(token_bytes); + switch (parsed) { + .int => |n| return @bitCast(n), + .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ + @tagName(parsed), + }), + .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), + } + } + + fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + + if (ast.nodeTag(node) != .enum_literal) + return fail(p, main_token, "expected enum literal", .{}); + + const ident_name = ast.tokenSlice(main_token); + if (mem.startsWith(u8, ident_name, "@")) + return fail(p, main_token, "name must be a valid bare zig identifier", .{}); + + if (ident_name.len > max_name_len) + return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{ + std.zig.fmtId(ident_name), max_name_len, + }); + + return ident_name; + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + if (ast.nodeTag(node) != .string_literal) { + return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); + } + const str_lit_token = ast.nodeMainToken(node); + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const tok = ast.nodeMainToken(node); + const h = try parseString(p, node); + switch (Package.Hash.validate(h)) { + .ok => return h, + else => |t| return fail(p, tok, "invalid hash: {t}", .{t}), + } + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + assert(ast.tokenTag(token) == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayList(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + const result = r: { + var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf); + defer buf.* = aw.toArrayList(); + break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; + }; + switch (result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "unicode escape does not correspond to a valid unicode scalar value", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_char_literal => { + try p.appendErrorOff(token, offset, "empty character literal", .{}); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +pub fn load( + io: Io, + arena: Allocator, + manifest_path: std.Build.Cache.Path, + ast: *std.zig.Ast, + error_bundle: *std.zig.ErrorBundle.Wip, + manifest: *Manifest, + allow_missing_paths_field: bool, +) !void { + const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions( + io, + manifest_path.sub_path, + arena, + .limited(max_bytes), + .@"1", + 0, + ); + + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); + + if (ast.errors.len > 0) { + const file_path = try manifest_path.joinString(arena, ""); + try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle); + return error.ErrorsBundled; + } + + const rng: std.Random.IoSource = .{ .io = io }; + + manifest.* = try parse(arena, ast, rng.interface(), .{ + .allow_missing_paths_field = allow_missing_paths_field, + }); + + if (manifest.errors.len > 0) { + const src_path = try error_bundle.printString("{f}", .{manifest_path}); + try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle); + return error.ErrorsBundled; + } +} + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, + \\ .version = "3.2.1", + \\ .paths = .{""}, + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", + \\ }, + \\ }, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var rng = std.Random.DefaultPrng.init(0); + + var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].location.url, + ); + try testing.expectEqualStrings( + "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); + + try testing.expect(manifest.minimum_zig_version == null); +} + +test "minimum_zig_version" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, + \\ .version = "3.2.1", + \\ .paths = .{""}, + \\ .minimum_zig_version = "0.11.1", + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var rng = std.Random.DefaultPrng.init(0); + + var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expect(manifest.dependencies.count() == 0); + + try testing.expect(manifest.minimum_zig_version != null); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 0, + .minor = 11, + .patch = 1, + }), manifest.minimum_zig_version.?); +} + +test "minimum_zig_version - invalid version" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, + \\ .version = "3.2.1", + \\ .minimum_zig_version = "X.11.1", + \\ .paths = .{""}, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var rng = std.Random.DefaultPrng.init(0); + + var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 1); + try testing.expect(manifest.dependencies.count() == 0); + + try testing.expect(manifest.minimum_zig_version == null); +} diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 809bb9e6263c3a9c0e128bb6a677589fef6054c9..a7b7ae44274356702ffe34923d39405d23715f11 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -633,7 +633,8 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { var s: Serialize = .{ .wc = wc, .arena = arena }; try wc.path_deps.ensureTotalCapacityPrecise(gpa, graph.configure_dependencies.items.len); - for ( + // TODO remove this + if (false) for ( graph.configure_dependencies.items, wc.path_deps.addManyAsSliceAssumeCapacity(graph.configure_dependencies.items.len), ) |src, *dest| { @@ -661,7 +662,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .dependency => |d| .init(try s.builderToPackage(d.dependency.builder)), }, }; - } + }; // Starting from all top-level steps in `b`, traverse the entire step graph // and add all step dependencies implied by module graphs. diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 0880d2abeaa9e4fa9580673b3a16f028c2b3d0f4..68f5cfc49af5e8e48724db42455115313108f9e1 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1035,7 +1035,7 @@ pub const Manifest = struct { pub fn addPathPost(man: *Manifest, path: Path) !void { _ = man; _ = path; - @panic("TODO"); + std.log.err("TODO Build.Cache.addPathPost", .{}); } /// Like `addFilePost` but when the file contents have already been loaded from disk. diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 8e36cad974a76f24ef6e12f08802995c2b748544..72b876e577296b359ba86cee7df7d2fe65a8bb92 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1881,7 +1881,7 @@ pub const PathDep = extern struct { _ = c; _ = arena; _ = path; - @panic("TODO"); + std.log.err("TODO Configuration.PathDep.toCachePath", .{}); } }; diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 05891c582e38646da978177e34a3e8b26ac7f0af..2e1e0769436f003a829af900cde18c3b9f440236 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -2,12 +2,17 @@ //! source lives here. These APIs are provided as-is and have absolutely no API //! guarantees whatsoever. +const builtin = @import("builtin"); + const std = @import("std.zig"); const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; const Io = std.Io; const Writer = std.Io.Writer; +const Cache = std.Build.Cache; +const fatal = std.process.fatal; +const Dir = std.Io.Dir; const tokenizer = @import("zig/tokenizer.zig"); @@ -47,6 +52,9 @@ pub const c_translation = struct { pub const helpers = @import("zig/c_translation/helpers.zig"); }; +pub const default_local_zig_cache_basename = ".zig-cache"; +pub const build_zig_basename = "build.zig"; + pub const SrcHasher = std.crypto.hash.Blake3; pub const SrcHash = [16]u8; @@ -70,7 +78,7 @@ pub const Color = enum { /// CLICOLOR_FORCE environment variables. Color is always disabled on WASI per /// https://github.com/WebAssembly/WASI/issues/162 pub fn settingFromEnvironment(environ_map: *const std.process.Environ.Map) Color { - return if (@import("builtin").os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map)) + return if (builtin.os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map)) .off else if (EnvVar.CLICOLOR_FORCE.isSet(environ_map)) .on @@ -163,8 +171,8 @@ pub const BinNameOptions = struct { os_tag: std.Target.Os.Tag, ofmt: std.Target.ObjectFormat, abi: std.Target.Abi, - output_mode: std.builtin.OutputMode, - link_mode: ?std.builtin.LinkMode = null, + output_mode: std.lang.OutputMode, + link_mode: ?std.lang.LinkMode = null, version: ?std.SemanticVersion = null, }; @@ -512,7 +520,7 @@ pub const FormatId = struct { pub fn format(ctx: FormatId, writer: *Writer) Writer.Error!void { const bytes = ctx.bytes; if (isValidId(bytes) and - (ctx.flags.allow_primitive or !std.zig.isPrimitive(bytes)) and + (ctx.flags.allow_primitive or !isPrimitive(bytes)) and (ctx.flags.allow_underscore or !isUnderscore(bytes))) { return writer.writeAll(bytes); @@ -592,7 +600,7 @@ pub fn isValidId(bytes: []const u8) bool { else => return false, } } - return std.zig.Token.getKeyword(bytes) == null; + return Token.getKeyword(bytes) == null; } test isValidId { @@ -658,7 +666,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *Io.File.Reader) ![ } pub fn printAstErrorsToStderr(gpa: Allocator, io: Io, tree: Ast, path: []const u8, color: Color) !void { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; + var wip_errors: ErrorBundle.Wip = undefined; try wip_errors.init(gpa); defer wip_errors.deinit(); @@ -673,7 +681,7 @@ pub fn putAstErrorsIntoBundle( gpa: Allocator, tree: Ast, path: []const u8, - wip_errors: *std.zig.ErrorBundle.Wip, + wip_errors: *ErrorBundle.Wip, ) Allocator.Error!void { switch (tree.mode) { .zig => { @@ -692,7 +700,7 @@ pub fn putAstErrorsIntoBundle( } pub fn resolveTargetQueryOrFatal(io: Io, target_query: std.Target.Query) std.Target { - return std.zig.system.resolveTargetQuery(io, target_query) catch |err| + return system.resolveTargetQuery(io, target_query) catch |err| std.process.fatal("unable to resolve target: {t}", .{err}); } @@ -1242,6 +1250,365 @@ pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPri return aw.toOwnedSlice(); } +/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This +/// means the path has no repeated separators, no "." or ".." components, and no trailing separator. +/// On WASI, "" is returned instead of ".". +pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 { + if (builtin.os.tag == .wasi) { + if (std.debug.runtime_safety) { + const cwd = try std.process.currentPathAlloc(io, gpa); + defer gpa.free(cwd); + assert(mem.eql(u8, cwd, ".")); + } + return ""; + } + const cwd = try std.process.currentPathAlloc(io, gpa); + defer gpa.free(cwd); + const resolved = try Dir.path.resolve(gpa, &.{cwd}); + assert(Dir.path.isAbsolute(resolved)); + return resolved; +} + +pub const Directories = struct { + /// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path, + /// but on WASI is the empty string "" instead, because WASI does not have absolute paths. + cwd: []const u8, + /// The Zig 'lib' directory. + /// `zig_lib.path` is resolved (`resolvePath`) or `null` for cwd. + /// Guaranteed to be a different path from `global_cache` and `local_cache`. + zig_lib: Cache.Directory, + /// The global Zig cache directory. + /// `global_cache.path` is resolved (`resolvePath`) or `null` for cwd. + global_cache: Cache.Directory, + /// The local Zig cache directory. + /// `local_cache.path` is resolved (`resolvePath`) or `null` for cwd. + /// This may be the same as `global_cache`. + local_cache: Cache.Directory, + + pub fn deinit(dirs: *Directories, io: Io) void { + // The local and global caches could be the same. + const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle; + + dirs.global_cache.handle.close(io); + if (close_local) dirs.local_cache.handle.close(io); + dirs.zig_lib.handle.close(io); + } + + /// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for + /// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it + /// shares handles with `dirs`. + pub fn withoutLocalCache(dirs: Directories) Directories { + return .{ + .cwd = dirs.cwd, + .zig_lib = dirs.zig_lib, + .global_cache = dirs.global_cache, + .local_cache = dirs.global_cache, + }; + } + + const LocalCacheStrategy = union(enum) { + override: []const u8, + search, + global, + }; + + /// Uses `std.process.fatal` on error conditions. + pub fn init( + arena: Allocator, + io: Io, + override_zig_lib: ?[]const u8, + override_global_cache: ?[]const u8, + local_cache_strat: LocalCacheStrategy, + preopens: std.process.Preopens, + self_exe_path: switch (builtin.target.os.tag) { + .wasi => void, + else => []const u8, + }, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, + ) Directories { + const wasi = builtin.target.os.tag == .wasi; + + const zig_lib: Cache.Directory = d: { + if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); + if (wasi) break :d getPreopen(preopens, "/lib"); + break :d findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch |err| { + fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err }); + }; + }; + + const global_cache: Cache.Directory = d: { + if (override_global_cache) |path| break :d openUnresolved(arena, io, cwd, path, .@"global cache"); + if (wasi) break :d getPreopen(preopens, "/cache"); + const path = resolveGlobalCacheDir(arena, environ_map) catch |err| { + fatal("unable to resolve zig cache directory: {t}", .{err}); + }; + break :d openUnresolved(arena, io, cwd, path, .@"global cache"); + }; + + const local_cache = getLocalCacheDirectory(arena, io, cwd, global_cache, local_cache_strat); + + if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) { + fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache }); + } + if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) { + fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache }); + } + + return .{ + .cwd = cwd, + .zig_lib = zig_lib, + .global_cache = global_cache, + .local_cache = local_cache, + }; + } + + fn getLocalCacheDirectory( + arena: Allocator, + io: Io, + cwd: []const u8, + global_cache: Cache.Directory, + local_cache_strat: LocalCacheStrategy, + ) Cache.Directory { + return switch (local_cache_strat) { + .override => |path| openUnresolved(arena, io, cwd, path, .@"local cache"), + .search => d: { + const maybe_path = resolveSuitableLocalCacheDir(arena, io, cwd) catch |err| + fatal("unable to resolve zig cache directory: {t}", .{err}); + const path = maybe_path orelse break :d global_cache; + break :d openUnresolved(arena, io, cwd, path, .@"local cache"); + }, + .global => global_cache, + }; + } + + fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory { + return .{ + .path = if (std.mem.eql(u8, name, ".")) null else name, + .handle = switch (preopens.get(name) orelse fatal("preopen not found: {q}", .{name})) { + .file => fatal("preopen {q} is not a directory", .{name}), + .dir => |d| d, + }, + }; + } + fn openUnresolved( + arena: Allocator, + io: Io, + cwd: []const u8, + unresolved_path: []const u8, + thing: enum { @"zig lib", @"global cache", @"local cache" }, + ) Cache.Directory { + const path = resolvePath(arena, cwd, &.{unresolved_path}) catch |err| { + fatal("unable to resolve {t} directory: {t}", .{ thing, err }); + }; + const nonempty_path = if (path.len == 0) "." else path; + const handle_or_err = switch (thing) { + .@"zig lib" => Dir.cwd().openDir(io, nonempty_path, .{}), + .@"global cache", .@"local cache" => Dir.cwd().createDirPathOpen(io, nonempty_path, .{}), + }; + return .{ + .path = if (path.len == 0) null else path, + .handle = handle_or_err catch |err| { + const extra_str: []const u8 = e: { + if (thing == .@"global cache") switch (err) { + error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++ + "If this location is not writable then consider specifying an alternative with " ++ + "the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.", + else => {}, + }; + break :e ""; + }; + fatal("unable to open {t} directory {q}: {t}{s}", .{ thing, nonempty_path, err, extra_str }); + }, + }; + } +}; + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory { + const cwd_path = try getResolvedCwd(io, gpa); + defer gpa.free(cwd_path); + const self_exe_path = try std.process.executablePathAlloc(io, gpa); + defer gpa.free(self_exe_path); + + return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path); +} + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDirFromSelfExe( + allocator: Allocator, + io: Io, + /// The return value of `getResolvedCwd`. + /// Passed as an argument to avoid pointlessly repeating the call. + cwd_path: []const u8, + self_exe_path: []const u8, +) error{ OutOfMemory, FileNotFound }!Cache.Directory { + const cwd = Dir.cwd(); + var cur_path: []const u8 = self_exe_path; + while (Dir.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { + var base_dir = cwd.openDir(io, dirname, .{}) catch continue; + defer base_dir.close(io); + + const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue; + const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? }); + defer allocator.free(p); + + const resolved = try resolvePath(allocator, cwd_path, &.{p}); + return .{ + .handle = sub_directory.handle, + .path = if (resolved.len == 0) null else resolved, + }; + } + return error.FileNotFound; +} + +/// Returns the sub_path that worked, or `null` if none did. +/// The path of the returned Directory is relative to `base`. +/// The handle of the returned Directory is open. +fn testZigInstallPrefix(io: Io, base_dir: Dir) ?Cache.Directory { + const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig"; + + zig_dir: { + // Try lib/zig/std/std.zig + const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig"; + var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir; + const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { + test_zig_dir.close(io); + break :zig_dir; + }; + file.close(io); + return .{ .handle = test_zig_dir, .path = lib_zig }; + } + + // Try lib/std/std.zig + var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null; + const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { + test_zig_dir.close(io); + return null; + }; + file.close(io); + return .{ .handle = test_zig_dir, .path = "lib" }; +} + +pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 { + if (EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) |value| return value; + + const app_name = "zig"; + + switch (builtin.os.tag) { + .wasi => @compileError("on WASI the global cache dir must be resolved with preopens"), + .windows => { + const local_app_data_dir = EnvVar.LOCALAPPDATA.get(environ_map) orelse + return error.AppDataDirUnavailable; + return Dir.path.join(arena, &.{ local_app_data_dir, app_name }); + }, + else => { + if (EnvVar.XDG_CACHE_HOME.get(environ_map)) |cache_root| { + if (cache_root.len > 0) { + return Dir.path.join(arena, &.{ cache_root, app_name }); + } + } + if (EnvVar.HOME.get(environ_map)) |home| { + if (home.len > 0) { + return Dir.path.join(arena, &.{ home, ".cache", app_name }); + } + } + return error.AppDataDirUnavailable; + }, + } +} + +/// Searches upwards from `cwd` for a directory containing a `build.zig` file. +/// If such a directory is found, returns the path to it joined to the `.zig_cache` name. +/// Otherwise, returns `null`, indicating no suitable local cache location. +pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 { + var cur_dir = cwd; + while (true) { + const joined = try Dir.path.join(arena, &.{ cur_dir, build_zig_basename }); + if (Dir.cwd().access(io, joined, .{})) |_| { + return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename }); + } else |err| switch (err) { + error.FileNotFound => { + cur_dir = Dir.path.dirname(cur_dir) orelse return null; + continue; + }, + else => return null, + } + } +} + +/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would +/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd +/// returns the empty string ("") instead of ".". +pub fn resolvePath( + gpa: Allocator, + /// The return value of `getResolvedCwd`. + /// Passed as an argument to avoid pointlessly repeating the call. + cwd_resolved: []const u8, + paths: []const []const u8, +) Allocator.Error![]u8 { + if (builtin.target.os.tag == .wasi) { + assert(mem.eql(u8, cwd_resolved, "")); + const res = try Dir.path.resolve(gpa, paths); + if (mem.eql(u8, res, ".")) { + gpa.free(res); + return ""; + } + return res; + } + + // Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`. + for (paths) |p| { + if (Dir.path.isAbsolute(p)) break; // absolute path + if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir + } else { + // no absolute path, no "..". + const res = try Dir.path.resolve(gpa, paths); + if (mem.eql(u8, res, ".")) { + gpa.free(res); + return ""; + } + assert(!Dir.path.isAbsolute(res)); + assert(!isUpDir(res)); + return res; + } + + // The fast path failed; resolve the whole thing. + // Optimization: `paths` often has just one element. + const path_resolved = switch (paths.len) { + 0 => unreachable, + 1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }), + else => r: { + const all_paths = try gpa.alloc([]const u8, paths.len + 1); + defer gpa.free(all_paths); + all_paths[0] = cwd_resolved; + @memcpy(all_paths[1..], paths); + break :r try Dir.path.resolve(gpa, all_paths); + }, + }; + errdefer gpa.free(path_resolved); + + assert(Dir.path.isAbsolute(path_resolved)); + assert(Dir.path.isAbsolute(cwd_resolved)); + + if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd + if (path_resolved.len == cwd_resolved.len) { + // equal to cwd + gpa.free(path_resolved); + return ""; + } + if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs) + + // in cwd; extract sub path + const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]); + gpa.free(path_resolved); + return sub_path; +} + +pub fn isUpDir(p: []const u8) bool { + return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep); +} + test { _ = Ast; _ = AstRlAnnotate; diff --git a/src/Compilation.zig b/src/Compilation.zig index 70ed4c7f9a55d8b1695520be78c264925f39928b..b130d4917c17a86a856f0ab02ee4552ddd179bda 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -17,7 +17,6 @@ const Value = @import("Value.zig"); const Type = @import("Type.zig"); const target_util = @import("target.zig"); const Package = @import("Package.zig"); -const introspect = @import("introspect.zig"); const link = @import("link.zig"); const tracy = @import("tracy.zig"); const trace = tracy.trace; @@ -190,7 +189,7 @@ parent_whole_cache: ?ParentWholeCache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, /// Owned by the caller of `Compilation.create`. -dirs: Directories, +dirs: std.zig.Directories, libc_include_dir_list: []const []const u8, libc_framework_dir_list: []const []const u8, rc_includes: std.zig.RcIncludes, @@ -431,7 +430,7 @@ pub const Path = struct { } /// Given a `Path`, returns the directory handle and sub path to be used to open the path. - pub fn openInfo(p: Path, dirs: Directories) struct { Io.Dir, []const u8 } { + pub fn openInfo(p: Path, dirs: std.zig.Directories) struct { Io.Dir, []const u8 } { const dir = switch (p.root) { .none => { const cwd_sub_path = absToCwdRelative(p.sub_path, dirs.cwd); @@ -492,7 +491,7 @@ pub const Path = struct { /// From an unresolved path (which can be made of multiple not-yet-joined strings), construct a /// canonical `Path`. pub fn fromUnresolved(gpa: Allocator, dirs: Compilation.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path { - const resolved = try introspect.resolvePath(gpa, dirs.cwd, unresolved_parts); + const resolved = try std.zig.resolvePath(gpa, dirs.cwd, unresolved_parts); errdefer gpa.free(resolved); // If, for instance, `dirs.local_cache.path` is within the lib dir, it must take priority, @@ -626,7 +625,7 @@ pub const Path = struct { }); } - pub fn toCachePath(p: Path, dirs: Directories) Cache.Path { + pub fn toCachePath(p: Path, dirs: std.zig.Directories) Cache.Path { const root_dir: Cache.Directory = switch (p.root) { .zig_lib => dirs.zig_lib, .global_cache => dirs.global_cache, @@ -649,7 +648,7 @@ pub const Path = struct { /// This should not be used for most of the compiler pipeline, but is useful when emitting /// paths from the compilation (e.g. in debug info), because they will not depend on the cwd. /// The returned path is owned by the caller and allocated into `gpa`. - pub fn toAbsolute(p: Path, dirs: Directories, gpa: Allocator) Allocator.Error![]u8 { + pub fn toAbsolute(p: Path, dirs: std.zig.Directories, gpa: Allocator) Allocator.Error![]u8 { const root_path: []const u8 = switch (p.root) { .zig_lib => dirs.zig_lib.path orelse "", .global_cache => dirs.global_cache.path orelse "", @@ -680,7 +679,7 @@ pub const Path = struct { /// Returns whether this `Path` is illegal to have as a user-imported `Zcu.File` (including /// as the root of a module). Such paths exist in directories which the Zig compiler treats /// specially, like 'global_cache/b/', which stores 'builtin.zig' files. - pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: Directories) Allocator.Error!bool { + pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: std.zig.Directories) Allocator.Error!bool { const zig_builtin_dir: Path = try .fromRoot(gpa, dirs, .global_cache, "b"); defer zig_builtin_dir.deinit(gpa); return switch (p.isNested(zig_builtin_dir)) { @@ -690,149 +689,6 @@ pub const Path = struct { } }; -pub const Directories = struct { - /// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path, - /// but on WASI is the empty string "" instead, because WASI does not have absolute paths. - cwd: []const u8, - /// The Zig 'lib' directory. - /// `zig_lib.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - /// Guaranteed to be a different path from `global_cache` and `local_cache`. - zig_lib: Cache.Directory, - /// The global Zig cache directory. - /// `global_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - global_cache: Cache.Directory, - /// The local Zig cache directory. - /// `local_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - /// This may be the same as `global_cache`. - local_cache: Cache.Directory, - - pub fn deinit(dirs: *Directories, io: Io) void { - // The local and global caches could be the same. - const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle; - - dirs.global_cache.handle.close(io); - if (close_local) dirs.local_cache.handle.close(io); - dirs.zig_lib.handle.close(io); - } - - /// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for - /// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it - /// shares handles with `dirs`. - pub fn withoutLocalCache(dirs: Directories) Directories { - return .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }; - } - - /// Uses `std.process.fatal` on error conditions. - pub fn init( - arena: Allocator, - io: Io, - override_zig_lib: ?[]const u8, - override_global_cache: ?[]const u8, - local_cache_strat: union(enum) { - override: []const u8, - search, - global, - }, - preopens: std.process.Preopens, - self_exe_path: switch (builtin.target.os.tag) { - .wasi => void, - else => []const u8, - }, - environ_map: *const std.process.Environ.Map, - cwd: []const u8, - ) Directories { - const wasi = builtin.target.os.tag == .wasi; - - const zig_lib: Cache.Directory = d: { - if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); - if (wasi) break :d getPreopen(preopens, "/lib"); - break :d introspect.findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch |err| { - fatal("unable to find zig installation directory '{s}': {t}", .{ self_exe_path, err }); - }; - }; - - const global_cache: Cache.Directory = d: { - if (override_global_cache) |path| break :d openUnresolved(arena, io, cwd, path, .@"global cache"); - if (wasi) break :d getPreopen(preopens, "/cache"); - const path = introspect.resolveGlobalCacheDir(arena, environ_map) catch |err| { - fatal("unable to resolve zig cache directory: {t}", .{err}); - }; - break :d openUnresolved(arena, io, cwd, path, .@"global cache"); - }; - - const local_cache: Cache.Directory = switch (local_cache_strat) { - .override => |path| openUnresolved(arena, io, cwd, path, .@"local cache"), - .search => d: { - const maybe_path = introspect.resolveSuitableLocalCacheDir(arena, io, cwd) catch |err| { - fatal("unable to resolve zig cache directory: {t}", .{err}); - }; - const path = maybe_path orelse break :d global_cache; - break :d openUnresolved(arena, io, cwd, path, .@"local cache"); - }, - .global => global_cache, - }; - - if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) { - fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache }); - } - if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) { - fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache }); - } - - return .{ - .cwd = cwd, - .zig_lib = zig_lib, - .global_cache = global_cache, - .local_cache = local_cache, - }; - } - fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory { - return .{ - .path = if (std.mem.eql(u8, name, ".")) null else name, - .handle = switch (preopens.get(name) orelse fatal("preopen not found: '{s}'", .{name})) { - .file => fatal("preopen {s} is not a directory", .{name}), - .dir => |d| d, - }, - }; - } - fn openUnresolved( - arena: Allocator, - io: Io, - cwd: []const u8, - unresolved_path: []const u8, - thing: enum { @"zig lib", @"global cache", @"local cache" }, - ) Cache.Directory { - const path = introspect.resolvePath(arena, cwd, &.{unresolved_path}) catch |err| { - fatal("unable to resolve {s} directory: {s}", .{ @tagName(thing), @errorName(err) }); - }; - const nonempty_path = if (path.len == 0) "." else path; - const handle_or_err = switch (thing) { - .@"zig lib" => Io.Dir.cwd().openDir(io, nonempty_path, .{}), - .@"global cache", .@"local cache" => Io.Dir.cwd().createDirPathOpen(io, nonempty_path, .{}), - }; - return .{ - .path = if (path.len == 0) null else path, - .handle = handle_or_err catch |err| { - const extra_str: []const u8 = e: { - if (thing == .@"global cache") switch (err) { - error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++ - "If this location is not writable then consider specifying an alternative with " ++ - "the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.", - else => {}, - }; - break :e ""; - }; - fatal("unable to open {s} directory '{s}': {s}{s}", .{ @tagName(thing), nonempty_path, @errorName(err), extra_str }); - }, - }; - } -}; - /// This small wrapper function just checks whether debug extensions are enabled before checking /// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller, /// preventing debugging features from making it into release builds of the compiler. @@ -1549,7 +1405,7 @@ const CacheUse = union(CacheMode) { }; pub const CreateOptions = struct { - dirs: Directories, + dirs: std.zig.Directories, thread_limit: usize, self_exe_path: ?[]const u8 = null, diff --git a/src/Module.zig b/src/Module.zig new file mode 100644 index 0000000000000000000000000000000000000000..02c65b09fcb13eb9ead45c4e9ce925d01b627e43 --- /dev/null +++ b/src/Module.zig @@ -0,0 +1,523 @@ +//! Corresponds to something that Zig source code can `@import`. +const Module = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const assert = std.debug.assert; + +const target_util = @import("../target.zig"); +const Builtin = @import("../Builtin.zig"); +const Compilation = @import("../Compilation.zig"); +const File = @import("../Zcu.zig").File; + +/// The root directory of the module. Only files inside this directory can be imported. +root: Compilation.Path, +/// Path to the root source file of this module. Relative to `root`. May contain path separators. +root_src_path: []const u8, +/// Name used in compile errors. Looks like "root.foo.bar". +fully_qualified_name: []const u8, +/// The dependency table of this module. The shared dependencies 'std' and +/// 'root' are not specified in every module dependency table, but are stored +/// separately in `Zcu`. 'builtin' is also not stored here, although it is +/// not necessarily the same between all modules. Handling of `@import` in +/// the rest of the compiler must detect these special names and use the +/// correct module instead of consulting `deps`. +deps: Deps = .{}, + +resolved_target: ResolvedTarget, +optimize_mode: std.lang.OptimizeMode, +code_model: std.lang.CodeModel, +single_threaded: bool, +error_tracing: bool, +valgrind: bool, +pic: bool, +strip: bool, +omit_frame_pointer: bool, +stack_check: bool, +stack_protector: u32, +red_zone: bool, +sanitize_c: std.zig.SanitizeC, +sanitize_thread: bool, +fuzz: bool, +unwind_tables: std.lang.UnwindTables, +cc_argv: []const []const u8, +/// (SPIR-V) whether to generate a structured control flow graph or not +structured_cfg: bool, +no_builtin: bool, + +pub const Deps = std.array_hash_map.String(*Module); + +pub const CreateOptions = struct { + paths: Paths, + fully_qualified_name: []const u8, + + cc_argv: []const []const u8, + inherited: Inherited, + global: Compilation.Config, + /// If this is null then `resolved_target` must be non-null. + parent: ?*Module, + + pub const Paths = struct { + root: Compilation.Path, + /// Relative to `root`. May contain path separators. + root_src_path: []const u8, + }; + + pub const Inherited = struct { + /// If this is null then `parent` must be non-null. + resolved_target: ?ResolvedTarget = null, + optimize_mode: ?std.lang.OptimizeMode = null, + code_model: ?std.lang.CodeModel = null, + single_threaded: ?bool = null, + error_tracing: ?bool = null, + valgrind: ?bool = null, + pic: ?bool = null, + strip: ?bool = null, + omit_frame_pointer: ?bool = null, + stack_check: ?bool = null, + /// null means default. + /// 0 means no stack protector. + /// other number means stack protection with that buffer size. + stack_protector: ?u32 = null, + red_zone: ?bool = null, + unwind_tables: ?std.lang.UnwindTables = null, + sanitize_c: ?std.zig.SanitizeC = null, + sanitize_thread: ?bool = null, + fuzz: ?bool = null, + structured_cfg: ?bool = null, + no_builtin: ?bool = null, + }; +}; + +pub const ResolvedTarget = struct { + result: std.Target, + is_native_os: bool, + is_native_abi: bool, + is_explicit_dynamic_linker: bool, + llvm_cpu_features: ?[*:0]const u8 = null, +}; + +pub const CreateError = error{ + OutOfMemory, + ValgrindUnsupportedOnTarget, + TargetRequiresSingleThreaded, + BackendRequiresSingleThreaded, + TargetRequiresPic, + PieRequiresPic, + DynamicLinkingRequiresPic, + TargetHasNoRedZone, + StackCheckUnsupportedByTarget, + StackProtectorUnsupportedByTarget, + StackProtectorUnavailableWithoutLibC, +}; + +/// At least one of `parent` and `resolved_target` must be non-null. +pub fn create(arena: Allocator, options: CreateOptions) !*Module { + if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); + if (options.inherited.fuzz == true) assert(options.global.any_fuzz); + if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); + if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); + if (options.inherited.sanitize_c) |sc| if (sc != .off) assert(options.global.any_sanitize_c != .off); + if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); + + const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; + const target = &resolved_target.result; + + const optimize_mode = options.inherited.optimize_mode orelse + if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; + + const strip = b: { + if (options.inherited.strip) |x| break :b x; + if (options.parent) |p| break :b p.strip; + break :b options.global.root_strip; + }; + + const zig_backend = target_util.zigBackend(target, options.global.use_llvm); + + const valgrind = b: { + if (!target_util.hasValgrindSupport(target, zig_backend)) { + if (options.inherited.valgrind == true) + return error.ValgrindUnsupportedOnTarget; + break :b false; + } + if (options.inherited.valgrind) |x| break :b x; + if (options.parent) |p| break :b p.valgrind; + if (strip) break :b false; + break :b optimize_mode == .Debug; + }; + + const single_threaded = b: { + if (target_util.alwaysSingleThreaded(target)) { + if (options.inherited.single_threaded == false) + return error.TargetRequiresSingleThreaded; + break :b true; + } + + if (options.global.have_zcu) { + if (!target_util.supportsThreads(target, zig_backend)) { + if (options.inherited.single_threaded == false) + return error.BackendRequiresSingleThreaded; + break :b true; + } + } + + if (options.inherited.single_threaded) |x| break :b x; + if (options.parent) |p| break :b p.single_threaded; + break :b target_util.defaultSingleThreaded(target); + }; + + const error_tracing = b: { + if (options.inherited.error_tracing) |x| break :b x; + if (options.parent) |p| break :b p.error_tracing; + break :b options.global.root_error_tracing; + }; + + const pic = b: { + if (target_util.requiresPic(target, options.global.link_libc)) { + if (options.inherited.pic == false) + return error.TargetRequiresPic; + break :b true; + } + if (options.global.pie) { + if (options.inherited.pic == false) + return error.PieRequiresPic; + break :b true; + } + if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) { + if (options.inherited.pic == false) + return error.DynamicLinkingRequiresPic; + break :b true; + } + if (options.inherited.pic) |x| break :b x; + if (options.parent) |p| break :b p.pic; + + // Default to PIC on targets where we default to producing PIEs to make + // the common case of linking objects and static libraries into an + // executable work out of the box. + break :b target_util.defaultPie(target); + }; + + const red_zone = b: { + if (!target_util.hasRedZone(target)) { + if (options.inherited.red_zone == true) + return error.TargetHasNoRedZone; + break :b false; + } + if (options.inherited.red_zone) |x| break :b x; + if (options.parent) |p| break :b p.red_zone; + break :b true; + }; + + const omit_frame_pointer = b: { + if (options.inherited.omit_frame_pointer) |x| break :b x; + if (options.parent) |p| break :b p.omit_frame_pointer; + if (optimize_mode == .ReleaseSmall) { + // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. + // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) + // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). + break :b !target.cpu.arch.isX86(); + } + break :b false; + }; + + const sanitize_thread = b: { + if (options.inherited.sanitize_thread) |x| break :b x; + if (options.parent) |p| break :b p.sanitize_thread; + break :b false; + }; + + const unwind_tables = b: { + if (options.inherited.unwind_tables) |x| break :b x; + if (options.parent) |p| break :b p.unwind_tables; + + break :b target_util.defaultUnwindTables( + target, + options.global.link_libunwind, + sanitize_thread or options.global.any_sanitize_thread, + ); + }; + + const fuzz = b: { + if (options.inherited.fuzz) |x| break :b x; + if (options.parent) |p| break :b p.fuzz; + break :b false; + }; + + const code_model: std.lang.CodeModel = b: { + if (options.inherited.code_model) |x| break :b x; + if (options.parent) |p| break :b p.code_model; + break :b .default; + }; + + const is_safe_mode = switch (optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, + }; + + const sanitize_c: std.zig.SanitizeC = b: { + if (options.inherited.sanitize_c) |x| break :b x; + if (options.parent) |p| break :b p.sanitize_c; + break :b switch (optimize_mode) { + .Debug => .full, + // It's recommended to use the minimal runtime in production + // environments due to the security implications of the full runtime. + // The minimal runtime doesn't provide much benefit over simply + // trapping, however, so we do that instead. + .ReleaseSafe => .trap, + .ReleaseFast, .ReleaseSmall => .off, + }; + }; + + const stack_check = b: { + if (!target_util.supportsStackProbing(target, zig_backend)) { + if (options.inherited.stack_check == true) + return error.StackCheckUnsupportedByTarget; + break :b false; + } + if (options.inherited.stack_check) |x| break :b x; + if (options.parent) |p| break :b p.stack_check; + break :b is_safe_mode; + }; + + const stack_protector: u32 = sp: { + const use_zig_backend = options.global.have_zcu or + (options.global.any_c_source_files and options.global.c_frontend == .aro); + if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnsupportedByTarget; + } + break :sp 0; + } + + if (options.global.any_c_source_files and options.global.c_frontend == .clang and + !target_util.clangSupportsStackProtector(target)) + { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnsupportedByTarget; + } + break :sp 0; + } + + // This logic is checking for linking libc because otherwise our start code + // which is trying to set up TLS (i.e. the fs/gs registers) but the stack + // protection code depends on fs/gs registers being already set up. + // If we were able to annotate start code, or perhaps the entire std lib, + // as being exempt from stack protection checks, we could change this logic + // to supporting stack protection even when not linking libc. + // TODO file issue about this + if (!options.global.link_libc) { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnavailableWithoutLibC; + } + break :sp 0; + } + + if (options.inherited.stack_protector) |x| break :sp x; + if (options.parent) |p| break :sp p.stack_protector; + if (!is_safe_mode) break :sp 0; + + break :sp target_util.default_stack_protector_buffer_size; + }; + + const structured_cfg = b: { + if (options.inherited.structured_cfg) |x| break :b x; + if (options.parent) |p| break :b p.structured_cfg; + // We always want a structured control flow in shaders. This option is + // only relevant for OpenCL kernels. + break :b switch (target.os.tag) { + .opencl => false, + else => true, + }; + }; + + const no_builtin = b: { + if (options.inherited.no_builtin) |x| break :b x; + if (options.parent) |p| break :b p.no_builtin; + + break :b target.cpu.arch.isBpf(); + }; + + const llvm_cpu_features: ?[*:0]const u8 = b: { + if (resolved_target.llvm_cpu_features) |x| break :b x; + if (!options.global.use_llvm) break :b null; + + var buf = std.array_list.Managed(u8).init(arena); + var disabled_features = std.array_list.Managed(u8).init(arena); + defer disabled_features.deinit(); + + // Append disabled features after enabled ones, so that their effects aren't overwritten. + for (target.cpu.arch.allFeaturesList()) |feature| { + if (feature.llvm_name) |llvm_name| { + // Ignore these until we figure out how to handle the concept of omitting features. + // See https://github.com/ziglang/zig/issues/23539 + if (target_util.isDynamicAMDGCNFeature(target, feature)) continue; + + if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue; + if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue; + + var is_enabled = target.cpu.features.isEnabled(feature.index); + if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) { + is_enabled = !omit_frame_pointer; + } + + if (is_enabled) { + try buf.ensureUnusedCapacity(2 + llvm_name.len); + buf.appendAssumeCapacity('+'); + buf.appendSliceAssumeCapacity(llvm_name); + buf.appendAssumeCapacity(','); + } else { + try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); + disabled_features.appendAssumeCapacity('-'); + disabled_features.appendSliceAssumeCapacity(llvm_name); + disabled_features.appendAssumeCapacity(','); + } + } + } + + try buf.appendSlice(disabled_features.items); + if (buf.items.len == 0) break :b ""; + assert(std.mem.endsWith(u8, buf.items, ",")); + buf.items[buf.items.len - 1] = 0; + buf.shrinkAndFree(buf.items.len); + break :b buf.items[0 .. buf.items.len - 1 :0].ptr; + }; + + const mod = try arena.create(Module); + mod.* = .{ + .root = options.paths.root, + .root_src_path = options.paths.root_src_path, + .fully_qualified_name = options.fully_qualified_name, + .resolved_target = .{ + .result = target.*, + .is_native_os = resolved_target.is_native_os, + .is_native_abi = resolved_target.is_native_abi, + .is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker, + .llvm_cpu_features = llvm_cpu_features, + }, + .optimize_mode = optimize_mode, + .single_threaded = single_threaded, + .error_tracing = error_tracing, + .valgrind = valgrind, + .pic = pic, + .strip = strip, + .omit_frame_pointer = omit_frame_pointer, + .stack_check = stack_check, + .stack_protector = stack_protector, + .code_model = code_model, + .red_zone = red_zone, + .sanitize_c = sanitize_c, + .sanitize_thread = sanitize_thread, + .fuzz = fuzz, + .unwind_tables = unwind_tables, + .cc_argv = options.cc_argv, + .structured_cfg = structured_cfg, + .no_builtin = no_builtin, + }; + return mod; +} + +/// All fields correspond to `CreateOptions`. +pub const LimitedOptions = struct { + root: Compilation.Path, + root_src_path: []const u8, + fully_qualified_name: []const u8, +}; + +/// This one can only be used if the Module will only be used for AstGen and earlier in +/// the pipeline. Illegal behavior occurs if a limited module touches Sema. +pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Module { + const mod = try gpa.create(Module); + mod.* = .{ + .root = options.root, + .root_src_path = options.root_src_path, + .fully_qualified_name = options.fully_qualified_name, + + .resolved_target = undefined, + .optimize_mode = undefined, + .code_model = undefined, + .single_threaded = undefined, + .error_tracing = undefined, + .valgrind = undefined, + .pic = undefined, + .strip = undefined, + .omit_frame_pointer = undefined, + .stack_check = undefined, + .stack_protector = undefined, + .red_zone = undefined, + .sanitize_c = undefined, + .sanitize_thread = undefined, + .fuzz = undefined, + .unwind_tables = undefined, + .cc_argv = undefined, + .structured_cfg = undefined, + .no_builtin = undefined, + }; + return mod; +} + +/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task. +pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: Compilation.Directories) Allocator.Error!*Module { + const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash()); + const new = try arena.create(Module); + new.* = .{ + .root = try .fromRoot(arena, dirs, .global_cache, sub_path), + .root_src_path = "builtin.zig", + .fully_qualified_name = "builtin", + .resolved_target = .{ + .result = opts.target, + // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. + .is_native_os = false, + .is_native_abi = false, + .is_explicit_dynamic_linker = false, + .llvm_cpu_features = null, + }, + .optimize_mode = opts.optimize_mode, + .single_threaded = opts.single_threaded, + .error_tracing = opts.error_tracing, + .valgrind = opts.valgrind, + .pic = opts.pic, + .strip = opts.strip, + .omit_frame_pointer = opts.omit_frame_pointer, + .code_model = opts.code_model, + .sanitize_thread = opts.sanitize_thread, + .fuzz = opts.fuzz, + .unwind_tables = opts.unwind_tables, + .cc_argv = &.{}, + // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. + .stack_check = false, + .stack_protector = 0, + .red_zone = false, + .sanitize_c = .off, + .structured_cfg = false, + .no_builtin = false, + }; + return new; +} + +/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module. +pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin { + assert(global.have_zcu); + return .{ + .target = m.resolved_target.result, + .zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm), + .output_mode = global.output_mode, + .link_mode = global.link_mode, + .unwind_tables = m.unwind_tables, + .is_test = global.is_test, + .single_threaded = m.single_threaded, + .link_libc = global.link_libc, + .link_libcpp = global.link_libcpp, + .optimize_mode = m.optimize_mode, + .error_tracing = m.error_tracing, + .valgrind = m.valgrind, + .sanitize_thread = m.sanitize_thread, + .fuzz = m.fuzz, + .pic = m.pic, + .pie = global.pie, + .strip = m.strip, + .code_model = m.code_model, + .omit_frame_pointer = m.omit_frame_pointer, + .wasi_exec_model = global.wasi_exec_model, + }; +} diff --git a/src/Package.zig b/src/Package.zig deleted file mode 100644 index 8fb9995bd81315343e9b1da8d1741774ae4e9d82..0000000000000000000000000000000000000000 --- a/src/Package.zig +++ /dev/null @@ -1,209 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; - -pub const Module = @import("Package/Module.zig"); -pub const Fetch = @import("Package/Fetch.zig"); -pub const build_zig_basename = "build.zig"; -pub const Manifest = @import("Package/Manifest.zig"); - -pub const Fingerprint = packed struct(u64) { - id: u32, - checksum: u32, - - pub fn generate(rng: std.Random, name: []const u8) Fingerprint { - return .{ - .id = rng.intRangeLessThan(u32, 1, 0xffffffff), - .checksum = std.hash.Crc32.hash(name), - }; - } - - pub fn validate(n: Fingerprint, name: []const u8) bool { - switch (n.id) { - 0x00000000, 0xffffffff => return false, - else => return std.hash.Crc32.hash(name) == n.checksum, - } - } - - pub fn int(n: Fingerprint) u64 { - return @bitCast(n); - } -}; - -/// A user-readable, file system safe hash that identifies an exact package -/// snapshot, including file contents. -/// -/// The hash is not only to prevent collisions but must resist attacks where -/// the adversary fully controls the contents being hashed. Thus, it contains -/// a full SHA-256 digest. -/// -/// This data structure can be used to store the legacy hash format too. Legacy -/// hash format is scheduled to be removed after 0.14.0 is tagged. -/// -/// There's also a third way this structure is used. When using path rather than -/// hash, a unique hash is still needed, so one is computed based on the path. -pub const Hash = struct { - /// Maximum size of a package hash. Unused bytes at the end are - /// filled with zeroes. - /// - /// Assumed to be already validated. - bytes: [max_len]u8, - - pub const Algo = std.crypto.hash.sha2.Sha256; - pub const Digest = [Algo.digest_length]u8; - - /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" - pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; - - /// Asserts `s` is valid. - pub fn fromSlice(s: []const u8) Hash { - assert(validate(s) == .ok); - var result: Hash = undefined; - @memcpy(result.bytes[0..s.len], s); - @memset(result.bytes[s.len..], 0); - return result; - } - - pub const Validation = enum { ok, short, long, incomplete }; - - pub fn validate(s: []const u8) Validation { - if (s.len > max_len) return .long; - if (s.len < 44) return .short; - const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-'); - if (n_dashes < 2) return .incomplete; - return .ok; - } - - test validate { - try std.testing.expectEqual(.short, validate("")); - } - - pub fn toSlice(ph: *const Hash) []const u8 { - var end: usize = ph.bytes.len; - while (true) { - end -= 1; - if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; - } - } - - pub fn eql(a: *const Hash, b: *const Hash) bool { - return std.mem.eql(u8, &a.bytes, &b.bytes); - } - - /// Produces "$name-$semver-$hashplus". - /// * name is the name field from build.zig.zon, asserted to be at most 32 - /// bytes and assumed be a valid zig identifier - /// * semver is the version field from build.zig.zon, asserted to be at - /// most 32 bytes - /// * hashplus is the following 33-byte array, base64 encoded using -_ to make - /// it filesystem safe: - /// - (4 bytes) LE u32 Package ID - /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated - /// - (25 bytes) truncated SHA-256 digest of hashed files of the package - pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { - assert(name.len <= 32); - assert(ver.len <= 32); - var result: Hash = undefined; - var buf: std.ArrayList(u8) = .initBuffer(&result.bytes); - buf.appendSliceAssumeCapacity(name); - buf.appendAssumeCapacity('-'); - buf.appendSliceAssumeCapacity(ver); - buf.appendAssumeCapacity('-'); - var hashplus: [33]u8 = undefined; - std.mem.writeInt(u32, hashplus[0..4], id, .little); - std.mem.writeInt(u32, hashplus[4..8], size, .little); - hashplus[8..].* = digest[0..25].*; - _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); - @memset(buf.unusedCapacitySlice(), 0); - return result; - } - - /// Produces a unique hash based on the path provided. The result should - /// not be user-visible. - pub fn initPath(sub_path: []const u8, is_global: bool) Hash { - var result: Hash = .{ .bytes = @splat(0) }; - var i: usize = 0; - if (is_global) { - result.bytes[0] = '/'; - i += 1; - } - if (i + sub_path.len <= result.bytes.len) { - @memcpy(result.bytes[i..][0..sub_path.len], sub_path); - return result; - } - var bin_digest: [Algo.digest_length]u8 = undefined; - Algo.hash(sub_path, &bin_digest, .{}); - _ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable; - return result; - } - - pub fn projectId(hash: *const Hash) ProjectId { - const bytes = hash.toSlice(); - const name = std.mem.sliceTo(bytes, '-'); - const encoded_hashplus = bytes[bytes.len - 44 ..]; - var hashplus: [33]u8 = undefined; - std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable; - const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little); - return .init(name, fingerprint_id); - } - - test projectId { - const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw"); - const project_id = hash.projectId(); - - var expected_name: [32]u8 = @splat(0); - expected_name[0.."pulseaudio".len].* = "pulseaudio".*; - try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); - - try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id); - } - - test "projectId with dashes in the base64" { - const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC"); - const project_id = hash.projectId(); - - var expected_name: [32]u8 = @splat(0); - expected_name[0.."dvui".len].* = "dvui".*; - try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); - - try std.testing.expectEqual(0x99490101, project_id.fingerprint_id); - } -}; - -/// Minimum information required to identify whether a package is an artifact -/// of a given project. -pub const ProjectId = struct { - /// Bytes after name.len are set to zero. - padded_name: [32]u8, - fingerprint_id: u32, - - pub fn init(name: []const u8, fingerprint_id: u32) ProjectId { - var padded_name: [32]u8 = @splat(0); - @memcpy(padded_name[0..name.len], name); - return .{ - .padded_name = padded_name, - .fingerprint_id = fingerprint_id, - }; - } - - pub fn eql(a: *const ProjectId, b: *const ProjectId) bool { - return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name); - } - - pub fn hash(a: *const ProjectId) u64 { - const x: u64 = @bitCast(a.padded_name[0..8].*); - return std.hash.int(x | a.fingerprint_id); - } -}; - -test Hash { - const example_digest: Hash.Digest = .{ - 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, - 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, - }; - const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); - try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); -} - -test { - _ = Fetch; -} diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig deleted file mode 100644 index 20e19cdaa7ad2e55b9ddb07f490df1ffdaa26eea..0000000000000000000000000000000000000000 --- a/src/Package/Fetch.zig +++ /dev/null @@ -1,2283 +0,0 @@ -//! Represents one independent job whose responsibility is to: -//! -//! 1. Check the local zig package directory to see if the hash already exists. -//! If so, load, parse, and validate the build.zig.zon file therein, and -//! goto step 9. Likewise if the location is a relative path, treat this -//! the same as a cache hit. Otherwise, proceed. -//! 2. Check the global package cache for a compressed tarball matching the -//! hash. If it is found, unpack the contents into a temporary directory inside -//! project local zig cache. Rename this directory into the local zig package -//! directory and goto step 9, skipping step 10. -//! 3. Fetch and unpack a URL into a temporary directory. -//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed -//! for the file to be missing, in which case this fetched package is considered -//! to be a "naked" package. -//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by -//! deleting excluded files. If any files had errors for files that were -//! ultimately excluded, those errors should be ignored, such as failure to -//! create symlinks that weren't supposed to be included anyway. -//! 6. Compute the package hash based on the remaining files in the temporary -//! directory. -//! 7. Rename the temporary directory into the local zig package directory. If -//! the hash already exists, delete the temporary directory and leave the zig -//! package directory untouched as it may be in use. This is done even if -//! the hash is invalid, in case the package with the different hash is used -//! in the future. -//! 8. Validate the computed hash against the expected hash. If invalid, -//! this job is done. -//! 9. Spawn a new fetch job for each dependency in the manifest file. Use -//! a mutex and a hash map so that redundant jobs do not get queued up. -//! 10.Compress the package directory and store it into the global package -//! cache. -//! -//! All of this must be done with only referring to the state inside this struct -//! because this work will be done in a dedicated thread. -const Fetch = @This(); - -const builtin = @import("builtin"); -const native_os = builtin.os.tag; - -const std = @import("std"); -const Io = std.Io; -const fs = std.fs; -const log = std.log.scoped(.fetch); -const assert = std.debug.assert; -const ascii = std.ascii; -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const git = @import("Fetch/git.zig"); -const Package = @import("../Package.zig"); -const Manifest = Package.Manifest; -const ErrorBundle = std.zig.ErrorBundle; - -arena: std.heap.ArenaAllocator, -location: Location, -location_tok: std.zig.Ast.TokenIndex, -hash_tok: std.zig.Ast.OptionalTokenIndex, -name_tok: std.zig.Ast.TokenIndex, -lazy_status: LazyStatus, -/// Same as `parent_packge_root` except it is unchanged when recursing into -/// relative file paths (as opposed to URL). -remote_package_root: Cache.Path, -parent_package_root: Cache.Path, -parent_manifest_ast: ?*const std.zig.Ast, -prog_node: std.Progress.Node, -job_queue: *JobQueue, -/// If true, don't add an error for a missing hash. This flag is not passed -/// down to recursive dependencies. It's intended to be used only be the CLI. -omit_missing_hash_error: bool, -/// If true, don't fail when a manifest file is missing the `paths` field, -/// which specifies inclusion rules. This is intended to be true for the first -/// fetch task and false for the recursive dependencies. -allow_missing_paths_field: bool, -/// If true and URL points to a Git repository, will use the latest commit. -use_latest_commit: bool, - -// Above this are fields provided as inputs to `run`. -// Below this are fields populated by `run`. - -/// Relative to the build root of the root package. -package_root: Cache.Path, -error_bundle: ErrorBundle.Wip, -manifest: Manifest, -manifest_ast: std.zig.Ast, -have_manifest: bool, -computed_hash: ComputedHash, -/// Fetch logic notices whether a package has a build.zig file and sets this flag. -has_build_zig: bool, -/// Indicates whether the task aborted due to an out-of-memory condition. -oom_flag: bool, -/// If `use_latest_commit` was true, this will be set to the commit that was used. -/// If the resource pointed to by the location is not a Git-repository, this -/// will be left unchanged. -latest_commit: ?git.Oid, - -// This field is used by the CLI only, untouched by this file. - -/// The module for this `Fetch` tasks's package, which exposes `build.zig` as -/// the root source file. -module: ?*Package.Module, - -pub const LazyStatus = enum { - /// Not lazy. - eager, - /// Lazy, found. - available, - /// Lazy, not found. - unavailable, -}; - -pub const LocalStorage = struct { - cache_root: Cache.Path, - /// Path to "zig-pkg" inside the package in which the user ran `zig build`. - pkg_root: Cache.Path, -}; - -/// Contains shared state among all `Fetch` tasks. -pub const JobQueue = struct { - io: Io, - mutex: Io.Mutex = .init, - /// It's an array hash map so that it can be sorted before rendering the - /// dependencies.zig source file. - /// Protected by `mutex`. - table: Table = .{}, - /// `table` may be missing some tasks such as ones that failed, so this - /// field contains references to all of them. - /// Protected by `mutex`. - all_fetches: std.ArrayList(*Fetch) = .empty, - prog_node: std.Progress.Node, - - http_client: *std.http.Client, - /// This tracks `Fetch` tasks as well as recompression tasks. - group: Io.Group = .init, - global_cache: Cache.Directory, - /// If `null`, indicates fetch globally only. - local_storage: ?*const LocalStorage, - /// If true then, no fetching occurs, and: - /// * The `global_cache` directory is assumed to be the direct parent - /// directory of on-disk packages rather than having the "p/" directory - /// prefix inside of it. - /// * An error occurs if any non-lazy packages are not already present in - /// the package cache directory. - /// * Missing hash field causes an error, and no fetching occurs so it does - /// not print the correct hash like usual. - read_only: bool, - recursive: bool, - /// Dumps hash information to stdout which can be used to troubleshoot why - /// two hashes of the same package do not match. - /// If this is true, `recursive` must be false. - debug_hash: bool, - mode: Mode, - /// Set of hashes that will be additionally fetched even if they are marked - /// as lazy. - unlazy_set: UnlazySet = .{}, - /// Identifies paths that override all packages in the tree with matching - /// project ids. - fork_set: ForkSet = .{}, - - pub const Mode = enum { - /// Non-lazy dependencies are always fetched. - /// Lazy dependencies are fetched only when needed. - needed, - /// Both non-lazy and lazy dependencies are always fetched. - all, - }; - pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch); - pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void); - pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false); - - pub const Fork = struct { - path: Cache.Path, - manifest_ast: std.zig.Ast, - manifest: Package.Manifest, - uses: usize, - - pub const Context = struct { - pub fn hash(_: @This(), a: Fork) u32 { - const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); - return @truncate(project_id.hash()); - } - - pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool { - const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); - const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); - return a_project_id.eql(&b_project_id); - } - }; - - pub const Adapter = struct { - pub fn hash(_: @This(), a: Package.ProjectId) u32 { - return @truncate(a.hash()); - } - - pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool { - const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); - return a_project_id.eql(&b_project_id); - } - }; - }; - - pub fn deinit(jq: *JobQueue) void { - const io = jq.io; - jq.group.cancel(io); - if (jq.all_fetches.items.len == 0) return; - const gpa = jq.all_fetches.items[0].arena.child_allocator; - jq.table.deinit(gpa); - // These must be deinitialized in reverse order because subsequent - // `Fetch` instances are allocated in prior ones' arenas. - // Sorry, I know it's a bit weird, but it slightly simplifies the - // critical section. - while (jq.all_fetches.pop()) |f| f.deinit(); - jq.all_fetches.deinit(gpa); - jq.* = undefined; - } - - /// Dumps all subsequent error bundles into the first one. - pub fn consolidateErrors(jq: *JobQueue) !void { - const root = &jq.all_fetches.items[0].error_bundle; - const gpa = root.gpa; - for (jq.all_fetches.items[1..]) |fetch| { - if (fetch.error_bundle.root_list.items.len > 0) { - var bundle = try fetch.error_bundle.toOwnedBundle(""); - defer bundle.deinit(gpa); - try root.addBundleAsRoots(bundle); - } - } - } - - /// Creates the dependencies.zig source code for the build runner to obtain - /// via `@import("@dependencies")`. - pub fn createDependenciesSource(jq: *JobQueue, buf: *std.array_list.Managed(u8)) Allocator.Error!void { - const keys = jq.table.keys(); - - assert(keys.len != 0); // caller should have added the first one - if (keys.len == 1) { - // This is the first one. It must have no dependencies. - return createEmptyDependenciesSource(buf); - } - - try buf.appendSlice("pub const packages = struct {\n"); - - // Ensure the generated .zig file is deterministic. - jq.table.sortUnstable(@as(struct { - keys: []const Package.Hash, - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); - } - }, .{ .keys = keys })); - - for (keys, jq.table.values()) |*hash, fetch| { - if (fetch == jq.all_fetches.items[0]) { - // The first one is a dummy package for the current project. - continue; - } - - const hash_slice = hash.toSlice(); - - try buf.print( - \\ pub const {f} = struct {{ - \\ - , .{std.zig.fmtId(hash_slice)}); - - lazy: { - switch (fetch.lazy_status) { - .eager => break :lazy, - .available => { - try buf.appendSlice( - \\ pub const available = true; - \\ - ); - break :lazy; - }, - .unavailable => { - try buf.appendSlice( - \\ pub const available = false; - \\ }; - \\ - ); - continue; - }, - } - } - - try buf.print( - \\ pub const build_root = "{f}"; - \\ - , .{std.fmt.alt(fetch.package_root, .formatEscapeString)}); - - if (fetch.has_build_zig) { - try buf.print( - \\ pub const build_zig = @import("{f}"); - \\ - , .{std.zig.fmtString(hash_slice)}); - } - - if (fetch.have_manifest) { - const manifest = &fetch.manifest; - try buf.appendSlice( - \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.print( - " .{{ \"{f}\", \"{f}\" }},\n", - .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, - ); - } - - try buf.appendSlice( - \\ }; - \\ }; - \\ - ); - } else { - try buf.appendSlice( - \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; - \\ }; - \\ - ); - } - } - - try buf.appendSlice( - \\}; - \\ - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - - const root_fetch = jq.all_fetches.items[0]; - assert(root_fetch.have_manifest); - const root_manifest = &root_fetch.manifest; - - for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { - const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.print( - " .{{ \"{f}\", \"{f}\" }},\n", - .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, - ); - } - try buf.appendSlice("};\n"); - } - - pub fn createEmptyDependenciesSource(buf: *std.array_list.Managed(u8)) Allocator.Error!void { - try buf.appendSlice( - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - } - - fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void { - const pkg_hash_slice = package_hash.toSlice(); - - const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice}); - defer prog_node.end(); - - var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; - const dest_path: Cache.Path = .{ - .root_dir = jq.global_cache, - .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable, - }; - - const gpa = jq.http_client.allocator; - - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) { - error.Canceled => |e| return e, - error.ReadFailed => comptime unreachable, - error.WriteFailed => comptime unreachable, - else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), - }; - } - - fn recompressFallible( - jq: *JobQueue, - arena: Allocator, - dest_path: Cache.Path, - pkg_hash_slice: []const u8, - package_root: Cache.Path, - prog_node: std.Progress.Node, - ) !void { - const gpa = jq.http_client.allocator; - const io = jq.io; - - // We have to walk the file system up front in order to sort the file - // list for determinism purposes. The hash of the recompressed file is - // not critical because the true hash is based on the content alone. - // However, if we want Zig users to be able to share cached package - // data with each other via peer-to-peer protocols, we benefit greatly - // from the data being identical on everyone's computers. - var scanned_files: std.ArrayList(ScannedFile) = .empty; - defer scanned_files.deinit(gpa); - - var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true }); - defer pkg_dir.close(io); - - { - var walker = try pkg_dir.walk(gpa); - defer walker.deinit(); - - while (try walker.next(io)) |entry| { - const symlink = switch (entry.kind) { - .directory => continue, - .file => false, - .sym_link => true, - else => return error.IllegalFileType, - }; - const entry_path = try arena.dupe(u8, entry.path); - // If necessary, normalize path separators to POSIX-style since the tar format requires that. - if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) { - std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix); - } - try scanned_files.append(gpa, .{ - .ptr = entry_path.ptr, - .len = @intCast(entry_path.len), - .symlink = symlink, - }); - } - - std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp); - } - - prog_node.setEstimatedTotalItems(scanned_files.items.len); - - var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ - .make_path = true, - .replace = true, - }); - defer atomic_file.deinit(io); - - var file_write_buffer: [4096]u8 = undefined; - var file_writer = atomic_file.file.writer(io, &file_write_buffer); - - var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) { - error.WriteFailed => return file_writer.err.?, - }; - - var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; - archiver.prefix = pkg_hash_slice; - - var file_read_buffer: [4096]u8 = undefined; - var link_buf: [fs.max_path_bytes]u8 = undefined; - - for (scanned_files.items) |scanned_file| { - const entry_path = scanned_file.ptr[0..scanned_file.len]; - if (scanned_file.symlink) { - const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)]; - archiver.writeLink(entry_path, link_name, .{}) catch |err| switch (err) { - error.WriteFailed => return file_writer.err.?, - else => |e| return e, - }; - } else { - var file = try pkg_dir.openFile(io, entry_path, .{}); - defer file.close(io); - var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer); - archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - error.WriteFailed => return file_writer.err.?, - else => |e| return e, - }; - } - prog_node.completeOne(); - } - - // intentionally omitting the pointless trailer - //try archiver.finish(); - compress.finish() catch |err| switch (err) { - error.WriteFailed => return file_writer.err.?, - }; - try file_writer.flush(); - try atomic_file.replace(io); - } -}; - -const ScannedFile = struct { - ptr: [*]const u8, - len: u32, - symlink: bool, -}; - -fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool { - return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]); -} - -pub const Location = union(enum) { - remote: Remote, - /// A directory found inside the parent package. - relative_path: Cache.Path, - /// Recursive Fetch tasks will never use this Location, but it may be - /// passed in by the CLI. Indicates the file contents here should be copied - /// into the global package cache. It may be a file relative to the cwd or - /// absolute, in which case it should be treated exactly like a `file://` - /// URL, or a directory, in which case it should be treated as an - /// already-unpacked directory (but still needs to be copied into the - /// global package cache and have inclusion rules applied). - path_or_url: []const u8, - - pub const Remote = struct { - url: []const u8, - /// If this is null it means the user omitted the hash field from a dependency. - /// It will be an error but the logic should still fetch and print the discovered hash. - hash: ?Package.Hash, - }; -}; - -pub const RunError = error{ - OutOfMemory, - Canceled, - /// This error code is intended to be handled by inspecting the - /// `error_bundle` field. - FetchFailed, -}; - -pub fn run(f: *Fetch) RunError!void { - const job_queue = f.job_queue; - const io = job_queue.io; - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - - try eb.init(gpa); - - // Check the global zig package cache to see if the hash already exists. If - // so, load, parse, and validate the build.zig.zon file therein, and skip - // ahead to queuing up jobs for dependencies. Likewise if the location is a - // relative path, treat this the same as a cache hit. Otherwise, proceed. - - const remote = switch (f.location) { - .relative_path => |pkg_root| { - if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( - f.location_tok, - try eb.addString("expected path relative to build root; found absolute path"), - ); - if (f.hash_tok.unwrap()) |hash_tok| return f.fail( - hash_tok, - try eb.addString("path-based dependencies are not hashed"), - ); - // Packages fetched by URL may not use relative paths to escape outside the - // fetched package directory from within the package cache. - - // This code path is only reachable recursively and the sub_path - // will already have been resolved to no longer have extra ".." or - // "." components. - assert(job_queue.local_storage != null); - log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{ - pkg_root.sub_path, f.remote_package_root.sub_path, - }); - assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir)); - if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail( - f.location_tok, - try eb.printString("dependency path outside project: '{f}'", .{pkg_root}), - ); - f.package_root = pkg_root; - try loadManifest(f, pkg_root); - if (!f.has_build_zig) try checkBuildFileExistence(f); - if (!job_queue.recursive) return; - return queueJobsForDeps(f); - }, - .remote => |remote| remote, - .path_or_url => |path_or_url| { - if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| { - var resource: Resource = .{ .dir = dir }; - return f.runResource(path_or_url, &resource, null, false); - } else |dir_err| { - var server_header_buffer: [init_resource_buffer_size]u8 = undefined; - - const file_err = if (dir_err == error.NotDir) e: { - if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| { - var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) }; - return f.runResource(path_or_url, &resource, null, false); - } else |err| break :e err; - } else dir_err; - - const uri = std.Uri.parse(path_or_url) catch |uri_err| { - return f.fail(0, try eb.printString( - "'{s}' could not be recognized as a file path ({t}) or an URL ({t})", - .{ path_or_url, file_err, uri_err }, - )); - }; - var resource: Resource = undefined; - try f.initResource(uri, &resource, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false); - } - }, - }; - - var resource_buffer: [init_resource_buffer_size]u8 = undefined; - - if (remote.hash) |expected_hash| { - const expected_project_id: Package.ProjectId = expected_hash.projectId(); - if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) |fork| { - log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name }); - fork.uses += 1; - f.package_root = fork.path; - f.remote_package_root = f.package_root; - f.manifest_ast = fork.manifest_ast; - f.manifest = fork.manifest; - f.have_manifest = true; - try checkBuildFileExistence(f); - if (!job_queue.recursive) return; - return queueJobsForDeps(f); - } - - if (job_queue.local_storage) |ls| { - const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice()); - if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { - assert(f.lazy_status != .unavailable); - f.package_root = package_root; - f.remote_package_root = f.package_root; - try loadManifest(f, f.package_root); - try checkBuildFileExistence(f); - if (!job_queue.recursive) return; - return queueJobsForDeps(f); - } else |err| switch (err) { - error.FileNotFound => { - log.debug("FileNotFound: {f}", .{package_root}); - if (job_queue.read_only and f.lazy_status == .eager) return f.fail( - f.name_tok, - try eb.printString("package not found at '{f}'", .{package_root}), - ); - }, - error.Canceled => |e| return e, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ - package_root, e, - }), - }); - return error.FetchFailed; - }, - } - } - - // Check global cache before remote fetch. - const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()}); - const cached_tarball_path: Cache.Path = .{ - .root_dir = job_queue.global_cache, - .sub_path = cached_tarball_sub_path, - }; - if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| { - log.debug("found global cached tarball {f}", .{cached_tarball_path}); - var resource: Resource = .{ .file = file.reader(io, &resource_buffer) }; - return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true); - } else |err| switch (err) { - error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}), - error.Canceled => |e| return e, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{ - cached_tarball_path, e, - }), - }); - return error.FetchFailed; - }, - } - - switch (f.lazy_status) { - .eager => {}, - .available => if (!job_queue.unlazy_set.contains(expected_hash)) { - f.lazy_status = .unavailable; - return; - }, - .unavailable => unreachable, - } - } else if (job_queue.read_only) { - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("dependency is missing hash field"), - .src_loc = try f.srcLoc(f.location_tok), - }); - return error.FetchFailed; - } - - // Fetch and unpack the remote into a temporary directory. - const uri = std.Uri.parse(remote.url) catch |err| return f.fail( - f.location_tok, - try eb.printString("invalid URI: {t}", .{err}), - ); - var resource: Resource = undefined; - try f.initResource(uri, &resource, &resource_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false); -} - -pub fn deinit(f: *Fetch) void { - f.error_bundle.deinit(); - f.arena.deinit(); -} - -/// Consumes `resource`, even if an error is returned. -fn runResource( - f: *Fetch, - uri_path: []const u8, - resource: *Resource, - remote_hash: ?Package.Hash, - disable_recompress: bool, -) RunError!void { - const job_queue = f.job_queue; - assert(!job_queue.read_only); - - const io = job_queue.io; - defer resource.deinit(io); - - const arena = f.arena.allocator(); - const eb = &f.error_bundle; - const rand_int = r: { - var x: u64 = undefined; - io.random(@ptrCast(&x)); - break :r x; - }; - const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int); - const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path; - const tmp_directory_path: Cache.Path = if (job_queue.local_storage) |ls| - try ls.pkg_root.join(arena, tmp_dir_sub_path) - else - .{ - .root_dir = job_queue.global_cache, - .sub_path = tmp_tmp_dir_sub_path, - }; - - const package_sub_path = blk: { - var tmp_directory: Cache.Directory = .{ - .path = tmp_directory_path.sub_path, - .handle = handle: { - const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{ - .open_options = .{ .iterate = true }, - }) catch |err| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{ - tmp_directory_path, err, - }), - }); - return error.FetchFailed; - }; - break :handle dir; - }, - }; - defer tmp_directory.handle.close(io); - - // Fetch and unpack a resource into a temporary directory. - var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); - - const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; - - // Load, parse, and validate the unpacked build.zig.zon file. It is allowed - // for the file to be missing, in which case this fetched package is - // considered to be a "naked" package. - try loadManifest(f, pkg_path); - - const filter: Filter = .{ - .include_paths = if (f.have_manifest) f.manifest.paths else .{}, - }; - - // Ignore errors that were excluded by manifest, such as failure to - // create symlinks that weren't supposed to be included anyway. - try unpack_result.validate(f, filter); - - // Apply the manifest's inclusion rules to the temporary directory by - // deleting excluded files. - // Empty directories have already been omitted by `unpackResource`. - // Compute the package hash based on the remaining files in the temporary - // directory. - f.computed_hash = try computeHash(f, pkg_path, filter); - - if (unpack_result.root_dir.len > 0) - break :blk try tmp_directory_path.join(arena, unpack_result.root_dir); - - break :blk tmp_directory_path; - }; - - const computed_package_hash = computedPackageHash(f); - - // Rename the temporary directory into the local zig package directory. If - // the hash already exists, delete the temporary directory and leave the - // zig package directory untouched as it may be in use. This is done even - // if the hash is invalid, in case the package with the different hash is - // used in the future. - if (job_queue.local_storage) |ls| { - f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice()); - renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "failed renaming temporary directory {f} into package cache directory {f}: {t}", - .{ package_sub_path, f.package_root, err }, - ) }); - return error.FetchFailed; - }; - } else { - f.package_root = tmp_directory_path; - } - f.remote_package_root = f.package_root; - - if (!disable_recompress) { - // Spin off a task to recompress the tarball, with filtered files deleted, into - // the global cache. - job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root }); - } - - // Remove temporary directory root if not already renamed to global cache. - if (!package_sub_path.eql(tmp_directory_path)) { - tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }), - }; - } - - // Validate the computed hash against the expected hash. If invalid, this - // job is done. - - if (remote_hash) |declared_hash| { - const hash_tok = f.hash_tok.unwrap().?; - if (!computed_package_hash.eql(&declared_hash)) { - return f.fail(hash_tok, try eb.printString( - "hash mismatch: manifest declares {s} but the fetched package has {s}", - .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, - )); - } - } else if (!f.omit_missing_hash_error) { - const notes_len = 1; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("dependency is missing hash field"), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = notes_len, - }); - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), - })); - return error.FetchFailed; - } - - // Spawn a new fetch job for each dependency in the manifest file. Use - // a mutex and a hash map so that redundant jobs do not get queued up. - if (!job_queue.recursive) return; - return queueJobsForDeps(f); -} - -pub fn computedPackageHash(f: *const Fetch) Package.Hash { - const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); - if (f.have_manifest) { - const man = &f.manifest; - var version_buffer: [32]u8 = undefined; - const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer; - return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); - } - // In the future build.zig.zon fields will be added to allow overriding these values - // for naked tarballs. - return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); -} - -/// `computeHash` gets a free check for the existence of `build.zig`, but when -/// not computing a hash, we need to do a syscall to check for it. -fn checkBuildFileExistence(f: *Fetch) RunError!void { - const io = f.job_queue.io; - const eb = &f.error_bundle; - if (f.package_root.access(io, Package.build_zig_basename, .{})) |_| { - f.has_build_zig = true; - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to access '{f}{s}': {t}", .{ - f.package_root, Package.build_zig_basename, e, - }), - }); - return error.FetchFailed; - }, - } -} - -/// This function populates `f.manifest` or leaves it `null`. -fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { - const io = f.job_queue.io; - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const manifest_path = try pkg_root.join(arena, Manifest.basename); - - Manifest.load( - io, - arena, - manifest_path, - &f.manifest_ast, - eb, - &f.manifest, - f.allow_missing_paths_field, - ) catch |err| switch (err) { - error.FileNotFound => return, - error.Canceled => |e| return e, - error.ErrorsBundled => return error.FetchFailed, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }), - }); - return error.FetchFailed; - }, - }; - f.have_manifest = true; -} - -fn queueJobsForDeps(f: *Fetch) RunError!void { - const io = f.job_queue.io; - - assert(f.job_queue.recursive); - - // If the package does not have a build.zig.zon file then there are no dependencies. - if (!f.have_manifest) return; - const manifest = &f.manifest; - - const new_fetches, const prog_names = nf: { - const parent_arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const cache_root = f.job_queue.global_cache; - const dep_names = manifest.dependencies.keys(); - const deps = manifest.dependencies.values(); - // Grab the new tasks into a temporary buffer so we can unlock that mutex - // as fast as possible. - // This overallocates any fetches that get skipped by the `continue` in the - // loop below. - const new_fetches = try parent_arena.alloc(Fetch, deps.len); - const prog_names = try parent_arena.alloc([]const u8, deps.len); - var new_fetch_index: usize = 0; - - try f.job_queue.mutex.lock(io); - defer f.job_queue.mutex.unlock(io); - - try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); - try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); - - // There are four cases here: - // * Correct hash is provided by manifest. - // - Hash map already has the entry, no need to add it again. - // * Incorrect hash is provided by manifest. - // - Hash mismatch error emitted; `queueJobsForDeps` is not called. - // * Hash is not provided by manifest. - // - Hash missing error emitted; `queueJobsForDeps` is not called. - // * path-based location is used without a hash. - // - Hash is added to the table based on the path alone before - // calling run(); no need to add it again. - // - // If we add a dep as lazy and then later try to add the same dep as eager, - // eagerness takes precedence and the existing entry is updated and re-scheduled - // for fetching. - - for (dep_names, deps) |dep_name, dep| { - var promoted_existing_to_eager = false; - const new_fetch = &new_fetches[new_fetch_index]; - const location: Location = switch (dep.location) { - .url => |url| .{ - .remote = .{ - .url = url, - .hash = h: { - const h = dep.hash orelse break :h null; - const pkg_hash: Package.Hash = .fromSlice(h); - if (h.len == 0) break :h pkg_hash; - const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); - if (gop.found_existing) { - if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { - gop.value_ptr.*.lazy_status = .eager; - promoted_existing_to_eager = true; - } else { - continue; - } - } - gop.value_ptr.* = new_fetch; - break :h pkg_hash; - }, - }, - }, - .path => |rel_path| l: { - // This might produce an invalid path, which is checked for - // at the beginning of run(). - const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); - const pkg_hash = relativePathDigest(new_root, cache_root); - const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); - if (gop.found_existing) { - if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { - gop.value_ptr.*.lazy_status = .eager; - promoted_existing_to_eager = true; - } else { - continue; - } - } - gop.value_ptr.* = new_fetch; - break :l .{ .relative_path = new_root }; - }, - }; - prog_names[new_fetch_index] = dep_name; - new_fetch_index += 1; - if (!promoted_existing_to_eager) { - f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); - } - new_fetch.* = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = location, - .location_tok = dep.location_tok, - .hash_tok = dep.hash_tok, - .name_tok = dep.name_tok, - .lazy_status = switch (f.job_queue.mode) { - .needed => if (dep.lazy) .available else .eager, - .all => .eager, - }, - .parent_package_root = f.package_root, - .remote_package_root = f.remote_package_root, - .parent_manifest_ast = &f.manifest_ast, - .prog_node = f.prog_node, - .job_queue = f.job_queue, - .omit_missing_hash_error = false, - .allow_missing_paths_field = true, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - } - - f.prog_node.increaseEstimatedTotalItems(new_fetch_index); - - break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; - }; - - // Now it's time to dispatch tasks. - for (new_fetches, prog_names) |*new_fetch, prog_name| { - f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name }); - } -} - -pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { - return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); -} - -pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void { - const prog_node = f.prog_node.start(prog_name, 0); - defer prog_node.end(); - - run(f) catch |err| switch (err) { - error.OutOfMemory => f.oom_flag = true, - error.Canceled => |e| return e, - error.FetchFailed => { - // Nothing to do because the errors are already reported in `error_bundle`, - // and a reference is kept to the `Fetch` task inside `all_fetches`. - }, - }; -} - -fn srcLoc( - f: *Fetch, - tok: std.zig.Ast.TokenIndex, -) Allocator.Error!ErrorBundle.SourceLocationIndex { - const ast = f.parent_manifest_ast orelse return .none; - const eb = &f.error_bundle; - const start_loc = ast.tokenLocation(0, tok); - const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); - const msg_off = 0; - return eb.addSourceLocation(.{ - .src_path = src_path, - .span_start = ast.tokenStart(tok), - .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), - .span_main = ast.tokenStart(tok) + msg_off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }); -} - -fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { - const eb = &f.error_bundle; - try eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = try f.srcLoc(msg_tok), - }); - return error.FetchFailed; -} - -const Resource = union(enum) { - file: Io.File.Reader, - http_request: HttpRequest, - git: Git, - dir: Io.Dir, - - const Git = struct { - session: git.Session, - fetch_stream: git.Session.FetchStream, - want_oid: git.Oid, - }; - - const HttpRequest = struct { - request: std.http.Client.Request, - response: std.http.Client.Response, - transfer_buffer: []u8, - decompress: std.http.Decompress, - decompress_buffer: []u8, - }; - - fn deinit(resource: *Resource, io: Io) void { - switch (resource.*) { - .file => |*file_reader| file_reader.file.close(io), - .http_request => |*http_request| http_request.request.deinit(), - .git => |*git_resource| { - git_resource.fetch_stream.deinit(); - }, - .dir => |*dir| dir.close(io), - } - resource.* = undefined; - } - - fn reader(resource: *Resource) *Io.Reader { - return switch (resource.*) { - .file => |*file_reader| return &file_reader.interface, - .http_request => |*http_request| return http_request.response.readerDecompressing( - http_request.transfer_buffer, - &http_request.decompress, - http_request.decompress_buffer, - ), - .git => |*g| return &g.fetch_stream.reader, - .dir => unreachable, - }; - } -}; - -const FileType = enum { - tar, - @"tar.gz", - @"tar.xz", - @"tar.zst", - git_pack, - zip, - - fn fromPath(file_path: []const u8) ?FileType { - if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; - if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; - if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; - if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; - if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip; - return null; - } - - /// Parameter is a content-disposition header value. - fn fromContentDisposition(cd_header: []const u8) ?FileType { - const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse - return null; - - var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse - return null; - value_start += "filename".len; - if (cd_header[value_start] == '*') { - value_start += 1; - } - if (cd_header[value_start] != '=') return null; - value_start += 1; - - var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; - if (cd_header[value_end - 1] == '\"') { - value_end -= 1; - } - return fromPath(cd_header[value_start..value_end]); - } - - test fromContentDisposition { - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); - try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); - - try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; size=42") == null); - try std.testing.expect(fromContentDisposition("inline; size=42") == null); - try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); - try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); - } -}; - -const init_resource_buffer_size = git.Packet.max_data_length; - -fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void { - const io = f.job_queue.io; - const arena = f.arena.allocator(); - const eb = &f.error_bundle; - - if (ascii.eqlIgnoreCase(uri.scheme, "file")) { - const path = try uri.path.toRawMaybeAlloc(arena); - const file = f.parent_package_root.openFile(io, path, .{}) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{ - f.parent_package_root, path, err, - })); - }; - resource.* = .{ .file = file.reader(io, reader_buffer) }; - return; - } - - const http_client = f.job_queue.http_client; - - if (ascii.eqlIgnoreCase(uri.scheme, "http") or - ascii.eqlIgnoreCase(uri.scheme, "https")) - { - resource.* = .{ .http_request = .{ - .request = http_client.request(.GET, uri, .{}) catch |err| - return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})), - .response = undefined, - .transfer_buffer = reader_buffer, - .decompress_buffer = &.{}, - .decompress = undefined, - } }; - const request = &resource.http_request.request; - errdefer request.deinit(); - - request.sendBodiless() catch |err| - return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err})); - - var redirect_buffer: [8000]u8 = undefined; - const response = &resource.http_request.response; - response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) { - error.ReadFailed => { - return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{ - request.connection.?.getReadError().?, - })); - }, - else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})), - }; - - if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString( - "bad HTTP response code: '{d} {s}'", - .{ response.head.status, response.head.status.phrase() orelse "" }, - )); - - resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - return; - } - - if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or - ascii.eqlIgnoreCase(uri.scheme, "git+https")) - { - var transport_uri = uri; - transport_uri.scheme = uri.scheme["git+".len..]; - var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| { - return f.fail( - f.location_tok, - try eb.printString("unable to discover remote git server capabilities: {t}", .{err}), - ); - }; - - const want_oid = want_oid: { - const want_ref = - if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; - if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} - - const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); - const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); - - var ref_iterator: git.Session.RefIterator = undefined; - session.listRefs(&ref_iterator, .{ - .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, - .include_peeled = true, - .buffer = reader_buffer, - }) catch |err| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err})); - defer ref_iterator.deinit(); - while (ref_iterator.next() catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to iterate refs: {s}", - .{@errorName(err)}, - )); - }) |ref| { - if (std.mem.eql(u8, ref.name, want_ref) or - std.mem.eql(u8, ref.name, want_ref_head) or - std.mem.eql(u8, ref.name, want_ref_tag)) - { - break :want_oid ref.peeled orelse ref.oid; - } - } - return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); - }; - if (f.use_latest_commit) { - f.latest_commit = want_oid; - } else if (uri.fragment == null) { - const notes_len = 1; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("url field is missing an explicit ref"), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = notes_len, - }); - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("try .url = \"{f}#{f}\",", .{ - uri.fmt(.{ .scheme = true, .authority = true, .path = true }), - want_oid, - }), - })); - return error.FetchFailed; - } - - var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; - _ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable; - resource.* = .{ .git = .{ - .session = session, - .fetch_stream = undefined, - .want_oid = want_oid, - } }; - const fetch_stream = &resource.git.fetch_stream; - session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err})); - }; - errdefer fetch_stream.deinit(fetch_stream); - - return; - } - - return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme})); -} - -fn unpackResource( - f: *Fetch, - resource: *Resource, - uri_path: []const u8, - tmp_directory: Cache.Directory, -) RunError!UnpackResult { - const eb = &f.error_bundle; - const file_type = switch (resource.*) { - .file => FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), - - .http_request => |*http_request| ft: { - const head = &http_request.response.head; - - // Content-Type takes first precedence. - const content_type = head.content_type orelse - return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); - - // Extract the MIME type, ignoring charset and boundary directives - const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; - const mime_type = content_type[0..mime_type_end]; - - if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) - break :ft .tar; - - if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or - ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or - ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or - ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or - ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) - { - break :ft .@"tar.gz"; - } - - if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) - break :ft .@"tar.xz"; - - if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) - break :ft .@"tar.zst"; - - if (ascii.eqlIgnoreCase(mime_type, "application/zip") or - ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or - ascii.eqlIgnoreCase(mime_type, "application/java-archive")) - { - break :ft .zip; - } - - if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and - !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) - { - return f.fail(f.location_tok, try eb.printString( - "unrecognized 'Content-Type' header: '{s}'", - .{content_type}, - )); - } - - // Next, the filename from 'content-disposition: attachment' takes precedence. - if (head.content_disposition) |cd_header| { - break :ft FileType.fromContentDisposition(cd_header) orelse { - return f.fail(f.location_tok, try eb.printString( - "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", - .{cd_header}, - )); - }; - } - - // Finally, the path from the URI is used. - break :ft FileType.fromPath(uri_path) orelse { - return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})); - }; - }, - - .git => .git_pack, - - .dir => |dir| { - f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{ - uri_path, err, - })); - }; - return .{}; - }, - }; - - switch (file_type) { - .tar => { - return unpackTarball(f, tmp_directory.handle, resource.reader()); - }, - .@"tar.gz" => { - var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer); - return try unpackTarball(f, tmp_directory.handle, &decompress.reader); - }, - .@"tar.xz" => { - const gpa = f.arena.child_allocator; - var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err| - return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err})); - defer decompress.deinit(); - return try unpackTarball(f, tmp_directory.handle, &decompress.reader); - }, - .@"tar.zst" => { - const window_len = std.compress.zstd.default_window_len; - const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max); - var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{ - .verify_checksum = false, - .window_len = window_len, - }); - return try unpackTarball(f, tmp_directory.handle, &decompress.reader); - }, - .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { - error.FetchFailed, error.OutOfMemory => |e| return e, - else => |e| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})), - }, - .zip => return unzip(f, tmp_directory.handle, resource.reader()) catch |err| switch (err) { - error.ReadFailed => return f.fail(f.location_tok, try eb.printString( - "failed reading resource: {t}", - .{err}, - )), - else => |e| return e, - }, - } -} - -fn unpackTarball(f: *Fetch, out_dir: Io.Dir, reader: *Io.Reader) RunError!UnpackResult { - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const io = f.job_queue.io; - - var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; - - std.tar.pipeToFileSystem(io, out_dir, reader, .{ - .diagnostics = &diagnostics, - .strip_components = 0, - .mode_mode = .ignore, - .exclude_empty_directories = true, - }) catch |err| return f.fail( - f.location_tok, - try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}), - ); - - var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; - if (diagnostics.errors.items.len > 0) { - try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); - for (diagnostics.errors.items) |item| { - switch (item) { - .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), - .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), - .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), - .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 - } - } - } - return res; -} - -fn unzip( - f: *Fetch, - out_dir: Io.Dir, - reader: *Io.Reader, -) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult { - // We write the entire contents to a file first because zip files - // must be processed back to front and they could be too large to - // load into memory. - - const io = f.job_queue.io; - const cache_root = f.job_queue.global_cache; - const prefix = "tmp/"; - const suffix = ".zip"; - const eb = &f.error_bundle; - const random_len = @sizeOf(u64) * 2; - - var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined; - zip_path[0..prefix.len].* = prefix.*; - zip_path[prefix.len + random_len ..].* = suffix.*; - - var zip_file = while (true) { - const random_integer = r: { - var x: u64 = undefined; - io.random(@ptrCast(&x)); - break :r x; - }; - zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer); - - break cache_root.handle.createFile(io, &zip_path, .{ - .exclusive = true, - .read = true, - }) catch |err| switch (err) { - error.PathAlreadyExists => continue, - error.FileNotFound => { - cache_root.handle.createDir(io, prefix, .default_dir) catch |dir_err| switch (dir_err) { - error.Canceled => |e| return e, - // error.PathAlreadyExists is considered a failure here because - // it implies that the prefix is not a directory. - else => |e| return f.fail( - f.location_tok, - try eb.printString("failed to create temporary directory: {t}", .{e}), - ), - }; - continue; - }, - error.Canceled => |e| return e, - else => |e| return f.fail( - f.location_tok, - try eb.printString("failed to create temporary zip file: {t}", .{e}), - ), - }; - }; - defer zip_file.close(io); - var zip_file_buffer: [4096]u8 = undefined; - var zip_file_reader = b: { - var zip_file_writer = zip_file.writer(io, &zip_file_buffer); - - _ = reader.streamRemaining(&zip_file_writer.interface) catch |err| switch (err) { - error.ReadFailed => |e| return e, - error.WriteFailed => return f.fail( - f.location_tok, - try eb.printString("failed writing temporary zip file: {t}", .{err}), - ), - }; - zip_file_writer.interface.flush() catch |err| return f.fail( - f.location_tok, - try eb.printString("failed writing temporary zip file: {t}", .{err}), - ); - break :b zip_file_writer.moveToReader(); - }; - - var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; - // no need to deinit since we are using an arena allocator - - zip_file_reader.seekTo(0) catch |err| - return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err})); - std.zip.extract(out_dir, &zip_file_reader, .{ - .allow_backslashes = true, - .diagnostics = &diagnostics, - }) catch |err| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err})); - - cache_root.handle.deleteFile(io, &zip_path) catch |err| - return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err})); - - return .{ .root_dir = diagnostics.root_dir }; -} - -fn unpackGitPack(f: *Fetch, out_dir: Io.Dir, resource: *Resource.Git) anyerror!UnpackResult { - const io = f.job_queue.io; - const arena = f.arena.allocator(); - // TODO don't try to get a gpa from an arena. expose this dependency higher up - // because the backing of arena could be page allocator - const gpa = f.arena.child_allocator; - const object_format: git.Oid.Format = resource.want_oid; - - var res: UnpackResult = .{}; - // The .git directory is used to store the packfile and associated index, but - // we do not attempt to replicate the exact structure of a real .git - // directory, since that isn't relevant for fetching a package. - { - var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{}); - defer pack_dir.close(io); - var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true }); - defer pack_file.close(io); - var pack_file_buffer: [4096]u8 = undefined; - var pack_file_reader = b: { - var pack_file_writer = pack_file.writer(io, &pack_file_buffer); - const fetch_reader = &resource.fetch_stream.reader; - _ = try fetch_reader.streamRemaining(&pack_file_writer.interface); - try pack_file_writer.interface.flush(); - break :b pack_file_writer.moveToReader(); - }; - - var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true }); - defer index_file.close(io); - var index_file_buffer: [2000]u8 = undefined; - var index_file_writer = index_file.writer(io, &index_file_buffer); - { - const index_prog_node = f.prog_node.start("Index pack", 0); - defer index_prog_node.end(); - try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer); - } - - { - var index_file_reader = index_file.reader(io, &index_file_buffer); - const checkout_prog_node = f.prog_node.start("Checkout", 0); - defer checkout_prog_node.end(); - var repository: git.Repository = undefined; - try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader); - defer repository.deinit(); - var diagnostics: git.Diagnostics = .{ .allocator = arena }; - try repository.checkout(io, out_dir, resource.want_oid, &diagnostics); - - if (diagnostics.errors.items.len > 0) { - try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); - for (diagnostics.errors.items) |item| { - switch (item) { - .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), - .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), - } - } - } - } - } - - try out_dir.deleteTree(io, ".git"); - return res; -} - -fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void { - const gpa = f.arena.child_allocator; - const io = f.job_queue.io; - // Recursive directory copy. - var it = try dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - switch (entry.kind) { - .directory => {}, // omit empty directories - .file => { - dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch |err| switch (err) { - error.FileNotFound => { - if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); - try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}); - }, - else => |e| return e, - }; - }, - .sym_link => { - var buf: [fs.max_path_bytes]u8 = undefined; - const link_name = buf[0..try dir.readLink(io, entry.path, &buf)]; - // TODO: if this would create a symlink to outside - // the destination directory, fail with an error instead. - tmp_dir.symLink(io, link_name, entry.path, .{}) catch |err| switch (err) { - error.FileNotFound => { - if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); - try tmp_dir.symLink(io, link_name, entry.path, .{}); - }, - else => |e| return e, - }; - }, - else => return error.IllegalFileTypeInPackage, - } - } -} - -pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void { - var handled_missing_dir = false; - while (true) { - Io.Dir.rename( - tmp_path.root_dir.handle, - tmp_path.sub_path, - dest_path.root_dir.handle, - dest_path.sub_path, - io, - ) catch |err| switch (err) { - error.FileNotFound => { - if (handled_missing_dir) return err; - const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?; - dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) { - error.PathAlreadyExists => handled_missing_dir = true, - else => |e| return e, - }; - continue; - }, - error.DirNotEmpty, error.AccessDenied => { - // Package has been already downloaded and may already be in use on the system. - tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) { - error.Canceled => |e| return e, - // Garbage files leftover in zig-cache/tmp/ is, as they say - // on Star Trek, "operating within normal parameters". - else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), - }; - }, - else => |e| return e, - }; - break; - } -} - -const ComputedHash = struct { - digest: Package.Hash.Digest, - total_size: u64, -}; - -/// Assumes that files not included in the package have already been filtered -/// prior to calling this function. This ensures that files not protected by -/// the hash are not present on the file system. Empty directories are *not -/// hashed* and must not be present on the file system when calling this -/// function. -fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { - const io = f.job_queue.io; - // All the path name strings need to be in memory for sorting. - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const eb = &f.error_bundle; - const root_dir = pkg_path.root_dir.handle; - - // Collect all files, recursively, then sort. - var all_files = std.array_list.Managed(*HashedFile).init(gpa); - defer all_files.deinit(); - - var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa); - defer deleted_files.deinit(); - - // Track directories which had any files deleted from them so that empty directories - // can be deleted. - var sus_dirs: std.array_hash_map.String(void) = .empty; - defer sus_dirs.deinit(gpa); - - var walker = try root_dir.walk(gpa); - defer walker.deinit(); - - // Total number of bytes of file contents included in the package. - var total_size: u64 = 0; - - { - // The final hash will be a hash of each file hashed independently. This - // allows hashing in parallel. - var group: Io.Group = .init; - defer group.cancel(io); - - while (walker.next(io) catch |err| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to walk temporary directory '{f}': {t}", - .{ pkg_path, err }, - ) }); - return error.FetchFailed; - }) |entry| { - if (entry.kind == .directory) continue; - - const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); - if (!filter.includePath(entry_pkg_path)) { - // Delete instead of including in hash calculation. - const fs_path = try arena.dupe(u8, entry.path); - - // Also track the parent directory in case it becomes empty. - if (fs.path.dirname(fs_path)) |parent| - try sus_dirs.put(gpa, parent, {}); - - const deleted_file = try arena.create(DeletedFile); - deleted_file.* = .{ - .fs_path = fs_path, - .failure = undefined, // to be populated by the worker - }; - group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file }); - try deleted_files.append(deleted_file); - continue; - } - - const kind: HashedFile.Kind = switch (entry.kind) { - .directory => unreachable, - .file => .file, - .sym_link => .link, - else => return f.fail(f.location_tok, try eb.printString( - "package contains '{s}' which has illegal file type '{t}'", - .{ entry.path, entry.kind }, - )), - }; - - if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) - f.has_build_zig = true; - - const fs_path = try arena.dupe(u8, entry.path); - const hashed_file = try arena.create(HashedFile); - hashed_file.* = .{ - .fs_path = fs_path, - .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), - .kind = kind, - .hash = undefined, // to be populated by the worker - .failure = undefined, // to be populated by the worker - .size = undefined, // to be populated by the worker - }; - group.async(io, workerHashFile, .{ io, root_dir, hashed_file }); - try all_files.append(hashed_file); - } - - try group.await(io); - } - - { - // Sort by length, descending, so that child directories get removed first. - sus_dirs.sortUnstable(@as(struct { - keys: []const []const u8, - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return ctx.keys[b_index].len < ctx.keys[a_index].len; - } - }, .{ .keys = sus_dirs.keys() })); - - // During this loop, more entries will be added, so we must loop by index. - var i: usize = 0; - while (i < sus_dirs.count()) : (i += 1) { - const sus_dir = sus_dirs.keys()[i]; - root_dir.deleteDir(io, sus_dir) catch |err| switch (err) { - error.DirNotEmpty => continue, - error.FileNotFound => continue, - else => |e| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to delete empty directory '{s}': {s}", - .{ sus_dir, @errorName(e) }, - ) }); - return error.FetchFailed; - }, - }; - if (fs.path.dirname(sus_dir)) |parent| { - try sus_dirs.put(gpa, parent, {}); - } - } - } - - std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); - - var hasher = Package.Hash.Algo.init(.{}); - var any_failures = false; - for (all_files.items) |hashed_file| { - hashed_file.failure catch |err| { - any_failures = true; - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to hash '{s}': {s}", .{ - hashed_file.fs_path, @errorName(err), - }), - }); - }; - hasher.update(&hashed_file.hash); - total_size += hashed_file.size; - } - for (deleted_files.items) |deleted_file| { - deleted_file.failure catch |err| { - any_failures = true; - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ - deleted_file.fs_path, @errorName(err), - }), - }); - }; - } - - if (any_failures) return error.FetchFailed; - - if (f.job_queue.debug_hash) { - assert(!f.job_queue.recursive); - // Print something to stdout that can be text diffed to figure out why - // the package hash is different. - dumpHashInfo(io, all_files.items) catch |err| - std.process.fatal("unable to write to stdout: {t}", .{err}); - } - - return .{ - .digest = hasher.finalResult(), - .total_size = total_size, - }; -} - -fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void { - var stdout_buffer: [1024]u8 = undefined; - var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer); - dumpHashInfoWriter(&stdout_writer.interface, all_files) catch |err| switch (err) { - error.WriteFailed => return stdout_writer.err.?, - }; - try stdout_writer.flush(); -} - -fn dumpHashInfoWriter(w: *Io.Writer, all_files: []const *const HashedFile) Io.Writer.Error!void { - for (all_files) |hashed_file| { - try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path }); - } -} - -fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void { - hashed_file.failure = hashFileFallible(io, dir, hashed_file); -} - -fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void { - deleted_file.failure = deleteFileFallible(io, dir, deleted_file); -} - -fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void { - var buf: [8000]u8 = undefined; - var hasher = Package.Hash.Algo.init(.{}); - hasher.update(hashed_file.normalized_path); - var file_size: u64 = 0; - - switch (hashed_file.kind) { - .file => { - var file = try dir.openFile(io, hashed_file.fs_path, .{}); - defer file.close(io); - // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 - hasher.update(&.{ 0, 0 }); - var file_header: FileHeader = .{}; - while (true) { - const bytes_read = try file.readPositional(io, &.{&buf}, file_size); - if (bytes_read == 0) break; - file_size += bytes_read; - hasher.update(buf[0..bytes_read]); - file_header.update(buf[0..bytes_read]); - } - if (file_header.isExecutable()) { - try setExecutable(io, file); - } - }, - .link => { - const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)]; - if (fs.path.sep != canonical_sep) { - // Package hashes are intended to be consistent across - // platforms which means we must normalize path separators - // inside symlinks. - normalizePath(link_name); - } - hasher.update(link_name); - }, - } - hasher.final(&hashed_file.hash); - hashed_file.size = file_size; -} - -fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { - try dir.deleteFile(io, deleted_file.fs_path); -} - -fn setExecutable(io: Io, file: Io.File) !void { - if (!Io.File.Permissions.has_executable_bit) return; - try file.setPermissions(io, .executable_file); -} - -const DeletedFile = struct { - fs_path: []const u8, - failure: Error!void, - - const Error = - Io.Dir.DeleteFileError || - Io.Dir.DeleteDirError; -}; - -const HashedFile = struct { - fs_path: []const u8, - normalized_path: []const u8, - hash: Package.Hash.Digest, - failure: Error!void, - kind: Kind, - size: u64, - - const Error = - Io.File.OpenError || - Io.File.ReadPositionalError || - Io.File.StatError || - Io.File.SetPermissionsError || - Io.Dir.ReadLinkError; - - const Kind = enum { file, link }; - - fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { - _ = context; - return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); - } -}; - -/// Strips root directory name from file system path. -fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { - if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; - - if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { - return fs_path[root_dir.len + 1 ..]; - } - - return fs_path; -} - -/// Make a file system path identical independently of operating system path inconsistencies. -/// This converts backslashes into forward slashes. -fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { - const normalized = try arena.dupe(u8, pkg_path); - if (fs.path.sep == canonical_sep) return normalized; - normalizePath(normalized); - return normalized; -} - -const canonical_sep = fs.path.sep_posix; - -fn normalizePath(bytes: []u8) void { - assert(fs.path.sep != canonical_sep); - std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); -} - -const Filter = struct { - include_paths: std.array_hash_map.String(void) = .empty, - - /// sub_path is relative to the package root. - pub fn includePath(self: *const Filter, sub_path: []const u8) bool { - if (self.include_paths.count() == 0) return true; - if (self.include_paths.contains("")) return true; - if (self.include_paths.contains(".")) return true; - if (self.include_paths.contains(sub_path)) return true; - - // Check if any included paths are parent directories of sub_path. - var dirname = sub_path; - while (std.fs.path.dirname(dirname)) |next_dirname| { - if (self.include_paths.contains(next_dirname)) return true; - dirname = next_dirname; - } - - return false; - } - - test includePath { - const gpa = std.testing.allocator; - var filter: Filter = .{}; - defer filter.include_paths.deinit(gpa); - - try filter.include_paths.put(gpa, "src", {}); - try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); - try std.testing.expect(!filter.includePath(".gitignore")); - } -}; - -pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { - if (dep.hash) |h| return .fromSlice(h); - - switch (dep.location) { - .url => return null, - .path => |rel_path| { - var buf: [fs.max_path_bytes]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch - return null; - return relativePathDigest(new_root, cache_root); - }, - } -} - -// Detects executable header: ELF or Macho-O magic header or shebang line. -const FileHeader = struct { - header: [4]u8 = undefined, - bytes_read: usize = 0, - - pub fn update(self: *FileHeader, buf: []const u8) void { - if (self.bytes_read >= self.header.len) return; - const n = @min(self.header.len - self.bytes_read, buf.len); - @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); - self.bytes_read += n; - } - - fn isScript(self: *FileHeader) bool { - const shebang = "#!"; - return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); - } - - fn isElf(self: *FileHeader) bool { - const elf_magic = std.elf.MAGIC; - return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); - } - - fn isMachO(self: *FileHeader) bool { - if (self.bytes_read < 4) return false; - const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); - return magic_number == std.macho.MH_MAGIC or - magic_number == std.macho.MH_MAGIC_64 or - magic_number == std.macho.FAT_MAGIC or - magic_number == std.macho.FAT_MAGIC_64 or - magic_number == std.macho.MH_CIGAM or - magic_number == std.macho.MH_CIGAM_64 or - magic_number == std.macho.FAT_CIGAM or - magic_number == std.macho.FAT_CIGAM_64; - } - - pub fn isExecutable(self: *FileHeader) bool { - return self.isScript() or self.isElf() or self.isMachO(); - } -}; - -test FileHeader { - var h: FileHeader = .{}; - try std.testing.expect(!h.isExecutable()); - - const elf_magic = std.elf.MAGIC; - h.update(elf_magic[0..2]); - try std.testing.expect(!h.isExecutable()); - h.update(elf_magic[2..4]); - try std.testing.expect(h.isExecutable()); - - h.update(elf_magic[2..4]); - try std.testing.expect(h.isExecutable()); - - const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; - h.bytes_read = 0; - h.update(&macho64_magic_bytes); - try std.testing.expect(h.isExecutable()); - - const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; - h.bytes_read = 0; - h.update(&macho64_cigam_bytes); - try std.testing.expect(h.isExecutable()); -} - -// Result of the `unpackResource` operation. Enables collecting errors from -// tar/git diagnostic, filtering that errors by manifest inclusion rules and -// emitting remaining errors to an `ErrorBundle`. -const UnpackResult = struct { - errors: []Error = undefined, - errors_count: usize = 0, - root_error_message: []const u8 = "", - - // A non empty value means that the package contents are inside a - // sub-directory indicated by the named path. - root_dir: []const u8 = "", - - const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - unable_to_create_file: struct { - code: anyerror, - file_name: []const u8, - }, - unsupported_file_type: struct { - file_name: []const u8, - file_type: u8, - }, - - fn excluded(self: Error, filter: Filter) bool { - const file_name = switch (self) { - .unable_to_create_file => |info| info.file_name, - .unable_to_create_sym_link => |info| info.file_name, - .unsupported_file_type => |info| info.file_name, - }; - return !filter.includePath(file_name); - } - }; - - fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { - self.root_error_message = try arena.dupe(u8, root_error_message); - self.errors = try arena.alloc(UnpackResult.Error, n); - } - - fn hasErrors(self: *UnpackResult) bool { - return self.errors_count > 0; - } - - fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { - self.errors[self.errors_count] = .{ .unable_to_create_file = .{ - .code = err, - .file_name = file_name, - } }; - self.errors_count += 1; - } - - fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { - self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ - .code = err, - .file_name = file_name, - .link_name = link_name, - } }; - self.errors_count += 1; - } - - fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { - self.errors[self.errors_count] = .{ .unsupported_file_type = .{ - .file_name = file_name, - .file_type = file_type, - } }; - self.errors_count += 1; - } - - fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { - if (self.errors_count == 0) return; - - var unfiltered_errors: u32 = 0; - for (self.errors) |item| { - if (item.excluded(filter)) continue; - unfiltered_errors += 1; - } - if (unfiltered_errors == 0) return; - - // Emmit errors to an `ErrorBundle`. - const eb = &f.error_bundle; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString(self.root_error_message), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = unfiltered_errors, - }); - var note_i: u32 = try eb.reserveNotes(unfiltered_errors); - for (self.errors) |item| { - if (item.excluded(filter)) continue; - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - .unable_to_create_file => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create file '{s}': {s}", .{ - info.file_name, @errorName(info.code), - }), - })); - }, - .unsupported_file_type => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ - info.file_name, info.file_type, - }), - })); - }, - } - note_i += 1; - } - - return error.FetchFailed; - } - - test validate { - const gpa = std.testing.allocator; - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - // fill UnpackResult with errors - var res: UnpackResult = .{}; - try res.allocErrors(arena, 4, "unable to unpack"); - try std.testing.expectEqual(0, res.errors_count); - res.unableToCreateFile("dir1/file1", error.File1); - res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); - res.unableToCreateFile("dir1/file3", error.File3); - res.unsupportedFileType("dir2/file4", 'x'); - try std.testing.expectEqual(4, res.errors_count); - - // create filter, includes dir2, excludes dir1 - var filter: Filter = .{}; - try filter.include_paths.put(arena, "dir2", {}); - - // init Fetch - var fetch: Fetch = undefined; - fetch.parent_manifest_ast = null; - fetch.location_tok = 0; - try fetch.error_bundle.init(gpa); - defer fetch.error_bundle.deinit(); - - // validate errors with filter - try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); - - // output errors to string - var errors = try fetch.error_bundle.toOwnedBundle(""); - defer errors.deinit(gpa); - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - try errors.renderToWriter(.{}, &aw.writer); - try std.testing.expectEqualStrings( - \\error: unable to unpack - \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError - \\ note: file 'dir2/file4' has unsupported type 'x' - \\ - , aw.written()); - } -}; - -test { - _ = Filter; - _ = FileType; - _ = UnpackResult; -} diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig deleted file mode 100644 index d3bd1d701a618281355dba5e585d286cb3f9107f..0000000000000000000000000000000000000000 --- a/src/Package/Fetch/git.zig +++ /dev/null @@ -1,1750 +0,0 @@ -//! Git support for package fetching. -//! -//! This is not intended to support all features of Git: it is limited to the -//! basic functionality needed to clone a repository for the purpose of fetching -//! a package. - -const std = @import("std"); -const Io = std.Io; -const mem = std.mem; -const testing = std.testing; -const Allocator = mem.Allocator; -const Sha1 = std.crypto.hash.Sha1; -const Sha256 = std.crypto.hash.sha2.Sha256; -const assert = std.debug.assert; - -/// The ID of a Git object. -pub const Oid = union(Format) { - sha1: [Sha1.digest_length]u8, - sha256: [Sha256.digest_length]u8, - - pub const max_formatted_length = len: { - var max: usize = 0; - for (std.enums.values(Format)) |f| { - max = @max(max, f.formattedLength()); - } - break :len max; - }; - - pub const Format = enum { - sha1, - sha256, - - pub fn byteLength(f: Format) usize { - return switch (f) { - .sha1 => Sha1.digest_length, - .sha256 => Sha256.digest_length, - }; - } - - pub fn formattedLength(f: Format) usize { - return 2 * f.byteLength(); - } - }; - - const Hasher = union(Format) { - sha1: Sha1, - sha256: Sha256, - - fn init(oid_format: Format) Hasher { - return switch (oid_format) { - .sha1 => .{ .sha1 = Sha1.init(.{}) }, - .sha256 => .{ .sha256 = Sha256.init(.{}) }, - }; - } - - // Must be public for use from HashedReader and HashedWriter. - pub fn update(hasher: *Hasher, b: []const u8) void { - switch (hasher.*) { - inline else => |*inner| inner.update(b), - } - } - - fn finalResult(hasher: *Hasher) Oid { - return switch (hasher.*) { - inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), - }; - } - }; - - const Hashing = union(Format) { - sha1: Io.Writer.Hashing(Sha1), - sha256: Io.Writer.Hashing(Sha256), - - fn init(oid_format: Format, buffer: []u8) Hashing { - return switch (oid_format) { - .sha1 => .{ .sha1 = .init(buffer) }, - .sha256 => .{ .sha256 = .init(buffer) }, - }; - } - - fn writer(h: *@This()) *Io.Writer { - return switch (h.*) { - inline else => |*inner| &inner.writer, - }; - } - - fn final(h: *@This()) Oid { - switch (h.*) { - inline else => |*inner, tag| { - inner.writer.flush() catch unreachable; // hashers cannot fail - return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult()); - }, - } - } - }; - - pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { - assert(bytes.len == oid_format.byteLength()); - return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), - }; - } - - pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid { - return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*), - }; - } - - pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { - switch (oid_format) { - inline else => |tag| { - if (s.len != tag.formattedLength()) return error.InvalidOid; - var bytes: [tag.byteLength()]u8 = undefined; - for (&bytes, 0..) |*b, i| { - b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; - } - return @unionInit(Oid, @tagName(tag), bytes); - }, - } - } - - test parse { - try testing.expectEqualSlices( - u8, - &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, - &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, - ); - try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); - try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); - try testing.expectEqualSlices( - u8, - &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, - &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, - ); - try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); - try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); - try testing.expectError(error.InvalidOid, parse(.sha1, "master")); - try testing.expectError(error.InvalidOid, parse(.sha256, "master")); - try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); - try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); - } - - pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { - return for (std.enums.values(Format)) |f| { - if (s.len == f.formattedLength()) break parse(f, s); - } else error.InvalidOid; - } - - pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void { - try writer.print("{x}", .{oid.slice()}); - } - - pub fn slice(oid: *const Oid) []const u8 { - return switch (oid.*) { - inline else => |*bytes| bytes, - }; - } -}; - -pub const Diagnostics = struct { - allocator: Allocator, - errors: std.ArrayList(Error) = .empty, - - pub const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - unable_to_create_file: struct { - code: anyerror, - file_name: []const u8, - }, - }; - - pub fn deinit(d: *Diagnostics) void { - for (d.errors.items) |item| { - switch (item) { - .unable_to_create_sym_link => |info| { - d.allocator.free(info.file_name); - d.allocator.free(info.link_name); - }, - .unable_to_create_file => |info| { - d.allocator.free(info.file_name); - }, - } - } - d.errors.deinit(d.allocator); - d.* = undefined; - } -}; - -pub const Repository = struct { - odb: Odb, - - pub fn init( - repo: *Repository, - allocator: Allocator, - format: Oid.Format, - pack_file: *Io.File.Reader, - index_file: *Io.File.Reader, - ) !void { - repo.* = .{ .odb = undefined }; - try repo.odb.init(allocator, format, pack_file, index_file); - } - - pub fn deinit(repository: *Repository) void { - repository.odb.deinit(); - repository.* = undefined; - } - - /// Checks out the repository at `commit_oid` to `worktree`. - pub fn checkout( - repository: *Repository, - io: Io, - worktree: Io.Dir, - commit_oid: Oid, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(commit_oid); - const tree_oid = tree_oid: { - const commit_object = try repository.odb.readObject(); - if (commit_object.type != .commit) return error.NotACommit; - break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); - }; - try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics); - } - - /// Checks out the tree at `tree_oid` to `worktree`. - fn checkoutTree( - repository: *Repository, - io: Io, - dir: Io.Dir, - tree_oid: Oid, - current_path: []const u8, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(tree_oid); - const tree_object = try repository.odb.readObject(); - if (tree_object.type != .tree) return error.NotATree; - // The tree object may be evicted from the object cache while we're - // iterating over it, so we can make a defensive copy here to make sure - // it remains valid until we're done with it - const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); - defer repository.odb.allocator.free(tree_data); - - var tree_iter: TreeIterator = .{ - .format = repository.odb.format, - .data = tree_data, - .pos = 0, - }; - while (try tree_iter.next()) |entry| { - switch (entry.type) { - .directory => { - try dir.createDir(io, entry.name, .default_dir); - var subdir = try dir.openDir(io, entry.name, .{}); - defer subdir.close(io); - const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); - defer repository.odb.allocator.free(sub_path); - try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics); - }, - .file => { - try repository.odb.seekOid(entry.oid); - const file_object = try repository.odb.readObject(); - if (file_object.type != .blob) return error.InvalidFile; - var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ - .code = e, - .file_name = file_name, - } }); - continue; - }; - defer file.close(io); - try file.writePositionalAll(io, file_object.data, 0); - }, - .symlink => { - try repository.odb.seekOid(entry.oid); - const symlink_object = try repository.odb.readObject(); - if (symlink_object.type != .blob) return error.InvalidFile; - const link_name = symlink_object.data; - dir.symLink(io, link_name, entry.name, .{}) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); - errdefer diagnostics.allocator.free(link_name_dup); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ - .code = e, - .file_name = file_name, - .link_name = link_name_dup, - } }); - }; - }, - .gitlink => { - // Consistent with git archive behavior, create the directory but - // do nothing else - try dir.createDir(io, entry.name, .default_dir); - }, - } - } - } - - /// Returns the ID of the tree associated with the given commit (provided as - /// raw object data). - fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { - if (!mem.startsWith(u8, commit_data, "tree ") or - commit_data.len < "tree ".len + format.formattedLength() + "\n".len or - commit_data["tree ".len + format.formattedLength()] != '\n') - { - return error.InvalidCommit; - } - return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); - } - - const TreeIterator = struct { - format: Oid.Format, - data: []const u8, - pos: usize, - - const Entry = struct { - type: Type, - executable: bool, - name: [:0]const u8, - oid: Oid, - - const Type = enum(u4) { - directory = 0o4, - file = 0o10, - symlink = 0o12, - gitlink = 0o16, - }; - }; - - fn next(iterator: *TreeIterator) !?Entry { - if (iterator.pos == iterator.data.len) return null; - - const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; - const mode: packed struct { - permission: u9, - unused: u3, - type: u4, - } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); - const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree; - const executable = switch (mode.permission) { - 0 => if (@"type" == .file) return error.InvalidTree else false, - 0o644 => if (@"type" != .file) return error.InvalidTree else false, - 0o755 => if (@"type" != .file) return error.InvalidTree else true, - else => return error.InvalidTree, - }; - iterator.pos = mode_end + 1; - - const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; - const name = iterator.data[iterator.pos..name_end :0]; - iterator.pos = name_end + 1; - - const oid_length = iterator.format.byteLength(); - if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; - const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); - iterator.pos += oid_length; - - return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; - } - }; -}; - -/// A Git object database backed by a packfile. A packfile index is also used -/// for efficient access to objects in the packfile. -/// -/// The format of the packfile and its associated index are documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -const Odb = struct { - format: Oid.Format, - pack_file: *Io.File.Reader, - index_header: IndexHeader, - index_file: *Io.File.Reader, - cache: ObjectCache = .{}, - allocator: Allocator, - - /// Initializes the database from open pack and index files. - fn init( - odb: *Odb, - allocator: Allocator, - format: Oid.Format, - pack_file: *Io.File.Reader, - index_file: *Io.File.Reader, - ) !void { - try pack_file.seekTo(0); - try index_file.seekTo(0); - odb.* = .{ - .format = format, - .pack_file = pack_file, - .index_header = undefined, - .index_file = index_file, - .allocator = allocator, - }; - try odb.index_header.read(&index_file.interface); - } - - fn deinit(odb: *Odb) void { - odb.cache.deinit(odb.allocator); - odb.* = undefined; - } - - /// Reads the object at the current position in the database. - fn readObject(odb: *Odb) !Object { - var base_offset = odb.pack_file.logicalPos(); - var base_header: EntryHeader = undefined; - var delta_offsets: std.ArrayList(u64) = .empty; - defer delta_offsets.deinit(odb.allocator); - const base_object = while (true) { - if (odb.cache.get(base_offset)) |base_object| break base_object; - - base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(odb.allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; - try odb.pack_file.seekTo(base_offset); - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(odb.allocator, base_offset); - try odb.seekOid(ref_delta.base_object); - base_offset = odb.pack_file.logicalPos(); - }, - else => { - const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength()); - errdefer odb.allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try odb.cache.put(odb.allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain( - odb.allocator, - odb.format, - odb.pack_file, - base_object, - delta_offsets.items, - &odb.cache, - ); - - return .{ .type = base_object.type, .data = base_data }; - } - - /// Seeks to the beginning of the object with the given ID. - fn seekOid(odb: *Odb, oid: Oid) !void { - const oid_length = odb.format.byteLength(); - const key = oid.slice()[0]; - var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; - var end_index = odb.index_header.fan_out_table[key]; - const found_index = while (start_index < end_index) { - const mid_index = start_index + (end_index - start_index) / 2; - try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface); - switch (mem.order(u8, mid_oid.slice(), oid.slice())) { - .lt => start_index = mid_index + 1, - .gt => end_index = mid_index, - .eq => break mid_index, - } - } else return error.ObjectNotFound; - - const n_objects = odb.index_header.fan_out_table[255]; - const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); - try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big)); - const pack_offset = pack_offset: { - if (l1_offset.big) { - const l2_offset_values_start = offset_values_start + n_objects * 4; - try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.interface.takeInt(u64, .big); - } else { - break :pack_offset l1_offset.value; - } - }; - - try odb.pack_file.seekTo(pack_offset); - } -}; - -const Object = struct { - type: Type, - data: []const u8, - - const Type = enum { - commit, - tree, - blob, - tag, - }; -}; - -/// A cache for object data. -/// -/// The purpose of this cache is to speed up resolution of deltas by caching the -/// results of resolving delta objects, while maintaining a maximum cache size -/// to avoid excessive memory usage. If the total size of the objects in the -/// cache exceeds the maximum, the cache will begin evicting the least recently -/// used objects: when resolving delta chains, the most recently used objects -/// will likely be more helpful as they will be further along in the chain -/// (skipping earlier reconstruction steps). -/// -/// Object data stored in the cache is managed by the cache. It should not be -/// freed by the caller at any point after inserting it into the cache. Any -/// objects remaining in the cache will be freed when the cache itself is freed. -const ObjectCache = struct { - objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, - lru_nodes: std.DoublyLinkedList = .{}, - lru_nodes_len: usize = 0, - byte_size: usize = 0, - - const max_byte_size = 128 * 1024 * 1024; // 128MiB - /// A list of offsets stored in the cache, with the most recently used - /// entries at the end. - const LruListNode = struct { - data: u64, - node: std.DoublyLinkedList.Node, - }; - const CacheEntry = struct { object: Object, lru_node: *LruListNode }; - - fn deinit(cache: *ObjectCache, allocator: Allocator) void { - var object_iterator = cache.objects.iterator(); - while (object_iterator.next()) |object| { - allocator.free(object.value_ptr.object.data); - allocator.destroy(object.value_ptr.lru_node); - } - cache.objects.deinit(allocator); - cache.* = undefined; - } - - /// Gets an object from the cache, moving it to the most recently used - /// position if it is present. - fn get(cache: *ObjectCache, offset: u64) ?Object { - if (cache.objects.get(offset)) |entry| { - cache.lru_nodes.remove(&entry.lru_node.node); - cache.lru_nodes.append(&entry.lru_node.node); - return entry.object; - } else { - return null; - } - } - - /// Puts an object in the cache, possibly evicting older entries if the - /// cache exceeds its maximum size. Note that, although old objects may - /// be evicted, the object just added to the cache with this function - /// will not be evicted before the next call to `put` or `deinit` even if - /// it exceeds the maximum cache size. - fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { - const lru_node = try allocator.create(LruListNode); - errdefer allocator.destroy(lru_node); - lru_node.data = offset; - - const gop = try cache.objects.getOrPut(allocator, offset); - if (gop.found_existing) { - cache.byte_size -= gop.value_ptr.object.data.len; - cache.lru_nodes.remove(&gop.value_ptr.lru_node.node); - cache.lru_nodes_len -= 1; - allocator.destroy(gop.value_ptr.lru_node); - allocator.free(gop.value_ptr.object.data); - } - gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; - cache.byte_size += object.data.len; - cache.lru_nodes.append(&lru_node.node); - cache.lru_nodes_len += 1; - - while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) { - // The > 1 check is to make sure that we don't evict the most - // recently added node, even if it by itself happens to exceed the - // maximum size of the cache. - const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?)); - cache.lru_nodes_len -= 1; - const evict_offset = evict_node.data; - allocator.destroy(evict_node); - const evict_object = cache.objects.get(evict_offset).?.object; - cache.byte_size -= evict_object.data.len; - allocator.free(evict_object.data); - _ = cache.objects.remove(evict_offset); - } - } -}; - -/// A single pkt-line in the Git protocol. -/// -/// The format of a pkt-line is documented in -/// [protocol-common](https://git-scm.com/docs/protocol-common). The special -/// meanings of the delimiter and response-end packets are documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -pub const Packet = union(enum) { - flush, - delimiter, - response_end, - data: []const u8, - - pub const max_data_length = 65516; - - /// Reads a packet in pkt-line format. - fn read(reader: *Io.Reader) !Packet { - const packet: Packet = try .peek(reader); - switch (packet) { - .data => |data| reader.toss(data.len), - else => {}, - } - return packet; - } - - /// Consumes the header of a pkt-line packet and reads any associated data - /// into the reader's buffer, but does not consume the data. - fn peek(reader: *Io.Reader) !Packet { - const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket; - switch (length) { - 0 => return .flush, - 1 => return .delimiter, - 2 => return .response_end, - 3 => return error.InvalidPacket, - else => if (length - 4 > max_data_length) return error.InvalidPacket, - } - return .{ .data = try reader.peek(length - 4) }; - } - - /// Writes a packet in pkt-line format. - fn write(packet: Packet, writer: *Io.Writer) !void { - switch (packet) { - .flush => try writer.writeAll("0000"), - .delimiter => try writer.writeAll("0001"), - .response_end => try writer.writeAll("0002"), - .data => |data| { - assert(data.len <= max_data_length); - try writer.print("{x:0>4}", .{data.len + 4}); - try writer.writeAll(data); - }, - } - } - - /// Returns the normalized form of textual packet data, stripping any - /// trailing '\n'. - /// - /// As documented in - /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), - /// non-binary (textual) pkt-line data should contain a trailing '\n', but - /// is not required to do so (implementations must support both forms). - fn normalizeText(data: []const u8) []const u8 { - return if (mem.endsWith(u8, data, "\n")) - data[0 .. data.len - 1] - else - data; - } -}; - -/// A client session for the Git protocol, currently limited to an HTTP(S) -/// transport. Only protocol version 2 is supported, as documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -pub const Session = struct { - transport: *std.http.Client, - location: Location, - supports_agent: bool, - supports_shallow: bool, - object_format: Oid.Format, - arena: Allocator, - - const agent = "zig/" ++ @import("builtin").zig_version_string; - const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); - - /// Initializes a client session and discovers the capabilities of the - /// server for optimal transport. - pub fn init( - arena: Allocator, - transport: *std.http.Client, - uri: std.Uri, - /// Asserted to be at least `Packet.max_data_length` - response_buffer: []u8, - ) !Session { - assert(response_buffer.len >= Packet.max_data_length); - var session: Session = .{ - .transport = transport, - .location = try .init(arena, uri), - .supports_agent = false, - .supports_shallow = false, - .object_format = .sha1, - .arena = arena, - }; - var capability_iterator: CapabilityIterator = undefined; - try session.getCapabilities(&capability_iterator, response_buffer); - defer capability_iterator.deinit(); - while (try capability_iterator.next()) |capability| { - if (mem.eql(u8, capability.key, "agent")) { - session.supports_agent = true; - } else if (mem.eql(u8, capability.key, "fetch")) { - var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); - while (feature_iterator.next()) |feature| { - if (mem.eql(u8, feature, "shallow")) { - session.supports_shallow = true; - } - } - } else if (mem.eql(u8, capability.key, "object-format")) { - if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { - session.object_format = format; - } - } - } - return session; - } - - /// An owned `std.Uri` representing the location of the server (base URI). - const Location = struct { - uri: std.Uri, - - fn init(arena: Allocator, uri: std.Uri) !Location { - const scheme = try arena.dupe(u8, uri.scheme); - const user = if (uri.user) |user| try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(user, .formatUser), - }) else null; - const password = if (uri.password) |password| try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(password, .formatPassword), - }) else null; - const host = if (uri.host) |host| try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(host, .formatHost), - }) else null; - const path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(uri.path, .formatPath), - }); - // The query and fragment are not used as part of the base server URI. - return .{ - .uri = .{ - .scheme = scheme, - .user = if (user) |s| .{ .percent_encoded = s } else null, - .password = if (password) |s| .{ .percent_encoded = s } else null, - .host = if (host) |s| .{ .percent_encoded = s } else null, - .port = uri.port, - .path = .{ .percent_encoded = path }, - }, - }; - } - }; - - /// Returns an iterator over capabilities supported by the server. - /// - /// The `session.location` is updated if the server returns a redirect, so - /// that subsequent session functions do not need to handle redirects. - fn getCapabilities(session: *Session, it: *CapabilityIterator, response_buffer: []u8) !void { - const arena = session.arena; - assert(response_buffer.len >= Packet.max_data_length); - var info_refs_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(session.location.uri.path, .formatPath), - }); - info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ - "/", session_uri_path, "info/refs", - }) }; - } - info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; - info_refs_uri.fragment = null; - - const max_redirects = 3; - it.* = .{ - .request = try session.transport.request(.GET, info_refs_uri, .{ - .redirect_behavior = .init(max_redirects), - .extra_headers = &.{ - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }), - .reader = undefined, - .decompress = undefined, - }; - errdefer it.deinit(); - const request = &it.request; - try request.sendBodiless(); - - var redirect_buffer: [1024]u8 = undefined; - var response = try request.receiveHead(&redirect_buffer); - if (response.head.status != .ok) return error.ProtocolError; - const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; - if (any_redirects_occurred) { - const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(request.uri.path, .formatPath), - }); - if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; - var new_uri = request.uri; - new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; - session.location = try .init(arena, new_uri); - } - - const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer); - var state: enum { response_start, response_content } = .response_start; - while (true) { - // Some Git servers (at least GitHub) include an additional - // '# service=git-upload-pack' informative response before sending - // the expected 'version 2' packet and capability information. - // This is not universal: SourceHut, for example, does not do this. - // Thus, we need to skip any such useless additional responses - // before we get the one we're actually looking for. The responses - // will be delimited by flush packets. - const packet = Packet.read(it.reader) catch |err| switch (err) { - error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found - else => |e| return e, - }; - switch (packet) { - .flush => state = .response_start, - .data => |data| switch (state) { - .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { - return; - } else { - state = .response_content; - }, - else => {}, - }, - else => return error.UnexpectedPacket, - } - } - } - - const CapabilityIterator = struct { - request: std.http.Client.Request, - reader: *Io.Reader, - decompress: std.http.Decompress, - - const Capability = struct { - key: []const u8, - value: ?[]const u8 = null, - - fn parse(data: []const u8) Capability { - return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| - .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } - else - .{ .key = data }; - } - }; - - fn deinit(it: *CapabilityIterator) void { - it.request.deinit(); - it.* = undefined; - } - - fn next(it: *CapabilityIterator) !?Capability { - switch (try Packet.read(it.reader)) { - .flush => return null, - .data => |data| return Capability.parse(Packet.normalizeText(data)), - else => return error.UnexpectedPacket, - } - } - }; - - const ListRefsOptions = struct { - /// The ref prefixes (if any) to use to filter the refs available on the - /// server. Note that the client must still check the returned refs - /// against its desired filters itself: the server is not required to - /// respect these prefix filters and may return other refs as well. - ref_prefixes: []const []const u8 = &.{}, - /// Whether to include symref targets for returned symbolic refs. - include_symrefs: bool = false, - /// Whether to include the peeled object ID for returned tag refs. - include_peeled: bool = false, - /// Asserted to be at least `Packet.max_data_length`. - buffer: []u8, - }; - - /// Returns an iterator over refs known to the server. - pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void { - const arena = session.arena; - assert(options.buffer.len >= Packet.max_data_length); - var upload_pack_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(session.location.uri.path, .formatPath), - }); - upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; - } - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var body: Io.Writer = .fixed(options.buffer); - try Packet.write(.{ .data = "command=ls-refs\n" }, &body); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, &body); - } - { - const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{ - session.object_format, - }); - try Packet.write(.{ .data = object_format_packet }, &body); - } - try Packet.write(.delimiter, &body); - for (options.ref_prefixes) |ref_prefix| { - const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix}); - try Packet.write(.{ .data = ref_prefix_packet }, &body); - } - if (options.include_symrefs) { - try Packet.write(.{ .data = "symrefs\n" }, &body); - } - if (options.include_peeled) { - try Packet.write(.{ .data = "peel\n" }, &body); - } - try Packet.write(.flush, &body); - - it.* = .{ - .request = try session.transport.request(.POST, upload_pack_uri, .{ - .redirect_behavior = .unhandled, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }), - .reader = undefined, - .format = session.object_format, - .decompress = undefined, - }; - const request = &it.request; - errdefer request.deinit(); - try request.sendBodyComplete(body.buffered()); - - var response = try request.receiveHead(options.buffer); - if (response.head.status != .ok) return error.ProtocolError; - const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer); - } - - pub const RefIterator = struct { - format: Oid.Format, - request: std.http.Client.Request, - reader: *Io.Reader, - decompress: std.http.Decompress, - - pub const Ref = struct { - oid: Oid, - name: []const u8, - symref_target: ?[]const u8, - peeled: ?Oid, - }; - - pub fn deinit(iterator: *RefIterator) void { - iterator.request.deinit(); - iterator.* = undefined; - } - - pub fn next(it: *RefIterator) !?Ref { - switch (try Packet.read(it.reader)) { - .flush => return null, - .data => |data| { - const ref_data = Packet.normalizeText(data); - const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; - const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; - - const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; - const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; - - var symref_target: ?[]const u8 = null; - var peeled: ?Oid = null; - var last_sep_pos = name_sep_pos; - while (last_sep_pos < ref_data.len) { - const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; - const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; - if (mem.startsWith(u8, attribute, "symref-target:")) { - symref_target = attribute["symref-target:".len..]; - } else if (mem.startsWith(u8, attribute, "peeled:")) { - peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; - } - last_sep_pos = next_sep_pos; - } - - return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; - }, - else => return error.UnexpectedPacket, - } - } - }; - - /// Fetches the given refs from the server. A shallow fetch (depth 1) is - /// performed if the server supports it. - pub fn fetch( - session: Session, - fs: *FetchStream, - wants: []const []const u8, - /// Asserted to be at least `Packet.max_data_length`. - response_buffer: []u8, - ) !void { - const arena = session.arena; - assert(response_buffer.len >= Packet.max_data_length); - var upload_pack_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(session.location.uri.path, .formatPath), - }); - upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; - } - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var body: Io.Writer = .fixed(response_buffer); - try Packet.write(.{ .data = "command=fetch\n" }, &body); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, &body); - } - { - const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)}); - try Packet.write(.{ .data = object_format_packet }, &body); - } - try Packet.write(.delimiter, &body); - // Our packfile parser supports the OFS_DELTA object type - try Packet.write(.{ .data = "ofs-delta\n" }, &body); - // We do not currently convey server progress information to the user - try Packet.write(.{ .data = "no-progress\n" }, &body); - if (session.supports_shallow) { - try Packet.write(.{ .data = "deepen 1\n" }, &body); - } - for (wants) |want| { - var buf: [Packet.max_data_length]u8 = undefined; - const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; - try Packet.write(.{ .data = arg }, &body); - } - try Packet.write(.{ .data = "done\n" }, &body); - try Packet.write(.flush, &body); - - fs.* = .{ - .request = try session.transport.request(.POST, upload_pack_uri, .{ - .redirect_behavior = .not_allowed, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }), - .input = undefined, - .reader = undefined, - .remaining_len = undefined, - .decompress = undefined, - }; - const request = &fs.request; - errdefer request.deinit(); - - try request.sendBodyComplete(body.buffered()); - - var response = try request.receiveHead(&.{}); - if (response.head.status != .ok) return error.ProtocolError; - - const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer); - // We are not interested in any of the sections of the returned fetch - // data other than the packfile section, since we aren't doing anything - // complex like ref negotiation (this is a fresh clone). - var state: enum { section_start, section_content } = .section_start; - while (true) { - const packet = try Packet.read(reader); - switch (state) { - .section_start => switch (packet) { - .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { - fs.input = reader; - fs.reader = .{ - .buffer = &.{}, - .vtable = &.{ .stream = FetchStream.stream }, - .seek = 0, - .end = 0, - }; - fs.remaining_len = 0; - return; - } else { - state = .section_content; - }, - else => return error.UnexpectedPacket, - }, - .section_content => switch (packet) { - .delimiter => state = .section_start, - .data => {}, - else => return error.UnexpectedPacket, - }, - } - } - } - - pub const FetchStream = struct { - request: std.http.Client.Request, - input: *Io.Reader, - reader: Io.Reader, - err: ?Error = null, - remaining_len: usize, - decompress: std.http.Decompress, - - pub fn deinit(fs: *FetchStream) void { - fs.request.deinit(); - } - - pub const Error = error{ - InvalidPacket, - ProtocolError, - UnexpectedPacket, - WriteFailed, - ReadFailed, - EndOfStream, - }; - - const StreamCode = enum(u8) { - pack_data = 1, - progress = 2, - fatal_error = 3, - _, - }; - - pub fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { - const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r)); - const input = fs.input; - if (fs.remaining_len == 0) { - while (true) { - switch (Packet.peek(input) catch |err| { - fs.err = err; - return error.ReadFailed; - }) { - .flush => return error.EndOfStream, - .data => |data| switch (@as(StreamCode, @enumFromInt(data[0]))) { - .pack_data => { - input.toss(1); - fs.remaining_len = data.len - 1; - break; - }, - .fatal_error => { - fs.err = error.ProtocolError; - return error.ReadFailed; - }, - else => { - input.toss(data.len); - }, - }, - else => { - fs.err = error.UnexpectedPacket; - return error.ReadFailed; - }, - } - } - } - const buf = limit.slice(try w.writableSliceGreedy(1)); - const n = @min(buf.len, fs.remaining_len); - try input.readSliceAll(buf[0..n]); - w.advance(n); - fs.remaining_len -= n; - return n; - } - }; -}; - -const PackHeader = struct { - total_objects: u32, - - const signature = "PACK"; - const supported_version = 2; - - fn read(reader: *Io.Reader) !PackHeader { - const actual_signature = reader.take(4) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; - const version = reader.takeInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.takeInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - return .{ .total_objects = total_objects }; - } -}; - -const EntryHeader = union(Type) { - commit: Undeltified, - tree: Undeltified, - blob: Undeltified, - tag: Undeltified, - ofs_delta: OfsDelta, - ref_delta: RefDelta, - - const Type = enum(u3) { - commit = 1, - tree = 2, - blob = 3, - tag = 4, - ofs_delta = 6, - ref_delta = 7, - }; - - const Undeltified = struct { - uncompressed_length: u64, - }; - - const OfsDelta = struct { - offset: u64, - uncompressed_length: u64, - }; - - const RefDelta = struct { - base_object: Oid, - uncompressed_length: u64, - }; - - fn objectType(header: EntryHeader) Object.Type { - return switch (header) { - inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), - else => unreachable, - }; - } - - fn uncompressedLength(header: EntryHeader) u64 { - return switch (header) { - inline else => |entry| entry.uncompressed_length, - }; - } - - fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader { - const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.takeByte() catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }); - const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0; - var uncompressed_length: u64 = initial.len; - uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; - const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; - return switch (@"type") { - inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ - .uncompressed_length = uncompressed_length, - }), - .ofs_delta => .{ .ofs_delta = .{ - .offset = try readOffsetVarInt(reader), - .uncompressed_length = uncompressed_length, - } }, - .ref_delta => .{ .ref_delta = .{ - .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }, - .uncompressed_length = uncompressed_length, - } }, - }; - } -}; - -fn readOffsetVarInt(r: *Io.Reader) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.takeByte()); - var value: u64 = b.value; - while (b.has_next) { - b = @bitCast(try r.takeByte()); - value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; - value |= b.value; - } - return value; -} - -const IndexHeader = struct { - fan_out_table: [256]u32, - - const signature = "\xFFtOc"; - const supported_version = 2; - const size = 4 + 4 + @sizeOf([256]u32); - - fn read(index_header: *IndexHeader, reader: *Io.Reader) !void { - const sig = try reader.take(4); - if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; - const version = try reader.takeInt(u32, .big); - if (version != supported_version) return error.UnsupportedVersion; - try reader.readSliceEndian(u32, &index_header.fan_out_table, .big); - } -}; - -const IndexEntry = struct { - offset: u64, - crc32: u32, -}; - -/// Writes out a version 2 index for the given packfile, as documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - index_writer: *Io.File.Writer, -) !void { - try pack.seekTo(0); - - var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; - defer index_entries.deinit(allocator); - var pending_deltas: std.ArrayList(IndexEntry) = .empty; - defer pending_deltas.deinit(allocator); - - const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); - - var cache: ObjectCache = .{}; - defer cache.deinit(allocator); - var remaining_deltas = pending_deltas.items.len; - while (remaining_deltas > 0) { - var i: usize = remaining_deltas; - while (i > 0) { - i -= 1; - const delta = pending_deltas.items[i]; - if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { - try index_entries.put(allocator, oid, delta); - _ = pending_deltas.swapRemove(i); - } - } - if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; - remaining_deltas = pending_deltas.items.len; - } - - var oids: std.ArrayList(Oid) = .empty; - defer oids.deinit(allocator); - try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); - var index_entries_iter = index_entries.iterator(); - while (index_entries_iter.next()) |entry| { - oids.appendAssumeCapacity(entry.key_ptr.*); - } - mem.sortUnstable(Oid, oids.items, {}, struct { - fn lessThan(_: void, o1: Oid, o2: Oid) bool { - return mem.lessThan(u8, o1.slice(), o2.slice()); - } - }.lessThan); - - var fan_out_table: [256]u32 = undefined; - var count: u32 = 0; - var fan_out_index: u8 = 0; - for (oids.items) |oid| { - const key = oid.slice()[0]; - if (key > fan_out_index) { - @memset(fan_out_table[fan_out_index..key], count); - fan_out_index = key; - } - count += 1; - } - @memset(fan_out_table[fan_out_index..], count); - - var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{}); - const writer = &index_hashed_writer.writer; - try writer.writeAll(IndexHeader.signature); - try writer.writeInt(u32, IndexHeader.supported_version, .big); - for (fan_out_table) |fan_out_entry| { - try writer.writeInt(u32, fan_out_entry, .big); - } - - for (oids.items) |oid| { - try writer.writeAll(oid.slice()); - } - - for (oids.items) |oid| { - try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); - } - - var big_offsets: std.ArrayList(u64) = .empty; - defer big_offsets.deinit(allocator); - for (oids.items) |oid| { - const offset = index_entries.get(oid).?.offset; - if (offset <= std.math.maxInt(u31)) { - try writer.writeInt(u32, @intCast(offset), .big); - } else { - const index = big_offsets.items.len; - try big_offsets.append(allocator, offset); - try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); - } - } - for (big_offsets.items) |offset| { - try writer.writeInt(u64, offset, .big); - } - - try writer.writeAll(pack_checksum.slice()); - const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.interface.writeAll(index_checksum.slice()); - try index_writer.end(); -} - -/// Performs the first pass over the packfile data for index construction. -/// This will index all non-delta objects, queue delta objects for further -/// processing, and return the pack checksum (which is part of the index -/// format). -fn indexPackFirstPass( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), - pending_deltas: *std.ArrayList(IndexEntry), -) !Oid { - var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. - var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); - - const pack_header = try PackHeader.read(&pack_hashed.reader); - - for (0..pack_header.total_objects) |_| { - const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen(); - const entry_header = try EntryHeader.read(format, &pack_hashed.reader); - switch (entry_header) { - .commit, .tree, .blob, .tag => |object| { - var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{}); - var oid_hasher: Oid.Hashing = .init(format, &flate_buffer); - const oid_hasher_w = oid_hasher.writer(); - // The object header is not included in the pack data but is - // part of the object's ID - try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length }); - const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); - if (n != object.uncompressed_length) return error.InvalidObject; - const oid = oid_hasher.final(); - if (!skip_checksums) @compileError("TODO"); - try index_entries.put(allocator, oid, .{ - .offset = entry_offset, - .crc32 = 0, - }); - }, - inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer); - const n = try entry_decompress.reader.discardRemaining(); - if (n != delta.uncompressed_length) return error.InvalidObject; - if (!skip_checksums) @compileError("TODO"); - try pending_deltas.append(allocator, .{ - .offset = entry_offset, - .crc32 = 0, - }); - }, - } - } - - if (!skip_checksums) @compileError("TODO"); - return pack_hashed.hasher.finalResult(); -} - -/// Attempts to determine the final object ID of the given deltified object. -/// May return null if this is not yet possible (if the delta is a ref-based -/// delta and we do not yet know the offset of the base object). -fn indexPackHashDelta( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - delta: IndexEntry, - index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), - cache: *ObjectCache, -) !?Oid { - // Figure out the chain of deltas to resolve - var base_offset = delta.offset; - var base_header: EntryHeader = undefined; - var delta_offsets: std.ArrayList(u64) = .empty; - defer delta_offsets.deinit(allocator); - const base_object = while (true) { - if (cache.get(base_offset)) |base_object| break base_object; - - try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, &pack.interface); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; - }, - else => { - const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength()); - errdefer allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try cache.put(allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); - - var entry_hasher_buffer: [64]u8 = undefined; - var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer); - const entry_hasher_w = entry_hasher.writer(); - // Writes to hashers cannot fail. - entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable; - entry_hasher_w.writeAll(base_data) catch unreachable; - return entry_hasher.final(); -} - -/// Resolves a chain of deltas, returning the final base object data. `pack` is -/// assumed to be looking at the start of the object data for the base object of -/// the chain, and will then apply the deltas in `delta_offsets` in reverse order -/// to obtain the final object. -fn resolveDeltaChain( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - base_object: Object, - delta_offsets: []const u64, - cache: *ObjectCache, -) ![]const u8 { - var base_data = base_object.data; - var i: usize = delta_offsets.len; - while (i > 0) { - i -= 1; - - const delta_offset = delta_offsets[i]; - try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, &pack.interface); - const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); - defer allocator.free(delta_data); - var delta_reader: Io.Reader = .fixed(delta_data); - _ = try delta_reader.takeLeb128(u64); // base object size - const expanded_size = try delta_reader.takeLeb128(u64); - - const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; - const expanded_data = try allocator.alloc(u8, expanded_alloc_size); - errdefer allocator.free(expanded_data); - var expanded_delta_stream: Io.Writer = .fixed(expanded_data); - try expandDelta(base_data, &delta_reader, &expanded_delta_stream); - if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; - - try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); - base_data = expanded_data; - } - return base_data; -} - -/// Reads the complete contents of an object from `reader`. This function may -/// read more bytes than required from `reader`, so the reader position after -/// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 { - const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var aw: Io.Writer.Allocating = .init(allocator); - try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len); - defer aw.deinit(); - var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); - try decompress.reader.streamExact(&aw.writer, alloc_size); - return aw.toOwnedSlice(); -} - -/// Expands delta data from `delta_reader` to `writer`. -/// -/// The format of the delta data is documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: []const u8, delta_reader: *Io.Reader, writer: *Io.Writer) !void { - while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { - error.EndOfStream => return, - else => |other| return other, - }); - if (inst.copy) { - const available: packed struct { - offset1: bool, - offset2: bool, - offset3: bool, - offset4: bool, - size1: bool, - size2: bool, - size3: bool, - } = @bitCast(inst.value); - const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, - .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, - .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, - .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, - }; - const base_offset: u32 = @bitCast(offset_parts); - const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.takeByte() else 0, - .size2 = if (available.size2) try delta_reader.takeByte() else 0, - .size3 = if (available.size3) try delta_reader.takeByte() else 0, - }; - var size: u24 = @bitCast(size_parts); - if (size == 0) size = 0x10000; - try writer.writeAll(base_object[base_offset..][0..size]); - } else if (inst.value != 0) { - try delta_reader.streamExact(writer, inst.value); - } else { - return error.InvalidDeltaInstruction; - } - } -} - -/// Runs the packfile indexing and checkout test. -/// -/// The two testrepo repositories under testdata contain identical commit -/// histories and contents. -/// -/// To verify the contents of the packfiles using Git alone, run the -/// following commands in an empty directory: -/// -/// 1. `git init --object-format=(sha1|sha256)` -/// 2. `git unpack-objects assert(p.errors.items.len > 0), - else => |e| return e, - }; - - return .{ - .name = p.name, - .id = p.id, - .version = p.version, - .version_node = p.version_node, - .dependencies = try p.dependencies.clone(p.arena), - .dependencies_node = p.dependencies_node, - .paths = try p.paths.clone(p.arena), - .minimum_zig_version = p.minimum_zig_version, - .errors = try p.arena.dupe(ErrorMessage, p.errors.items), - .arena_state = arena_instance.state, - }; -} - -pub fn deinit(man: *Manifest, gpa: Allocator) void { - man.arena_state.promote(gpa).deinit(); - man.* = undefined; -} - -pub fn copyErrorsIntoBundle( - man: Manifest, - ast: Ast, - /// ErrorBundle null-terminated string index - src_path: u32, - eb: *std.zig.ErrorBundle.Wip, -) Allocator.Error!void { - for (man.errors) |msg| { - const start_loc = ast.tokenLocation(0, msg.tok); - - try eb.addRootErrorMessage(.{ - .msg = try eb.addString(msg.msg), - .src_loc = try eb.addSourceLocation(.{ - .src_path = src_path, - .span_start = ast.tokenStart(msg.tok), - .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), - .span_main = ast.tokenStart(msg.tok) + msg.off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - }); - } -} - -const Parse = struct { - gpa: Allocator, - ast: Ast, - arena: Allocator, - buf: std.ArrayList(u8), - errors: std.ArrayList(ErrorMessage), - - name: []const u8, - id: u32, - version: std.SemanticVersion, - version_node: Ast.Node.Index, - dependencies: std.array_hash_map.String(Dependency), - dependencies_node: Ast.Node.OptionalIndex, - paths: std.array_hash_map.String(void), - allow_missing_paths_field: bool, - minimum_zig_version: ?std.SemanticVersion, - - const InnerError = error{ ParseFailure, OutOfMemory }; - - fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - return fail(p, main_token, "expected top level expression to be a struct", .{}); - }; - - var have_name = false; - var have_version = false; - var have_included_paths = false; - var fingerprint: ?Package.Fingerprint = null; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "dependencies")) { - p.dependencies_node = field_init.toOptional(); - try parseDependencies(p, field_init); - } else if (mem.eql(u8, field_name, "paths")) { - have_included_paths = true; - try parseIncludedPaths(p, field_init); - } else if (mem.eql(u8, field_name, "name")) { - p.name = try parseName(p, field_init); - have_name = true; - } else if (mem.eql(u8, field_name, "fingerprint")) { - fingerprint = try parseFingerprint(p, field_init); - } else if (mem.eql(u8, field_name, "version")) { - p.version_node = field_init; - const version_text = try parseString(p, field_init); - if (version_text.len > max_version_len) { - try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); - } - p.version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v undefined; - }; - have_version = true; - } else if (mem.eql(u8, field_name, "minimum_zig_version")) { - const version_text = try parseString(p, field_init); - p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v null; - }; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!have_name) { - try appendError(p, main_token, "missing top-level 'name' field", .{}); - } else { - if (fingerprint) |n| { - if (!n.validate(p.name)) { - return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ - n.int(), Package.Fingerprint.generate(rng, p.name).int(), - }); - } - p.id = n.id; - } else { - try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ - Package.Fingerprint.generate(rng, p.name).int(), - }); - } - } - - if (!have_version) { - try appendError(p, main_token, "missing top-level 'version' field", .{}); - } - - if (!have_included_paths) { - if (p.allow_missing_paths_field) { - try p.paths.put(p.gpa, "", {}); - } else { - try appendError(p, main_token, "missing top-level 'paths' field", .{}); - } - } - } - - fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected dependencies expression to be a struct", .{}); - }; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const dep_name = try identifierTokenString(p, name_token); - const dep = try parseDependency(p, field_init); - try p.dependencies.put(p.gpa, dep_name, dep); - } - } - - fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected dependency expression to be a struct", .{}); - }; - - var dep: Dependency = .{ - .location = undefined, - .location_tok = undefined, - .location_node = undefined, - .hash = null, - .hash_tok = .none, - .hash_node = .none, - .node = node, - .name_tok = undefined, - .lazy = false, - }; - var has_location = false; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - dep.name_tok = name_token; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "url")) { - if (has_location) { - return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .url = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = ast.nodeMainToken(field_init); - dep.location_node = field_init; - } else if (mem.eql(u8, field_name, "path")) { - if (has_location) { - return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .path = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = ast.nodeMainToken(field_init); - dep.location_node = field_init; - } else if (mem.eql(u8, field_name, "hash")) { - dep.hash = parseHash(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); - dep.hash_node = field_init.toOptional(); - } else if (mem.eql(u8, field_name, "lazy")) { - dep.lazy = parseBool(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!has_location) { - try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); - } - - return dep; - } - - fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const array_init = ast.fullArrayInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected paths expression to be a list of strings", .{}); - }; - - for (array_init.ast.elements) |elem_node| { - const path_string = try parseString(p, elem_node); - // This is normalized so that it can be used in string comparisons - // against file system paths. - const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); - try p.paths.put(p.gpa, normalized, {}); - } - } - - fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { - const ast = p.ast; - if (ast.nodeTag(node) != .identifier) { - return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); - } - const ident_token = ast.nodeMainToken(node); - const token_bytes = ast.tokenSlice(ident_token); - if (mem.eql(u8, token_bytes, "true")) { - return true; - } else if (mem.eql(u8, token_bytes, "false")) { - return false; - } else { - return fail(p, ident_token, "expected boolean", .{}); - } - } - - fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - if (ast.nodeTag(node) != .number_literal) { - return fail(p, main_token, "expected integer literal", .{}); - } - const token_bytes = ast.tokenSlice(main_token); - const parsed = std.zig.parseNumberLiteral(token_bytes); - switch (parsed) { - .int => |n| return @bitCast(n), - .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ - @tagName(parsed), - }), - .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), - } - } - - fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - - if (ast.nodeTag(node) != .enum_literal) - return fail(p, main_token, "expected enum literal", .{}); - - const ident_name = ast.tokenSlice(main_token); - if (mem.startsWith(u8, ident_name, "@")) - return fail(p, main_token, "name must be a valid bare zig identifier", .{}); - - if (ident_name.len > max_name_len) - return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{ - std.zig.fmtId(ident_name), max_name_len, - }); - - return ident_name; - } - - fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - if (ast.nodeTag(node) != .string_literal) { - return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); - } - const str_lit_token = ast.nodeMainToken(node); - const token_bytes = ast.tokenSlice(str_lit_token); - p.buf.clearRetainingCapacity(); - try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const tok = ast.nodeMainToken(node); - const h = try parseString(p, node); - switch (Package.Hash.validate(h)) { - .ok => return h, - else => |t| return fail(p, tok, "invalid hash: {t}", .{t}), - } - } - - /// TODO: try to DRY this with AstGen.identifierTokenString - fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { - const ast = p.ast; - assert(ast.tokenTag(token) == .identifier); - const ident_name = ast.tokenSlice(token); - if (!mem.startsWith(u8, ident_name, "@")) { - return ident_name; - } - p.buf.clearRetainingCapacity(); - try parseStrLit(p, token, &p.buf, ident_name, 1); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - /// TODO: try to DRY this with AstGen.parseStrLit - fn parseStrLit( - p: *Parse, - token: Ast.TokenIndex, - buf: *std.ArrayList(u8), - bytes: []const u8, - offset: u32, - ) InnerError!void { - const raw_string = bytes[offset..]; - const result = r: { - var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf); - defer buf.* = aw.toArrayList(); - break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch |err| switch (err) { - error.WriteFailed => return error.OutOfMemory, - }; - }; - switch (result) { - .success => {}, - .failure => |err| try p.appendStrLitError(err, token, bytes, offset), - } - } - - /// TODO: try to DRY this with AstGen.failWithStrLitError - fn appendStrLitError( - p: *Parse, - err: std.zig.string_literal.Error, - token: Ast.TokenIndex, - bytes: []const u8, - offset: u32, - ) Allocator.Error!void { - const raw_string = bytes[offset..]; - switch (err) { - .invalid_escape_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid escape character: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .expected_hex_digit => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit, found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_unicode_escape_sequence => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "empty unicode escape sequence", - .{}, - ); - }, - .expected_hex_digit_or_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit or '}}', found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .invalid_unicode_codepoint => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "unicode escape does not correspond to a valid unicode scalar value", - .{}, - ); - }, - .expected_lbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '{{', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '}}', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_single_quote => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected single quote ('), found '{c}", - .{raw_string[bad_index]}, - ); - }, - .invalid_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid byte in string or character literal: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_char_literal => { - try p.appendErrorOff(token, offset, "empty character literal", .{}); - }, - } - } - - fn fail( - p: *Parse, - tok: Ast.TokenIndex, - comptime fmt: []const u8, - args: anytype, - ) InnerError { - try appendError(p, tok, fmt, args); - return error.ParseFailure; - } - - fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { - return appendErrorOff(p, tok, 0, fmt, args); - } - - fn appendErrorOff( - p: *Parse, - tok: Ast.TokenIndex, - byte_offset: u32, - comptime fmt: []const u8, - args: anytype, - ) Allocator.Error!void { - try p.errors.append(p.gpa, .{ - .msg = try std.fmt.allocPrint(p.arena, fmt, args), - .tok = tok, - .off = byte_offset, - }); - } -}; - -pub fn load( - io: Io, - arena: Allocator, - manifest_path: std.Build.Cache.Path, - ast: *std.zig.Ast, - error_bundle: *std.zig.ErrorBundle.Wip, - manifest: *Manifest, - allow_missing_paths_field: bool, -) !void { - const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions( - io, - manifest_path.sub_path, - arena, - .limited(max_bytes), - .@"1", - 0, - ); - - ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); - - if (ast.errors.len > 0) { - const file_path = try manifest_path.joinString(arena, ""); - try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle); - return error.ErrorsBundled; - } - - const rng: std.Random.IoSource = .{ .io = io }; - - manifest.* = try parse(arena, ast, rng.interface(), .{ - .allow_missing_paths_field = allow_missing_paths_field, - }); - - if (manifest.errors.len > 0) { - const src_path = try error_bundle.printString("{f}", .{manifest_path}); - try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle); - return error.ErrorsBundled; - } -} - -test "basic" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = .foo, - \\ .fingerprint = 0x8c736521490b23df, - \\ .version = "3.2.1", - \\ .paths = .{""}, - \\ .dependencies = .{ - \\ .bar = .{ - \\ .url = "https://example.com/baz.tar.gz", - \\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", - \\ }, - \\ }, - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var rng = std.Random.DefaultPrng.init(0); - - var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 0); - try testing.expectEqualStrings("foo", manifest.name); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 3, - .minor = 2, - .patch = 1, - }), manifest.version); - - try testing.expect(manifest.dependencies.count() == 1); - try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); - try testing.expectEqualStrings( - "https://example.com/baz.tar.gz", - manifest.dependencies.values()[0].location.url, - ); - try testing.expectEqualStrings( - "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", - manifest.dependencies.values()[0].hash orelse return error.TestFailed, - ); - - try testing.expect(manifest.minimum_zig_version == null); -} - -test "minimum_zig_version" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = .foo, - \\ .fingerprint = 0x8c736521490b23df, - \\ .version = "3.2.1", - \\ .paths = .{""}, - \\ .minimum_zig_version = "0.11.1", - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var rng = std.Random.DefaultPrng.init(0); - - var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 0); - try testing.expect(manifest.dependencies.count() == 0); - - try testing.expect(manifest.minimum_zig_version != null); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 0, - .minor = 11, - .patch = 1, - }), manifest.minimum_zig_version.?); -} - -test "minimum_zig_version - invalid version" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = .foo, - \\ .fingerprint = 0x8c736521490b23df, - \\ .version = "3.2.1", - \\ .minimum_zig_version = "X.11.1", - \\ .paths = .{""}, - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var rng = std.Random.DefaultPrng.init(0); - - var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 1); - try testing.expect(manifest.dependencies.count() == 0); - - try testing.expect(manifest.minimum_zig_version == null); -} diff --git a/src/Package/Module.zig b/src/Package/Module.zig deleted file mode 100644 index 0c7e4166adf7d6c290cbb330bb12857c1cf21d90..0000000000000000000000000000000000000000 --- a/src/Package/Module.zig +++ /dev/null @@ -1,529 +0,0 @@ -//! Corresponds to something that Zig source code can `@import`. - -/// The root directory of the module. Only files inside this directory can be imported. -root: Compilation.Path, -/// Path to the root source file of this module. Relative to `root`. May contain path separators. -root_src_path: []const u8, -/// Name used in compile errors. Looks like "root.foo.bar". -fully_qualified_name: []const u8, -/// The dependency table of this module. The shared dependencies 'std' and -/// 'root' are not specified in every module dependency table, but are stored -/// separately in `Zcu`. 'builtin' is also not stored here, although it is -/// not necessarily the same between all modules. Handling of `@import` in -/// the rest of the compiler must detect these special names and use the -/// correct module instead of consulting `deps`. -deps: Deps = .{}, - -resolved_target: ResolvedTarget, -optimize_mode: std.lang.OptimizeMode, -code_model: std.lang.CodeModel, -single_threaded: bool, -error_tracing: bool, -valgrind: bool, -pic: bool, -strip: bool, -omit_frame_pointer: bool, -stack_check: bool, -stack_protector: u32, -red_zone: bool, -sanitize_c: std.zig.SanitizeC, -sanitize_thread: bool, -fuzz: bool, -unwind_tables: std.lang.UnwindTables, -cc_argv: []const []const u8, -/// (SPIR-V) whether to generate a structured control flow graph or not -structured_cfg: bool, -no_builtin: bool, - -pub const Deps = std.array_hash_map.String(*Module); - -pub const Tree = struct { - /// Each `Package` exposes a `Module` with build.zig as its root source file. - build_module_table: std.array_hash_map.Auto(MultiHashHexDigest, *Module), -}; - -pub const CreateOptions = struct { - paths: Paths, - fully_qualified_name: []const u8, - - cc_argv: []const []const u8, - inherited: Inherited, - global: Compilation.Config, - /// If this is null then `resolved_target` must be non-null. - parent: ?*Package.Module, - - pub const Paths = struct { - root: Compilation.Path, - /// Relative to `root`. May contain path separators. - root_src_path: []const u8, - }; - - pub const Inherited = struct { - /// If this is null then `parent` must be non-null. - resolved_target: ?ResolvedTarget = null, - optimize_mode: ?std.lang.OptimizeMode = null, - code_model: ?std.lang.CodeModel = null, - single_threaded: ?bool = null, - error_tracing: ?bool = null, - valgrind: ?bool = null, - pic: ?bool = null, - strip: ?bool = null, - omit_frame_pointer: ?bool = null, - stack_check: ?bool = null, - /// null means default. - /// 0 means no stack protector. - /// other number means stack protection with that buffer size. - stack_protector: ?u32 = null, - red_zone: ?bool = null, - unwind_tables: ?std.lang.UnwindTables = null, - sanitize_c: ?std.zig.SanitizeC = null, - sanitize_thread: ?bool = null, - fuzz: ?bool = null, - structured_cfg: ?bool = null, - no_builtin: ?bool = null, - }; -}; - -pub const ResolvedTarget = struct { - result: std.Target, - is_native_os: bool, - is_native_abi: bool, - is_explicit_dynamic_linker: bool, - llvm_cpu_features: ?[*:0]const u8 = null, -}; - -pub const CreateError = error{ - OutOfMemory, - ValgrindUnsupportedOnTarget, - TargetRequiresSingleThreaded, - BackendRequiresSingleThreaded, - TargetRequiresPic, - PieRequiresPic, - DynamicLinkingRequiresPic, - TargetHasNoRedZone, - StackCheckUnsupportedByTarget, - StackProtectorUnsupportedByTarget, - StackProtectorUnavailableWithoutLibC, -}; - -/// At least one of `parent` and `resolved_target` must be non-null. -pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { - if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); - if (options.inherited.fuzz == true) assert(options.global.any_fuzz); - if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); - if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); - if (options.inherited.sanitize_c) |sc| if (sc != .off) assert(options.global.any_sanitize_c != .off); - if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); - - const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; - const target = &resolved_target.result; - - const optimize_mode = options.inherited.optimize_mode orelse - if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; - - const strip = b: { - if (options.inherited.strip) |x| break :b x; - if (options.parent) |p| break :b p.strip; - break :b options.global.root_strip; - }; - - const zig_backend = target_util.zigBackend(target, options.global.use_llvm); - - const valgrind = b: { - if (!target_util.hasValgrindSupport(target, zig_backend)) { - if (options.inherited.valgrind == true) - return error.ValgrindUnsupportedOnTarget; - break :b false; - } - if (options.inherited.valgrind) |x| break :b x; - if (options.parent) |p| break :b p.valgrind; - if (strip) break :b false; - break :b optimize_mode == .Debug; - }; - - const single_threaded = b: { - if (target_util.alwaysSingleThreaded(target)) { - if (options.inherited.single_threaded == false) - return error.TargetRequiresSingleThreaded; - break :b true; - } - - if (options.global.have_zcu) { - if (!target_util.supportsThreads(target, zig_backend)) { - if (options.inherited.single_threaded == false) - return error.BackendRequiresSingleThreaded; - break :b true; - } - } - - if (options.inherited.single_threaded) |x| break :b x; - if (options.parent) |p| break :b p.single_threaded; - break :b target_util.defaultSingleThreaded(target); - }; - - const error_tracing = b: { - if (options.inherited.error_tracing) |x| break :b x; - if (options.parent) |p| break :b p.error_tracing; - break :b options.global.root_error_tracing; - }; - - const pic = b: { - if (target_util.requiresPic(target, options.global.link_libc)) { - if (options.inherited.pic == false) - return error.TargetRequiresPic; - break :b true; - } - if (options.global.pie) { - if (options.inherited.pic == false) - return error.PieRequiresPic; - break :b true; - } - if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) { - if (options.inherited.pic == false) - return error.DynamicLinkingRequiresPic; - break :b true; - } - if (options.inherited.pic) |x| break :b x; - if (options.parent) |p| break :b p.pic; - - // Default to PIC on targets where we default to producing PIEs to make - // the common case of linking objects and static libraries into an - // executable work out of the box. - break :b target_util.defaultPie(target); - }; - - const red_zone = b: { - if (!target_util.hasRedZone(target)) { - if (options.inherited.red_zone == true) - return error.TargetHasNoRedZone; - break :b false; - } - if (options.inherited.red_zone) |x| break :b x; - if (options.parent) |p| break :b p.red_zone; - break :b true; - }; - - const omit_frame_pointer = b: { - if (options.inherited.omit_frame_pointer) |x| break :b x; - if (options.parent) |p| break :b p.omit_frame_pointer; - if (optimize_mode == .ReleaseSmall) { - // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. - // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) - // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). - break :b !target.cpu.arch.isX86(); - } - break :b false; - }; - - const sanitize_thread = b: { - if (options.inherited.sanitize_thread) |x| break :b x; - if (options.parent) |p| break :b p.sanitize_thread; - break :b false; - }; - - const unwind_tables = b: { - if (options.inherited.unwind_tables) |x| break :b x; - if (options.parent) |p| break :b p.unwind_tables; - - break :b target_util.defaultUnwindTables( - target, - options.global.link_libunwind, - sanitize_thread or options.global.any_sanitize_thread, - ); - }; - - const fuzz = b: { - if (options.inherited.fuzz) |x| break :b x; - if (options.parent) |p| break :b p.fuzz; - break :b false; - }; - - const code_model: std.lang.CodeModel = b: { - if (options.inherited.code_model) |x| break :b x; - if (options.parent) |p| break :b p.code_model; - break :b .default; - }; - - const is_safe_mode = switch (optimize_mode) { - .Debug, .ReleaseSafe => true, - .ReleaseFast, .ReleaseSmall => false, - }; - - const sanitize_c: std.zig.SanitizeC = b: { - if (options.inherited.sanitize_c) |x| break :b x; - if (options.parent) |p| break :b p.sanitize_c; - break :b switch (optimize_mode) { - .Debug => .full, - // It's recommended to use the minimal runtime in production - // environments due to the security implications of the full runtime. - // The minimal runtime doesn't provide much benefit over simply - // trapping, however, so we do that instead. - .ReleaseSafe => .trap, - .ReleaseFast, .ReleaseSmall => .off, - }; - }; - - const stack_check = b: { - if (!target_util.supportsStackProbing(target, zig_backend)) { - if (options.inherited.stack_check == true) - return error.StackCheckUnsupportedByTarget; - break :b false; - } - if (options.inherited.stack_check) |x| break :b x; - if (options.parent) |p| break :b p.stack_check; - break :b is_safe_mode; - }; - - const stack_protector: u32 = sp: { - const use_zig_backend = options.global.have_zcu or - (options.global.any_c_source_files and options.global.c_frontend == .aro); - if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnsupportedByTarget; - } - break :sp 0; - } - - if (options.global.any_c_source_files and options.global.c_frontend == .clang and - !target_util.clangSupportsStackProtector(target)) - { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnsupportedByTarget; - } - break :sp 0; - } - - // This logic is checking for linking libc because otherwise our start code - // which is trying to set up TLS (i.e. the fs/gs registers) but the stack - // protection code depends on fs/gs registers being already set up. - // If we were able to annotate start code, or perhaps the entire std lib, - // as being exempt from stack protection checks, we could change this logic - // to supporting stack protection even when not linking libc. - // TODO file issue about this - if (!options.global.link_libc) { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnavailableWithoutLibC; - } - break :sp 0; - } - - if (options.inherited.stack_protector) |x| break :sp x; - if (options.parent) |p| break :sp p.stack_protector; - if (!is_safe_mode) break :sp 0; - - break :sp target_util.default_stack_protector_buffer_size; - }; - - const structured_cfg = b: { - if (options.inherited.structured_cfg) |x| break :b x; - if (options.parent) |p| break :b p.structured_cfg; - // We always want a structured control flow in shaders. This option is - // only relevant for OpenCL kernels. - break :b switch (target.os.tag) { - .opencl => false, - else => true, - }; - }; - - const no_builtin = b: { - if (options.inherited.no_builtin) |x| break :b x; - if (options.parent) |p| break :b p.no_builtin; - - break :b target.cpu.arch.isBpf(); - }; - - const llvm_cpu_features: ?[*:0]const u8 = b: { - if (resolved_target.llvm_cpu_features) |x| break :b x; - if (!options.global.use_llvm) break :b null; - - var buf = std.array_list.Managed(u8).init(arena); - var disabled_features = std.array_list.Managed(u8).init(arena); - defer disabled_features.deinit(); - - // Append disabled features after enabled ones, so that their effects aren't overwritten. - for (target.cpu.arch.allFeaturesList()) |feature| { - if (feature.llvm_name) |llvm_name| { - // Ignore these until we figure out how to handle the concept of omitting features. - // See https://github.com/ziglang/zig/issues/23539 - if (target_util.isDynamicAMDGCNFeature(target, feature)) continue; - - if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue; - if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue; - - var is_enabled = target.cpu.features.isEnabled(feature.index); - if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) { - is_enabled = !omit_frame_pointer; - } - - if (is_enabled) { - try buf.ensureUnusedCapacity(2 + llvm_name.len); - buf.appendAssumeCapacity('+'); - buf.appendSliceAssumeCapacity(llvm_name); - buf.appendAssumeCapacity(','); - } else { - try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); - disabled_features.appendAssumeCapacity('-'); - disabled_features.appendSliceAssumeCapacity(llvm_name); - disabled_features.appendAssumeCapacity(','); - } - } - } - - try buf.appendSlice(disabled_features.items); - if (buf.items.len == 0) break :b ""; - assert(std.mem.endsWith(u8, buf.items, ",")); - buf.items[buf.items.len - 1] = 0; - buf.shrinkAndFree(buf.items.len); - break :b buf.items[0 .. buf.items.len - 1 :0].ptr; - }; - - const mod = try arena.create(Module); - mod.* = .{ - .root = options.paths.root, - .root_src_path = options.paths.root_src_path, - .fully_qualified_name = options.fully_qualified_name, - .resolved_target = .{ - .result = target.*, - .is_native_os = resolved_target.is_native_os, - .is_native_abi = resolved_target.is_native_abi, - .is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker, - .llvm_cpu_features = llvm_cpu_features, - }, - .optimize_mode = optimize_mode, - .single_threaded = single_threaded, - .error_tracing = error_tracing, - .valgrind = valgrind, - .pic = pic, - .strip = strip, - .omit_frame_pointer = omit_frame_pointer, - .stack_check = stack_check, - .stack_protector = stack_protector, - .code_model = code_model, - .red_zone = red_zone, - .sanitize_c = sanitize_c, - .sanitize_thread = sanitize_thread, - .fuzz = fuzz, - .unwind_tables = unwind_tables, - .cc_argv = options.cc_argv, - .structured_cfg = structured_cfg, - .no_builtin = no_builtin, - }; - return mod; -} - -/// All fields correspond to `CreateOptions`. -pub const LimitedOptions = struct { - root: Compilation.Path, - root_src_path: []const u8, - fully_qualified_name: []const u8, -}; - -/// This one can only be used if the Module will only be used for AstGen and earlier in -/// the pipeline. Illegal behavior occurs if a limited module touches Sema. -pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { - const mod = try gpa.create(Module); - mod.* = .{ - .root = options.root, - .root_src_path = options.root_src_path, - .fully_qualified_name = options.fully_qualified_name, - - .resolved_target = undefined, - .optimize_mode = undefined, - .code_model = undefined, - .single_threaded = undefined, - .error_tracing = undefined, - .valgrind = undefined, - .pic = undefined, - .strip = undefined, - .omit_frame_pointer = undefined, - .stack_check = undefined, - .stack_protector = undefined, - .red_zone = undefined, - .sanitize_c = undefined, - .sanitize_thread = undefined, - .fuzz = undefined, - .unwind_tables = undefined, - .cc_argv = undefined, - .structured_cfg = undefined, - .no_builtin = undefined, - }; - return mod; -} - -/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task. -pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: Compilation.Directories) Allocator.Error!*Module { - const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash()); - const new = try arena.create(Module); - new.* = .{ - .root = try .fromRoot(arena, dirs, .global_cache, sub_path), - .root_src_path = "builtin.zig", - .fully_qualified_name = "builtin", - .resolved_target = .{ - .result = opts.target, - // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - .llvm_cpu_features = null, - }, - .optimize_mode = opts.optimize_mode, - .single_threaded = opts.single_threaded, - .error_tracing = opts.error_tracing, - .valgrind = opts.valgrind, - .pic = opts.pic, - .strip = opts.strip, - .omit_frame_pointer = opts.omit_frame_pointer, - .code_model = opts.code_model, - .sanitize_thread = opts.sanitize_thread, - .fuzz = opts.fuzz, - .unwind_tables = opts.unwind_tables, - .cc_argv = &.{}, - // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. - .stack_check = false, - .stack_protector = 0, - .red_zone = false, - .sanitize_c = .off, - .structured_cfg = false, - .no_builtin = false, - }; - return new; -} - -/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module. -pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin { - assert(global.have_zcu); - return .{ - .target = m.resolved_target.result, - .zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm), - .output_mode = global.output_mode, - .link_mode = global.link_mode, - .unwind_tables = m.unwind_tables, - .is_test = global.is_test, - .single_threaded = m.single_threaded, - .link_libc = global.link_libc, - .link_libcpp = global.link_libcpp, - .optimize_mode = m.optimize_mode, - .error_tracing = m.error_tracing, - .valgrind = m.valgrind, - .sanitize_thread = m.sanitize_thread, - .fuzz = m.fuzz, - .pic = m.pic, - .pie = global.pie, - .strip = m.strip, - .code_model = m.code_model, - .omit_frame_pointer = m.omit_frame_pointer, - .wasi_exec_model = global.wasi_exec_model, - }; -} - -const Module = @This(); -const Package = @import("../Package.zig"); -const std = @import("std"); -const Allocator = std.mem.Allocator; -const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; -const target_util = @import("../target.zig"); -const Cache = std.Build.Cache; -const Builtin = @import("../Builtin.zig"); -const assert = std.debug.assert; -const Compilation = @import("../Compilation.zig"); -const File = @import("../Zcu.zig").File; diff --git a/src/Zcu.zig b/src/Zcu.zig index e3a36fe31b987dce03f89d1f4694cf718d72d3dc..41758eaf3c3b2ae6ef9ce3adfef9ccb734bda5f4 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -34,7 +34,6 @@ const AstGen = std.zig.AstGen; const Sema = @import("Sema.zig"); const target_util = @import("target.zig"); const build_options = @import("build_options"); -const isUpDir = @import("introspect.zig").isUpDir; const InternPool = @import("InternPool.zig"); const Alignment = InternPool.Alignment; const AnalUnit = InternPool.AnalUnit; diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index e1cda016c163cdfde8cb693c5f143f2f86057980..f9aede4f72b4c0a4e32b06f02ac509ef62d30d09 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -23,7 +23,6 @@ const builtin = @import("builtin"); const dev = @import("../dev.zig"); const InternPool = @import("../InternPool.zig"); const AnalUnit = InternPool.AnalUnit; -const introspect = @import("../introspect.zig"); const Module = @import("../Package.zig").Module; const Sema = @import("../Sema.zig"); const target_util = @import("../target.zig"); diff --git a/src/dev.zig b/src/dev.zig index a6f97799157440329f3b248959e47f1585cfc1b3..8d3723ec101b8984b9d33381754ec3388374d3f6 100644 --- a/src/dev.zig +++ b/src/dev.zig @@ -112,7 +112,6 @@ pub const Env = enum { .translate_c_command, .fmt_command, .jit_command, - .fetch_command, .init_command, .targets_command, .version_command, @@ -252,7 +251,6 @@ pub const Feature = enum { translate_c_command, fmt_command, jit_command, - fetch_command, init_command, targets_command, version_command, diff --git a/src/introspect.zig b/src/introspect.zig deleted file mode 100644 index 13d00520936973fccc37ede11256842e0826f423..0000000000000000000000000000000000000000 --- a/src/introspect.zig +++ /dev/null @@ -1,220 +0,0 @@ -const builtin = @import("builtin"); - -const std = @import("std"); -const Io = std.Io; -const Dir = std.Io.Dir; -const mem = std.mem; -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const assert = std.debug.assert; - -const build_options = @import("build_options"); - -const Compilation = @import("Compilation.zig"); -const Package = @import("Package.zig"); - -/// Returns the sub_path that worked, or `null` if none did. -/// The path of the returned Directory is relative to `base`. -/// The handle of the returned Directory is open. -fn testZigInstallPrefix(io: Io, base_dir: Io.Dir) ?Cache.Directory { - const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig"; - - zig_dir: { - // Try lib/zig/std/std.zig - const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig"; - var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir; - const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { - test_zig_dir.close(io); - break :zig_dir; - }; - file.close(io); - return .{ .handle = test_zig_dir, .path = lib_zig }; - } - - // Try lib/std/std.zig - var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null; - const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { - test_zig_dir.close(io); - return null; - }; - file.close(io); - return .{ .handle = test_zig_dir, .path = "lib" }; -} - -/// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory { - const cwd_path = try getResolvedCwd(io, gpa); - defer gpa.free(cwd_path); - const self_exe_path = try std.process.executablePathAlloc(io, gpa); - defer gpa.free(self_exe_path); - - return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path); -} - -/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This -/// means the path has no repeated separators, no "." or ".." components, and no trailing separator. -/// On WASI, "" is returned instead of ".". -pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 { - if (builtin.target.os.tag == .wasi) { - if (std.debug.runtime_safety) { - const cwd = try std.process.currentPathAlloc(io, gpa); - defer gpa.free(cwd); - assert(mem.eql(u8, cwd, ".")); - } - return ""; - } - const cwd = try std.process.currentPathAlloc(io, gpa); - defer gpa.free(cwd); - const resolved = try Dir.path.resolve(gpa, &.{cwd}); - assert(Dir.path.isAbsolute(resolved)); - return resolved; -} - -/// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDirFromSelfExe( - allocator: Allocator, - io: Io, - /// The return value of `getResolvedCwd`. - /// Passed as an argument to avoid pointlessly repeating the call. - cwd_path: []const u8, - self_exe_path: []const u8, -) error{ OutOfMemory, FileNotFound }!Cache.Directory { - const cwd = Io.Dir.cwd(); - var cur_path: []const u8 = self_exe_path; - while (Dir.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { - var base_dir = cwd.openDir(io, dirname, .{}) catch continue; - defer base_dir.close(io); - - const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue; - const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? }); - defer allocator.free(p); - - const resolved = try resolvePath(allocator, cwd_path, &.{p}); - return .{ - .handle = sub_directory.handle, - .path = if (resolved.len == 0) null else resolved, - }; - } - return error.FileNotFound; -} - -pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 { - if (std.zig.EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) |value| return value; - - const app_name = "zig"; - - switch (builtin.os.tag) { - .wasi => @compileError("on WASI the global cache dir must be resolved with preopens"), - .windows => { - const local_app_data_dir = std.zig.EnvVar.LOCALAPPDATA.get(environ_map) orelse - return error.AppDataDirUnavailable; - return Dir.path.join(arena, &.{ local_app_data_dir, app_name }); - }, - else => { - if (std.zig.EnvVar.XDG_CACHE_HOME.get(environ_map)) |cache_root| { - if (cache_root.len > 0) { - return Dir.path.join(arena, &.{ cache_root, app_name }); - } - } - if (std.zig.EnvVar.HOME.get(environ_map)) |home| { - if (home.len > 0) { - return Dir.path.join(arena, &.{ home, ".cache", app_name }); - } - } - return error.AppDataDirUnavailable; - }, - } -} - -/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would -/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd -/// returns the empty string ("") instead of ".". -pub fn resolvePath( - gpa: Allocator, - /// The return value of `getResolvedCwd`. - /// Passed as an argument to avoid pointlessly repeating the call. - cwd_resolved: []const u8, - paths: []const []const u8, -) Allocator.Error![]u8 { - if (builtin.target.os.tag == .wasi) { - assert(mem.eql(u8, cwd_resolved, "")); - const res = try Dir.path.resolve(gpa, paths); - if (mem.eql(u8, res, ".")) { - gpa.free(res); - return ""; - } - return res; - } - - // Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`. - for (paths) |p| { - if (Dir.path.isAbsolute(p)) break; // absolute path - if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir - } else { - // no absolute path, no "..". - const res = try Dir.path.resolve(gpa, paths); - if (mem.eql(u8, res, ".")) { - gpa.free(res); - return ""; - } - assert(!Dir.path.isAbsolute(res)); - assert(!isUpDir(res)); - return res; - } - - // The fast path failed; resolve the whole thing. - // Optimization: `paths` often has just one element. - const path_resolved = switch (paths.len) { - 0 => unreachable, - 1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }), - else => r: { - const all_paths = try gpa.alloc([]const u8, paths.len + 1); - defer gpa.free(all_paths); - all_paths[0] = cwd_resolved; - @memcpy(all_paths[1..], paths); - break :r try Dir.path.resolve(gpa, all_paths); - }, - }; - errdefer gpa.free(path_resolved); - - assert(Dir.path.isAbsolute(path_resolved)); - assert(Dir.path.isAbsolute(cwd_resolved)); - - if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd - if (path_resolved.len == cwd_resolved.len) { - // equal to cwd - gpa.free(path_resolved); - return ""; - } - if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs) - - // in cwd; extract sub path - const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]); - gpa.free(path_resolved); - return sub_path; -} - -pub fn isUpDir(p: []const u8) bool { - return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep); -} - -pub const default_local_zig_cache_basename = ".zig-cache"; - -/// Searches upwards from `cwd` for a directory containing a `build.zig` file. -/// If such a directory is found, returns the path to it joined to the `.zig_cache` name. -/// Otherwise, returns `null`, indicating no suitable local cache location. -pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 { - var cur_dir = cwd; - while (true) { - const joined = try Dir.path.join(arena, &.{ cur_dir, Package.build_zig_basename }); - if (Io.Dir.cwd().access(io, joined, .{})) |_| { - return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename }); - } else |err| switch (err) { - error.FileNotFound => { - cur_dir = Dir.path.dirname(cur_dir) orelse return null; - continue; - }, - else => return null, - } - } -} diff --git a/src/main.zig b/src/main.zig index 84b6553ff64750279537a242729f6ecf9c807639..4e255a4d496a9c7271c4d91b555b0e77647e5b81 100644 --- a/src/main.zig +++ b/src/main.zig @@ -21,13 +21,13 @@ const AstGen = std.zig.AstGen; const ZonGen = std.zig.ZonGen; const Server = std.zig.Server; const stringToEnum = std.meta.stringToEnum; +const allocPrint = std.fmt.allocPrint; pub const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); const link = @import("link.zig"); const Package = @import("Package.zig"); const build_options = @import("build_options"); -const introspect = @import("introspect.zig"); const wasi_libc = @import("libs/wasi_libc.zig"); const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); @@ -353,9 +353,15 @@ fn mainArgs( dev.check(.ar_command); return process.exit(try llvmArMain(arena, args)); }, - .build => { - dev.check(.build_command); - return cmdBuild(gpa, arena, io, cmd_args, environ_map); + .build, .fetch => { + return jitCmd(gpa, arena, io, args, environ_map, .{ + .cmd_name = "maker", + .root_src_path = "Maker.zig", + .prepend_zig_lib_dir_path = true, + .prepend_global_cache_path = true, + .prepend_zig_exe_path = true, + .prepend_seed = true, + }); }, .clang, .@"-cc1", .@"-cc1as" => { dev.check(.clang_command); @@ -385,7 +391,6 @@ fn mainArgs( .depend_on_aro = true, .prepend_zig_lib_dir_path = true, .server = use_server, - .color = Color.settingFromEnvironment(environ_map), }); }, .fmt => { @@ -396,25 +401,19 @@ fn mainArgs( return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "objcopy", .root_src_path = "objcopy.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, .objdump => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "objdump", .root_src_path = "objdump.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, - .fetch => { - return cmdFetch(gpa, arena, io, cmd_args, environ_map); - }, .libc => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "libc", .root_src_path = "libc.zig", .prepend_zig_lib_dir_path = true, - .color = Color.settingFromEnvironment(environ_map), }); }, .std => { @@ -424,7 +423,6 @@ fn mainArgs( .prepend_zig_lib_dir_path = true, .prepend_zig_exe_path = true, .prepend_global_cache_path = true, - .color = Color.settingFromEnvironment(environ_map), }); }, .init => { @@ -461,7 +459,6 @@ fn mainArgs( return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "reduce", .root_src_path = "reduce.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, .zen => { @@ -2977,7 +2974,7 @@ fn buildOutputType( while (preprocessor_args_it.next()) |arg| { if (mem.eql(u8, arg, "-MD") or mem.eql(u8, arg, "-MMD") or mem.eql(u8, arg, "-MT")) { disable_c_depfile = true; - const cc_arg = try std.fmt.allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() }); + const cc_arg = try allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() }); try cc_argv.append(arena, cc_arg); } else { fatal("unsupported preprocessor arg: {s}", .{arg}); @@ -3222,7 +3219,7 @@ fn buildOutputType( else => process.executablePathAlloc(io, arena) catch |err| fatal("unable to find zig self exe path: {t}", .{err}), }; - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( @@ -3421,9 +3418,9 @@ fn buildOutputType( .yes_default_value => if (create_module.resolved_options.output_mode == .Lib and create_module.resolved_options.link_mode == .dynamic and target.ofmt == .elf) if (have_version) - try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major }) + try allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major }) else - try std.fmt.allocPrint(arena, "lib{s}.so", .{root_name}) + try allocPrint(arena, "lib{s}.so", .{root_name}) else null, }; @@ -3433,7 +3430,7 @@ fn buildOutputType( .yes_default_path => emit: { if (output_to_cache != null) break :emit .yes_cache; const name = switch (clang_preprocessor_mode) { - .pch => try std.fmt.allocPrint(arena, "{s}.pch", .{root_name}), + .pch => try allocPrint(arena, "{s}.pch", .{root_name}), else => try std.zig.binNameAlloc(arena, .{ .root_name = root_name, .cpu_arch = target.cpu.arch, @@ -3469,16 +3466,16 @@ fn buildOutputType( }, }; - const default_h_basename = try std.fmt.allocPrint(arena, "{s}.h", .{root_name}); + const default_h_basename = try allocPrint(arena, "{s}.h", .{root_name}); const emit_h_resolved = emit_h.resolve(io, default_h_basename, output_to_cache); - const default_asm_basename = try std.fmt.allocPrint(arena, "{s}.s", .{root_name}); + const default_asm_basename = try allocPrint(arena, "{s}.s", .{root_name}); const emit_asm_resolved = emit_asm.resolve(io, default_asm_basename, output_to_cache); - const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{s}.ll", .{root_name}); + const default_llvm_ir_basename = try allocPrint(arena, "{s}.ll", .{root_name}); const emit_llvm_ir_resolved = emit_llvm_ir.resolve(io, default_llvm_ir_basename, output_to_cache); - const default_llvm_bc_basename = try std.fmt.allocPrint(arena, "{s}.bc", .{root_name}); + const default_llvm_bc_basename = try allocPrint(arena, "{s}.bc", .{root_name}); const emit_llvm_bc_resolved = emit_llvm_bc.resolve(io, default_llvm_bc_basename, output_to_cache); const emit_docs_resolved = emit_docs.resolve(io, "docs", output_to_cache); @@ -3499,7 +3496,7 @@ fn buildOutputType( fatal("the argument -femit-implib is allowed only when building a Windows DLL", .{}); } } - const default_implib_basename = try std.fmt.allocPrint(arena, "{s}.lib", .{root_name}); + const default_implib_basename = try allocPrint(arena, "{s}.lib", .{root_name}); const emit_implib_resolved: Compilation.CreateOptions.Emit = switch (emit_implib) { .no => .no, .yes => emit_implib.resolve(io, default_implib_basename, output_to_cache), @@ -3528,7 +3525,7 @@ fn buildOutputType( // "-" is stdin. Dump it to a real file. const sep = fs.path.sep_str; - const dump_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{ + const dump_path = try allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{ randInt(io, u64), ext.canonicalName(target), }); try dirs.local_cache.handle.createDirPath(io, "tmp"); @@ -3557,7 +3554,7 @@ fn buildOutputType( const bin_digest: Cache.BinDigest = hasher.hasher.finalResult(); - const sub_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{ + const sub_path = try allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{ &bin_digest, ext.canonicalName(target), }); try dirs.local_cache.handle.rename(dump_path, dirs.local_cache.handle, sub_path, io); @@ -4586,7 +4583,7 @@ fn runOrTest( try argv.append(exe_path); if (arg_mode == .zig_test) { try argv.append( - try std.fmt.allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}), + try allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}), ); } } else { @@ -4794,7 +4791,7 @@ fn cmdTranslateC( assert(comp.c_source_files.len == 1); const c_source_file = comp.c_source_files[0]; - const translated_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{comp.root_name}); + const translated_basename = try allocPrint(arena, "{s}.zig", .{comp.root_name}); var man: Cache.Manifest = comp.obtainCObjectCacheManifest(comp.root_mod); man.want_shared_lock = false; @@ -4872,7 +4869,6 @@ pub fn translateC( .root_src_path = "translate-c/main.zig", .depend_on_aro = true, .capture = capture, - .color = Color.settingFromEnvironment(environ_map), }); } @@ -4912,7 +4908,7 @@ fn cmdInit(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8) ! } } - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); const cwd_basename = fs.path.basename(cwd_path); const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename); @@ -5027,1065 +5023,17 @@ test sanitizeExampleName { try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); } -fn cmdBuild( - gpa: Allocator, - arena: Allocator, - io: Io, - args: []const []const u8, - environ_map: *process.Environ.Map, -) !void { - var build_file: ?[]const u8 = null; - var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); - var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) - .Debug - else - .ReleaseSafe; - var configure_argv: std.ArrayList([]const u8) = .empty; - var make_argv: std.ArrayList([]const u8) = .empty; - var cached_passthru_configure: std.ArrayList(u32) = .empty; - var forks: std.ArrayList(Fork) = .empty; - var reference_trace: ?u32 = null; - var debug_compile_errors = false; - var verbose_link = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_LINK.isSet(environ_map); - var verbose_cc = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_CC.isSet(environ_map); - var verbose_air = false; - var verbose_intern_pool = false; - var verbose_generic_instances = false; - var verbose_llvm_ir: ?[]const u8 = null; - var verbose_llvm_bc: ?[]const u8 = null; - var verbose_llvm_cpu_features = false; - var fetch_only = false; - var fetch_mode: Package.Fetch.JobQueue.Mode = .needed; - var system_pkg_dir_path: ?[]const u8 = null; - var debug_target: ?[]const u8 = null; - var debug_libc_paths_file: ?[]const u8 = null; - var cache_poison: std.Build.Graph.CachePoison = .pure; - var print_configuration_path: bool = false; - - const self_exe_path = try process.executablePathAlloc(io, arena); - const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); - - try configure_argv.ensureUnusedCapacity(arena, 16); - try make_argv.ensureUnusedCapacity(arena, 16); - try cached_passthru_configure.ensureUnusedCapacity(arena, 16); - - _ = configure_argv.addOneAssumeCapacity(); // configurer executable - _ = make_argv.addOneAssumeCapacity(); // maker executable - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined }; - const make_argv_index_zig_lib_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const make_argv_index_build_root = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined }; - const make_argv_index_cache_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined }; - const make_argv_index_global_cache_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined }; - const argv_index_configuration_file = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed }; - const argv_index_seed = make_argv.items.len - 1; - - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const conf_argv_index_build_root = configure_argv.items.len - 1; - - var color: Color = Color.settingFromEnvironment(environ_map); - var n_jobs: ?u32 = null; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - try configure_argv.ensureUnusedCapacity(arena, 2); - - if (mem.startsWith(u8, arg, "-D") or - mem.startsWith(u8, arg, "-fsys=") or - mem.startsWith(u8, arg, "-fno-sys=") or - mem.startsWith(u8, arg, "--release=") or - mem.eql(u8, arg, "--release")) - { - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--system")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - system_pkg_dir_path = args[i]; - - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. - continue; - } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { - color = stringToEnum(Color, rest) orelse - fatal("expected --color=[auto|on|off]; found {q}", .{arg}); - - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--cache-poison")) { - cache_poison = .poisoned; - configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); - continue; - } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { - // Allow the configurer process to report parse failure. - if (stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { - cache_poison = poison; - } - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--verbose")) { - // Intentionally is added both to make and configure but - // does not go into the cache hash. - configure_argv.appendAssumeCapacity(arg); - } else if (mem.eql(u8, arg, "--search-prefix")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - // This argument is cache poisonous: it does not go into - // the cache and configurer must set the poison bit when - // choosing to observe it. - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, args[i] }; - (try make_argv.addManyAsArray(arena, 2)).* = .{ arg, args[i] }; - continue; - } else if (mem.eql(u8, arg, "--build-file")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - build_file = args[i]; - continue; - } else if (mem.eql(u8, arg, "--zig-lib-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_lib_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--pkg-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_pkg_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--print-configuration-path")) { - print_configuration_path = true; - continue; - } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.eql(u8, arg, "--fetch")) { - fetch_only = true; - } else if (mem.cutPrefix(u8, arg, "--fetch=")) |sub_arg| { - fetch_only = true; - fetch_mode = stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse - fatal("expected [needed|all] after \"--fetch=\", found: {s}", .{sub_arg}); - } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| { - try forks.append(arena, .init(sub_arg)); - continue; - } else if (mem.eql(u8, arg, "--fork")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - try forks.append(arena, .init(args[i])); - continue; - } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - fatal("unable to parse reference_trace count {q}: {t}", .{ num, err }); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; - } else if (mem.cutPrefix(u8, arg, "--maker-opt=")) |rest| { - maker_optimize_mode = parseOptimizeMode(rest); - continue; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - try make_argv.appendSlice(arena, args[i .. i + 2]); - i += 1; - try addDebugLog(arena, args[i]); - continue; - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - if (build_options.enable_debug_extensions) { - debug_compile_errors = true; - } else { - warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{}); - } - } else if (mem.eql(u8, arg, "--debug-target")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_target = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{}); - } - continue; - } else if (mem.eql(u8, arg, "--debug-libc")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_libc_paths_file = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-libc has no effect.", .{}); - } - continue; - } else if (mem.eql(u8, arg, "--verbose-link")) { - verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-intern-pool")) { - verbose_intern_pool = true; - } else if (mem.eql(u8, arg, "--verbose-generic-instances")) { - verbose_generic_instances = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - verbose_llvm_ir = "-"; - } else if (mem.cutPrefix(u8, arg, "--verbose-llvm-ir=")) |rest| { - verbose_llvm_ir = rest; - } else if (mem.cutPrefix(u8, arg, "--verbose-llvm-bc=")) |rest| { - verbose_llvm_bc = rest; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - verbose_llvm_cpu_features = true; - } else if (mem.cutPrefix(u8, arg, "-j")) |str| { - const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| - fatal("unable to parse jobs count {s}: {t}", .{ str, err }); - if (num < 1) { - fatal("number of jobs must be at least 1", .{}); - } - n_jobs = num; - } else if (mem.eql(u8, arg, "--seed")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - make_argv.items[argv_index_seed] = args[i]; - continue; - } else if (mem.eql(u8, arg, "--")) { - try make_argv.appendSlice(arena, args[i..]); - break; - } - } - try make_argv.append(arena, arg); - } - } - - const root_prog_node = std.Progress.start(io, .{ - .disable_printing = (color == .off), - .root_name = "", - }); - defer root_prog_node.end(); - - process.raiseFileDescriptorLimit(); - - const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| - fatal("failed to get current directory path: {t}", .{err}); - - const build_root = try findBuildRoot(arena, io, .{ - .cwd_path = cwd_path, - .build_file = build_file, - }); - - { - // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( - arena, - io, - override_lib_dir, - override_global_cache_dir, - .{ .override = path: { - if (override_local_cache_dir) |d| break :path d; - break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); - } }, - .empty, - self_exe_path, - environ_map, - cwd_path, - ); - defer dirs.deinit(io); - - const thread_limit = @min( - @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), - std.math.maxInt(Zcu.PerThread.IdBacking), - ); - try setThreadLimit(arena, thread_limit); - - // Cache lookup for configure options. If we get a match, we can skip - // execution of the configure script. If not, we get the file path to pass - // to the configure process. - var local_cache: Cache = .{ - .gpa = gpa, - .io = io, - .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), - .cwd = cwd_path, - }; - local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); - local_cache.addPrefix(dirs.zig_lib); - local_cache.addPrefix(dirs.local_cache); - local_cache.addPrefix(dirs.global_cache); - defer local_cache.manifest_dir.close(io); - - var config_man = local_cache.obtain(); - defer config_man.deinit(); - config_man.hash.addBytes(build_options.version); - - for (cached_passthru_configure.items) |i| - config_man.hash.addBytes(configure_argv.items[i]); - - // Prevents a `zig build` from getting a false positive cache hit following - // a `zig build --cache-poison=ignored`. - config_man.hash.add(cache_poison == .ignored); - - // Normally the build runner is compiled for the host target but here is - // some code to help when debugging edits to the build runner so that you - // can make sure it compiles successfully on other targets. - const resolved_target: Package.Module.ResolvedTarget = t: { - if (build_options.enable_debug_extensions) { - if (debug_target) |triple| { - const target_query = try std.Target.Query.parse(.{ - .arch_os_abi = triple, - }); - config_man.hash.addBytes(triple); - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, target_query), - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - }; - } - } - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, .{}), - .is_native_os = true, - .is_native_abi = true, - .is_explicit_dynamic_linker = false, - }; - }; - - // Likewise, `--debug-libc` allows overriding the libc installation. - const libc_installation: ?*const LibCInstallation = lci: { - const paths_file = debug_libc_paths_file orelse break :lci null; - if (!build_options.enable_debug_extensions) unreachable; - const lci = try arena.create(LibCInstallation); - lci.* = try .parse(arena, io, paths_file, &resolved_target.result); - LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); - break :lci lci; - }; - - // Kick off an optimized compilation of the make runner. - var make_runner_task = if (print_configuration_path) undefined else io.async(compileMakeRunner, .{ gpa, arena, io, .{ - .dirs = .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }, - .environ_map = environ_map, - .parent_prog_node = root_prog_node, - .resolved_target = resolved_target, - .libc_installation = libc_installation, - .thread_limit = thread_limit, - .self_exe_path = self_exe_path, - .color = color, - .reference_trace = reference_trace, - .optimize_mode = maker_optimize_mode, - } }); - defer _ = if (!print_configuration_path) make_runner_task.cancel(io) catch {}; - - const pkg_root: Path = if (override_pkg_dir) |p| - .initCwd(p) - else if (system_pkg_dir_path) |p| - .initCwd(p) - else - .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }; - - make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; - make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - - configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; - - // Dummy http client that is not actually used when fetch_command is unsupported. - // Prevents bootstrap from depending on a bunch of unnecessary stuff. - var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { - allocator: Allocator, - io: Io, - fn deinit(_: @This()) void {} - } = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; - var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; - - { - // Populate fork_set. - var group: Io.Group = .init; - defer group.cancel(io); - - for (forks.items) |*fork| - group.async(io, Fork.load, .{ io, gpa, fork, color }); - - try group.await(io); - - for (forks.items) |*fork| { - if (fork.failed) process.exit(1); - try fork_set.put(arena, .{ - .path = fork.path, - .manifest_ast = fork.manifest_ast, - .manifest = fork.manifest, - .uses = 0, - }, {}); - } - } - defer Fork.deinitList(forks.items); - - var file_system_inputs: std.ArrayList(u8) = .empty; - defer file_system_inputs.deinit(gpa); - - // This loop is re-evaluated when the build script exits with an indication that it - // could not continue due to missing lazy dependencies. - const configuration_path: Path, const poisoned: bool = cp: while (true) { - // We want to release all the locks before executing the child process, so we make a nice - // big block here to ensure the cleanup gets run when we extract out our argv. - { - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "configurer.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .resolved_target = resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = resolved_target, - .single_threaded = true, - }, - .global = config, - .parent = null, - }); - - const build_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}), - .root_src_path = build_root.build_zig_basename, - }, - .fully_qualified_name = "root.@build", - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - - if (dev.env.supports(.fetch_command)) { - const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); - defer fetch_prog_node.end(); - - // Reset fork match counts. - for (fork_set.keys()) |*fork| fork.uses = 0; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = dirs.global_cache, - .local_storage = &.{ - .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" }, - .pkg_root = pkg_root, - }, - .recursive = true, - .debug_hash = false, - .unlazy_set = unlazy_set, - .fork_set = fork_set, - .mode = fetch_mode, - .prog_node = fetch_prog_node, - .read_only = system_pkg_dir_path != null, - }; - defer job_queue.deinit(); - - if (system_pkg_dir_path == null) { - try http_client.initDefaultProxies(arena, environ_map); - } - - try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); - try job_queue.table.ensureUnusedCapacity(gpa, 1); - - const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = phantom_package_root }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = phantom_package_root, - .parent_package_root = phantom_package_root, - .parent_manifest_ast = null, - .prog_node = fetch_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = true, - .oom_flag = false, - .latest_commit = null, - - .module = build_mod, - }; - - job_queue.all_fetches.appendAssumeCapacity(&fetch); - - job_queue.table.putAssumeCapacityNoClobber( - Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache), - &fetch, - ); - - job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); - try job_queue.group.await(io); - - { - // Ensure that forks were actually used. This is done - // before printing manifest errors because using a fork can - // prevent them. - var any_unused = false; - for (fork_set.keys()) |*fork| { - if (fork.uses == 0) { - std.log.err("fork {f} matched no {s} packages", .{ - fork.path, fork.manifest.name, - }); - any_unused = true; - } else { - std.log.info("fork {f} matched {d} {s} packages", .{ - fork.path, fork.uses, fork.manifest.name, - }); - } - } - if (any_unused) process.exit(1); - } - - try job_queue.consolidateErrors(); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } - - if (fetch_only) return cleanExit(io); - - var source_buf = std.array_list.Managed(u8).init(gpa); - defer source_buf.deinit(); - try job_queue.createDependenciesSource(&source_buf); - const deps_mod = try createDependenciesModule( - arena, - io, - source_buf.items, - root_mod, - dirs, - config, - ); - - { - // We need a Module for each package's build.zig. - const hashes = job_queue.table.keys(); - const fetches = job_queue.table.values(); - try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); - for (hashes, fetches) |*hash, f| { - if (f == &fetch) { - // The first one is a dummy package for the current project. - continue; - } - if (!f.has_build_zig) - continue; - const hash_slice = hash.toSlice(); - const mod_root_path = try f.package_root.toString(arena); - const m = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{mod_root_path}), - .root_src_path = Package.build_zig_basename, - }, - .fully_qualified_name = try std.fmt.allocPrint( - arena, - "root.@dependencies.{s}", - .{hash_slice}, - ), - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - const hash_cloned = try arena.dupe(u8, hash_slice); - deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); - f.module = m; - } - - // Each build.zig module needs access to each of its - // dependencies' build.zig modules by name. - for (fetches) |f| { - const mod = f.module orelse continue; - if (!f.have_manifest) continue; - const man = &f.manifest; - const dep_names = man.dependencies.keys(); - try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); - for (dep_names, man.dependencies.values()) |name, dep| { - const dep_digest = Package.Fetch.depDigest( - f.package_root, - dirs.global_cache, - dep, - ) orelse continue; - const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; - const name_cloned = try arena.dupe(u8, name); - mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); - } - } - } - } else try createEmptyDependenciesModule( - arena, - io, - root_mod, - dirs, - config, - ); - - const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); - defer compile_prog_node.end(); - - try root_mod.deps.put(arena, "@build", build_mod); - - file_system_inputs.clearRetainingCapacity(); - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .libc_installation = libc_installation, - .dirs = dirs, - .root_name = "configure", - .config = config, - .root_mod = root_mod, - .main_mod = build_mod, - .emit_bin = .yes_cache, - .self_exe_path = self_exe_path, - .thread_limit = thread_limit, - .verbose_cc = verbose_cc, - .verbose_link = verbose_link, - .verbose_air = verbose_air, - .verbose_intern_pool = verbose_intern_pool, - .verbose_generic_instances = verbose_generic_instances, - .verbose_llvm_ir = verbose_llvm_ir, - .verbose_llvm_bc = verbose_llvm_bc, - .verbose_llvm_cpu_features = verbose_llvm_cpu_features, - .cache_mode = .whole, - .reference_trace = reference_trace, - .debug_compile_errors = debug_compile_errors, - .environ_map = environ_map, - .file_system_inputs = &file_system_inputs, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - updateModule(comp, color, compile_prog_node) catch |err| switch (err) { - error.CompileErrorsReported => process.exit(2), - else => |e| return e, - }; - - // Since incremental compilation isn't done yet, we use cache_mode = whole - // above, and thus the output file is already closed. - //try comp.makeBinFileExecutable(); - const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); - const exe_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), - }; - _ = try config_man.addFilePath(exe_path, null); - configure_argv.items[0] = try exe_path.toString(arena); - - switch (cache_poison) { - .pure, .disallowed, .ignored => if (try config_man.hit()) { - const digest = config_man.final(); - break :cp .{ - .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), - }, - false, - }; - }, - .poisoned => {}, // Don't bother checking for cache hit. - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); - } - - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - const config_tmp_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( - io, - config_tmp_path.sub_path, - .{ .read = true, .exclusive = true }, - ); - defer config_tmp_file.close(io); - - const term = term: { - const child_node = root_prog_node.start("Run Configure Script", 0); - defer child_node.end(); - var child = std.process.spawn(io, .{ - .argv = configure_argv.items, - .stdout = .{ .file = config_tmp_file }, - .progress_node = child_node, - }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err }); - }; - if (!term.success()) { - // Failure to produce the configuration file. - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following configure command {f}:\n{s}", .{ term, cmd }); - } - // Even though the file is designed to be sent directly to make - // runner, we must load it now because: - // * If it contains additional file dependencies, we need to - // add them to `config_man` before obtaining the final digest. - // * If it contains a set of lazy packages that need to be - // fetched, we need to fetch those now and re-run configure. - var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - - if (configuration.unlazy_deps.len != 0) { - if (!dev.env.supports(.fetch_command)) process.exit(1); - var any_errors = false; - for (configuration.unlazy_deps) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); - any_errors = true; - continue; - } - try unlazy_set.put(arena, .fromSlice(hash), {}); - } - if (any_errors) process.exit(1); - if (system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. - const s = fs.path.sep_str; - for (unlazy_set.keys()) |*hash| { - std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); - } - std.log.info("remote package fetching disabled due to --system mode", .{}); - std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(1); - } - continue :cp; - } - - for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; - try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); - } - - // We need to add to the configuration cache the source files of - // configurer itself, so that the maker process can watch the file system - // for those changes and restart itself. By doing this, we make it - // possible to bypass creating a Compilation for configurer on - // Configuration cache hit. - { - var it = mem.splitScalar(u8, file_system_inputs.items, 0); - while (it.next()) |input| { - _ = try config_man.addPrefixedPathPost(.{ - .prefix = input[0], - .sub_path = input[1..], - }); - } - } - - // If it is poisoned, there is no point in moving it to cached - // location. Just leave it in the tmp directory. - if (configuration.poisoned) { - break :cp .{ config_tmp_path, true }; - } else { - const digest = config_man.final(); - const final_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), - }; - Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - ) catch |err| retry: { - const e = switch (err) { - error.FileNotFound => e: { - const dir_path = final_path.dirname().?; - dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| - fatal("failed to create directory {f}: {t}", .{ dir_path, e }); - if (Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - )) |_| break :retry else |e| break :e e; - }, - else => |e| e, - }; - fatal("failed to rename configuration file from {f} into {f}: {t}", .{ - config_tmp_path, final_path, e, - }); - }; - config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); - break :cp .{ final_path, false }; - } - }; - - { - // Release all file system locks just before running the maker process. - var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; - defer if (configuration_lock) |*l| l.release(io); - - if (print_configuration_path) { - var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); - stdout_writer.interface.print("{f}\n", .{configuration_path}) catch - fatal("failed printing cache file path: {t}", .{stdout_writer.err.?}); - stdout_writer.flush() catch |err| - fatal("failed printing cache file path: {t}", .{err}); - return cleanExit(io); - } - const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); - - make_argv.items[0] = try make_runner.exe_path.toString(arena); - make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ - native_os, cmd, - }); - } - - const term = term: { - _ = try io.lockStderr(&.{}, .no_color); - defer io.unlockStderr(); - var child = std.process.spawn(io, .{ - .argv = make_argv.items, - }) catch |err| fatal("failed spawning maker {s}: {t}", .{ make_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed waiting on maker {s}: {t}", .{ make_argv.items[0], err }); - }; - if (term.success()) return cleanExit(io); - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following maker command {f}:\n{s}", .{ term, cmd }); -} - -const MakeRunner = struct { - exe_path: Path, - - const Options = struct { - environ_map: *const process.Environ.Map, - dirs: Compilation.Directories, - parent_prog_node: std.Progress.Node, - resolved_target: Package.Module.ResolvedTarget, - libc_installation: ?*const LibCInstallation, - self_exe_path: []const u8, - thread_limit: usize, - color: Color, - reference_trace: ?u32, - optimize_mode: std.builtin.OptimizeMode, - }; -}; - -fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner { - const compile_prog_node = options.parent_prog_node.start("Compiling Maker (first time setup)", 0); - defer compile_prog_node.end(); - - const strip = options.optimize_mode != .Debug; - - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), - .root_src_path = "Maker.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .root_strip = strip, - .root_optimize_mode = options.optimize_mode, - .resolved_target = options.resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = options.resolved_target, - .optimize_mode = options.optimize_mode, - .strip = strip, - }, - .global = config, - .parent = null, - }); - - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .dirs = options.dirs, - .root_name = "maker", - .config = config, - .root_mod = root_mod, - .main_mod = root_mod, - .emit_bin = .yes_cache, - .self_exe_path = options.self_exe_path, - .thread_limit = options.thread_limit, - .cache_mode = .whole, - .environ_map = options.environ_map, - .reference_trace = options.reference_trace, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - error.Canceled => |e| return e, - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - try updateModule(comp, options.color, compile_prog_node); - - const exe_path: Path = .{ - .root_dir = options.dirs.global_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ - &Cache.binToHex(comp.digest.?), comp.emit_bin.?, - }), - }; - - return .{ - .exe_path = exe_path, - }; -} - -const Fork = struct { - path: Path, - manifest_ast: std.zig.Ast, - manifest: Package.Manifest, - error_bundle: std.zig.ErrorBundle.Wip, - failed: bool, - arena_allocator: std.heap.ArenaAllocator, - - fn init(cwd_relative_path: []const u8) Fork { - return .{ - .manifest_ast = undefined, - .manifest = undefined, - .error_bundle = undefined, - .arena_allocator = undefined, - .path = .{ - .root_dir = .cwd(), - .sub_path = cwd_relative_path, - }, - .failed = false, - }; - } - - fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { - loadFallible(io, gpa, fork, color) catch |err| switch (err) { - error.Canceled => |e| return e, - error.AlreadyReported => fork.failed = true, - else => |e| { - std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); - fork.failed = true; - }, - }; - } - - fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void { - fork.arena_allocator = .init(gpa); - const arena = fork.arena_allocator.allocator(); - - var error_bundle: std.zig.ErrorBundle.Wip = undefined; - try error_bundle.init(gpa); - defer error_bundle.deinit(); - - const manifest_path = try fork.path.join(arena, Package.Manifest.basename); - - Package.Manifest.load( - io, - arena, - manifest_path, - &fork.manifest_ast, - &error_bundle, - &fork.manifest, - true, - ) catch |err| switch (err) { - error.Canceled => |e| return e, - error.ErrorsBundled => { - assert(error_bundle.root_list.items.len > 0); - var errors = try error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - return error.AlreadyReported; - }, - else => |e| { - std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); - return error.AlreadyReported; - }, - }; - } - - fn deinitList(forks: []Fork) void { - for (forks) |*fork| fork.arena_allocator.deinit(); - } -}; - const JitCmdOptions = struct { cmd_name: []const u8, root_src_path: []const u8, prepend_zig_lib_dir_path: bool = false, prepend_global_cache_path: bool = false, prepend_zig_exe_path: bool = false, + prepend_seed: bool = false, depend_on_aro: bool = false, capture: ?*[]u8 = null, /// Send error bundles via std.zig.Server over stdout server: bool = false, - color: Color = .auto, }; fn jitCmd( @@ -6098,8 +5046,10 @@ fn jitCmd( ) !void { dev.check(.jit_command); + const color = Color.settingFromEnvironment(environ_map); + const root_prog_node = std.Progress.start(io, .{ - .disable_printing = (options.color == .off), + .disable_printing = (color == .off), }); defer root_prog_node.end(); @@ -6141,7 +5091,7 @@ fn jitCmdInner( const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( @@ -6158,7 +5108,7 @@ fn jitCmdInner( defer dirs.deinit(io); var child_argv: std.ArrayList([]const u8) = .empty; - try child_argv.ensureUnusedCapacity(arena, args.len + 4); + try child_argv.ensureUnusedCapacity(arena, args.len + 5); // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. @@ -6244,7 +5194,8 @@ fn jitCmdInner( process.exit(2); } } else { - updateModule(comp, options.color, root_prog_node) catch |err| switch (err) { + const color = Color.settingFromEnvironment(environ_map); + updateModule(comp, color, root_prog_node) catch |err| switch (err) { error.CompileErrorsReported => process.exit(2), else => |e| return e, }; @@ -6259,11 +5210,13 @@ fn jitCmdInner( } if (options.prepend_zig_lib_dir_path) - child_argv.appendAssumeCapacity(dirs.zig_lib.path.?); + child_argv.appendAssumeCapacity(try allocPrint(arena, "--zig-lib={s}", .{dirs.zig_lib.path.?})); if (options.prepend_zig_exe_path) - child_argv.appendAssumeCapacity(self_exe_path); + child_argv.appendAssumeCapacity(try allocPrint(arena, "--zig={s}", .{self_exe_path})); if (options.prepend_global_cache_path) - child_argv.appendAssumeCapacity(dirs.global_cache.path.?); + child_argv.appendAssumeCapacity(try allocPrint(arena, "--global-cache={s}", .{dirs.global_cache.path.?})); + if (options.prepend_seed) + child_argv.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)})); child_argv.appendSliceAssumeCapacity(args); @@ -7199,567 +6152,6 @@ fn parseRcIncludes(arg: []const u8) std.zig.RcIncludes { fatal("unsupported rc includes type: {q}", .{arg}); } -const usage_fetch = - \\Usage: zig fetch [options] - \\Usage: zig fetch [options] - \\ - \\ Copy a package into the global cache and print its hash. - \\ must point to one of the following: - \\ - A git+http / git+https server for the package - \\ - A tarball file (with or without compression) containing - \\ package source - \\ - A git bundle file containing package source - \\ - \\Examples: - \\ - \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git - \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz - \\ - \\Options: - \\ -h, --help Print this help and exit - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --cache-dir [path] Override path to local cache directory - \\ --pkg-dir [path] Override path to local package directory - \\ --debug-hash Print verbose hash information to stdout - \\ --debug-log [scope] Enable printing debug/info log messages for scope - \\ --save Add the fetched package to build.zig.zon - \\ --save=[name] Add the fetched package to build.zig.zon as name - \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim - \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim - \\ -; - -fn cmdFetch( - gpa: Allocator, - arena: Allocator, - io: Io, - args: []const []const u8, - environ_map: *process.Environ.Map, -) !void { - dev.check(.fetch_command); - - const color: Color = Color.settingFromEnvironment(environ_map); - var opt_path_or_url: ?[]const u8 = null; - var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var debug_hash: bool = false; - var save: union(enum) { - no, - yes: ?[]const u8, - exact: ?[]const u8, - } = .no; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - try Io.File.stdout().writeStreamingAll(io, usage_fetch); - return cleanExit(io); - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--pkg-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_pkg_dir = args[i]; - } else if (mem.eql(u8, arg, "--debug-hash")) { - debug_hash = true; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - try addDebugLog(arena, args[i]); - } else if (mem.eql(u8, arg, "--save")) { - save = .{ .yes = null }; - } else if (mem.cutPrefix(u8, arg, "--save=")) |rest| { - save = .{ .yes = rest }; - } else if (mem.eql(u8, arg, "--save-exact")) { - save = .{ .exact = null }; - } else if (mem.cutPrefix(u8, arg, "--save-exact=")) |rest| { - save = .{ .exact = rest }; - } else { - fatal("unrecognized parameter: {q}", .{arg}); - } - } else if (opt_path_or_url != null) { - fatal("unexpected extra parameter: {q}", .{arg}); - } else { - opt_path_or_url = arg; - } - } - } - - const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); - - var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - try http_client.initDefaultProxies(arena, environ_map); - - var root_prog_node = std.Progress.start(io, .{ - .root_name = "Fetch", - }); - defer root_prog_node.end(); - - var global_cache_directory: Directory = l: { - const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena, environ_map); - break :l .{ - .handle = try Io.Dir.cwd().createDirPathOpen(io, p, .{}), - .path = p, - }; - }; - defer global_cache_directory.handle.close(io); - - var local_storage: Package.Fetch.LocalStorage = undefined; - var build_root: BuildRoot = undefined; - var build_root_initialized = false; - defer if (build_root_initialized) build_root.deinit(io); - - const cwd_path = try introspect.getResolvedCwd(io, arena); - - const local_storage_ptr = switch (save) { - .no => null, - .yes, .exact => ls: { - build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path }); - build_root_initialized = true; - - local_storage = .{ - .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{ - .root_dir = build_root.directory, - .sub_path = ".zig-cache", - }, - .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }, - }; - - break :ls &local_storage; - }, - }; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = global_cache_directory, - .local_storage = local_storage_ptr, - .recursive = false, - .read_only = false, - .debug_hash = debug_hash, - .mode = .all, - .prog_node = root_prog_node, - }; - defer job_queue.deinit(); - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .path_or_url = path_or_url }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = undefined, - .parent_package_root = undefined, - .parent_manifest_ast = null, - .prog_node = root_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = true, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - defer fetch.deinit(); - - fetch.run() catch |err| switch (err) { - error.OutOfMemory, error.Canceled => |e| return e, - error.FetchFailed => {}, // error bundle checked below - }; - - try job_queue.group.await(io); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } - - const package_hash = fetch.computedPackageHash(); - const package_hash_slice = package_hash.toSlice(); - - root_prog_node.end(); - root_prog_node = .{ .index = .none }; - - const name = switch (save) { - .no => { - var stdout = Io.File.stdout().writerStreaming(io, &stdout_buffer); - try stdout.interface.print("{s}\n", .{package_hash_slice}); - try stdout.interface.flush(); - return cleanExit(io); - }, - .yes, .exact => |name| name: { - if (name) |n| break :name n; - if (!fetch.have_manifest) - fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); - break :name fetch.manifest.name; - }, - }; - - // The name to use in case the manifest file needs to be created now. - const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path); - var manifest, var ast = try loadManifest(gpa, arena, io, .{ - .root_name = try sanitizeExampleName(arena, init_root_name), - .dir = build_root.directory.handle, - .color = color, - }); - defer { - manifest.deinit(gpa); - ast.deinit(gpa); - } - - var fixups: Ast.Render.Fixups = .{}; - defer fixups.deinit(gpa); - - var saved_path_or_url = path_or_url; - - if (fetch.latest_commit) |latest_commit| resolved: { - const latest_commit_hex = try std.fmt.allocPrint(arena, "{f}", .{latest_commit}); - - var uri = try std.Uri.parse(path_or_url); - - if (uri.fragment) |fragment| { - const target_ref = try fragment.toRawMaybeAlloc(arena); - - // the refspec may already be fully resolved - if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; - - std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex }); - - // include the original refspec in a query parameter, could be used to check for updates - uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={f}", .{ - std.fmt.alt(fragment, .formatEscaped), - }) }; - } else { - std.log.info("resolved to commit {s}", .{latest_commit_hex}); - } - - // replace the refspec with the resolved commit SHA - uri.fragment = .{ .raw = latest_commit_hex }; - - switch (save) { - .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{f}", .{uri}), - .no, .exact => {}, // keep the original URL - } - } - - const new_node_init = try std.fmt.allocPrint(arena, - \\.{{ - \\ .url = "{f}", - \\ .hash = "{f}", - \\ }} - , .{ - std.zig.fmtString(saved_path_or_url), - std.zig.fmtString(package_hash_slice), - }); - - const new_node_text = try std.fmt.allocPrint(arena, ".{f} = {s},\n", .{ - std.zig.fmtIdPU(name), new_node_init, - }); - - const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ - new_node_text, - }); - - const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ - dependencies_init, - }); - - if (manifest.dependencies.get(name)) |dep| { - if (dep.hash) |h| { - switch (dep.location) { - .url => |u| { - if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { - std.log.info("existing dependency named {q} is up-to-date", .{name}); - process.exit(0); - } - }, - .path => {}, - } - } - - const location_replace = try std.fmt.allocPrint( - arena, - "\"{f}\"", - .{std.zig.fmtString(saved_path_or_url)}, - ); - const hash_replace = try std.fmt.allocPrint( - arena, - "\"{f}\"", - .{std.zig.fmtString(package_hash_slice)}, - ); - - warn("overwriting existing dependency named {q}", .{name}); - try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); - if (dep.hash_node.unwrap()) |hash_node| { - try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); - } else { - // https://github.com/ziglang/zig/issues/21690 - } - } else if (manifest.dependencies.count() > 0) { - // Add fixup for adding another dependency. - const deps = manifest.dependencies.values(); - const last_dep_node = deps[deps.len - 1].node; - try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); - } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { - // Add fixup for replacing the entire dependencies struct. - try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); - } else { - // Add fixup for adding dependencies struct. - try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); - } - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - try ast.render(gpa, &aw.writer, fixups); - const rendered = aw.written(); - - build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch |err| { - fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err }); - }; - - return cleanExit(io); -} - -fn createEmptyDependenciesModule( - arena: Allocator, - io: Io, - main_mod: *Package.Module, - dirs: Compilation.Directories, - global_options: Compilation.Config, -) !void { - var source = std.array_list.Managed(u8).init(arena); - try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); - _ = try createDependenciesModule( - arena, - io, - source.items, - main_mod, - dirs, - global_options, - ); -} - -/// Creates the dependencies.zig file and corresponding `Package.Module` for the -/// build runner to obtain via `@import("@dependencies")`. -fn createDependenciesModule( - arena: Allocator, - io: Io, - source: []const u8, - main_mod: *Package.Module, - dirs: Compilation.Directories, - global_options: Compilation.Config, -) !*Package.Module { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - { - var tmp_dir = try dirs.local_cache.handle.createDirPathOpen(io, tmp_dir_sub_path, .{}); - defer tmp_dir.close(io); - try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source }); - } - const tmp_dir_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(source); - const hex_digest = hh.final(); - - const o_dir_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest), - }; - try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path); - - const deps_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path), - .root_src_path = basename, - }, - .fully_qualified_name = "root.@dependencies", - .parent = main_mod, - .cc_argv = &.{}, - .inherited = .{}, - .global = global_options, - }); - try main_mod.deps.put(arena, "@dependencies", deps_mod); - return deps_mod; -} - -const BuildRoot = struct { - directory: Cache.Directory, - build_zig_basename: []const u8, - cleanup_build_dir: ?Io.Dir, - - fn deinit(br: *BuildRoot, io: Io) void { - if (br.cleanup_build_dir) |*dir| dir.close(io); - br.* = undefined; - } -}; - -const FindBuildRootOptions = struct { - build_file: ?[]const u8 = null, - cwd_path: ?[]const u8 = null, -}; - -fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot { - const cwd_path = options.cwd_path orelse try introspect.getResolvedCwd(io, arena); - const build_zig_basename = if (options.build_file) |bf| - fs.path.basename(bf) - else - Package.build_zig_basename; - - if (options.build_file) |bf| { - if (fs.path.dirname(bf)) |dirname| { - const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { - fatal("unable to open directory to build file from argument 'build-file', {q}: {t}", .{ dirname, err }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = dirname, .handle = dir }, - .cleanup_build_dir = dir, - }; - } - - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = null, .handle = Io.Dir.cwd() }, - .cleanup_build_dir = null, - }; - } - // Search up parent directories until we find build.zig. - var dirname: []const u8 = cwd_path; - while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); - if (Io.Dir.cwd().access(io, joined_path, .{})) |_| { - const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { - fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ - .path = dirname, - .handle = dir, - }, - .cleanup_build_dir = dir, - }; - } else |err| switch (err) { - error.FileNotFound => { - dirname = fs.path.dirname(dirname) orelse { - std.log.info("initialize {s} template file with 'zig init'", .{ - Package.build_zig_basename, - }); - std.log.info("see 'zig --help' for more options", .{}); - fatal("no build.zig file found, in the current directory or any parent directories", .{}); - }; - continue; - }, - else => |e| return e, - } - } -} - -const LoadManifestOptions = struct { - root_name: []const u8, - dir: Io.Dir, - color: Color, -}; - -fn loadManifest( - gpa: Allocator, - arena: Allocator, - io: Io, - options: LoadManifestOptions, -) !struct { Package.Manifest, Ast } { - const rng: std.Random.IoSource = .{ .io = io }; - - const manifest_bytes = while (true) { - break options.dir.readFileAllocOptions( - io, - Package.Manifest.basename, - arena, - .limited(Package.Manifest.max_bytes), - .@"1", - 0, - ) catch |err| switch (err) { - error.FileNotFound => { - writeSimpleTemplateFile(io, Package.Manifest.basename, - \\.{{ - \\ .name = .{s}, - \\ .version = "{s}", - \\ .paths = .{{""}}, - \\ .fingerprint = 0x{x}, - \\}} - \\ - , .{ - options.root_name, - build_options.version, - Package.Fingerprint.generate(rng.interface(), options.root_name).int(), - }) catch |e| { - fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e }); - }; - continue; - }, - else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }), - }; - }; - var ast = try Ast.parse(gpa, manifest_bytes, .zon); - errdefer ast.deinit(gpa); - - if (ast.errors.len > 0) { - try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color); - process.exit(2); - } - - var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); - errdefer manifest.deinit(gpa); - - if (manifest.errors.len > 0) { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - const src_path = try wip_errors.addString(Package.Manifest.basename); - try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); - - var error_bundle = try wip_errors.toOwnedBundle(""); - defer error_bundle.deinit(gpa); - error_bundle.renderToStderr(io, .{}, options.color) catch {}; - - process.exit(2); - } - return .{ manifest, ast }; -} - const Templates = struct { zig_lib_directory: Cache.Directory, dir: Io.Dir, @@ -7834,13 +6226,13 @@ fn writeSimpleTemplateFile(io: Io, file_name: []const u8, comptime fmt: []const } fn findTemplates(gpa: Allocator, arena: Allocator, io: Io) Templates { - const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| { + const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| { fatal("unable to get cwd: {t}", .{err}); }; const self_exe_path = process.executablePathAlloc(io, arena) catch |err| { fatal("unable to find self exe path: {t}", .{err}); }; - var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch |err| { + var zig_lib_directory = std.zig.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch |err| { fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err }); }; diff --git a/src/print_env.zig b/src/print_env.zig index 9370006c72df78adc2725178d2644ca44fbbfcae..93e14781a184db46288b9c5e8c3d77e598820f2c 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -8,7 +8,6 @@ const fatal = std.process.fatal; const build_options = @import("build_options"); const Compilation = @import("Compilation.zig"); -const introspect = @import("introspect.zig"); pub fn cmdEnv( arena: Allocator, @@ -29,7 +28,7 @@ pub fn cmdEnv( }, }; - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); var dirs: Compilation.Directories = .init( arena, diff --git a/src/print_targets.zig b/src/print_targets.zig index c3a9ff44584ee2eb304cb63531fbda56ee1c98a1..702a684de3e829b8e7d475c265a5ed2e584caa1e 100644 --- a/src/print_targets.zig +++ b/src/print_targets.zig @@ -9,7 +9,6 @@ const Target = std.Target; const assert = std.debug.assert; const glibc = @import("libs/glibc.zig"); -const introspect = @import("introspect.zig"); const target = @import("target.zig"); pub fn cmdTargets( @@ -20,7 +19,7 @@ pub fn cmdTargets( native_target: *const Target, ) !void { _ = args; - var zig_lib_directory = introspect.findZigLibDir(allocator, io) catch |err| + var zig_lib_directory = std.zig.findZigLibDir(allocator, io) catch |err| fatal("unable to find zig installation directory: {t}", .{err}); defer zig_lib_directory.handle.close(io); defer allocator.free(zig_lib_directory.path.?);