diff --git a/CMakeLists.txt b/CMakeLists.txt index ea25212fec73425108866d29e0acdd61c56f18c6..e98384f8af1a2604d6e9bfdfb8adeb4793d9b1d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,11 +513,6 @@ set(ZIG_STAGE2_SOURCES src/InternPool.zig src/Liveness.zig src/Liveness/Verify.zig - src/Package.zig - src/Package/Fetch.zig - src/Package/Fetch/git.zig - src/Package/Manifest.zig - src/Package/Module.zig src/RangeSet.zig src/Sema.zig src/Sema/bitcast.zig diff --git a/lib/compiler/build.zig b/lib/compiler/build.zig new file mode 100644 index 0000000000000000000000000000000000000000..0611bb06cfe93d2d9810b6e37c32ec2e43b8d64b --- /dev/null +++ b/lib/compiler/build.zig @@ -0,0 +1,1790 @@ +const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("std"); +const assert = std.debug.assert; +const io = std.io; +const fmt = std.fmt; +const mem = std.mem; +const process = std.process; +const ArrayList = std.ArrayList; +const File = std.fs.File; +const Step = std.Build.Step; +const Watch = std.Build.Watch; +const Fuzz = std.Build.Fuzz; +const Allocator = std.mem.Allocator; +const fatal = std.process.fatal; +const Directory = std.Build.Cache.Directory; +const Package = std.zig.Package; + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .crypto_fork_safety = false, +}; + +pub fn main() !void { + // Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived, + // one shot program. We don't need to waste time freeing memory and finding places to squish + // bytes into. So we free everything all at once at the very end. + var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer single_threaded_arena.deinit(); + + var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ + .child_allocator = single_threaded_arena.allocator(), + }; + const arena = thread_safe_arena.allocator(); + const gpa = arena; + + const args = try process.argsAlloc(arena); + + // skip my own exe name + var arg_idx: usize = 1; + + const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); + const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); + const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); + const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + + const zig_lib_directory: Directory = .{ + .path = zig_lib_dir, + .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), + }; + + const local_cache_directory: Directory = .{ + .path = cache_root, + .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), + }; + + const global_cache_directory: Directory = .{ + .path = global_cache_root, + .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), + }; + + var graph: std.Build.Graph = .{ + .arena = arena, + .cache = .{ + .gpa = gpa, + .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), + }, + .zig_exe = zig_exe, + .env_map = try process.getEnvMap(arena), + .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + .host = .{ + .query = .{}, + .result = try std.zig.system.resolveTargetQuery(.{}), + }, + }; + + var targets = ArrayList([]const u8).init(arena); + var debug_log_scopes = ArrayList([]const u8).init(arena); + var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = gpa }; + var options_args: std.ArrayListUnmanaged([]const u8) = .empty; + + var install_prefix: ?[]const u8 = null; + var install_paths: std.Build.InstallPaths = .{}; + var summary: ?Summary = null; + var max_rss: u64 = 0; + var skip_oom_steps = false; + var color: Color = .auto; + var prominent_compile_errors = false; + var help_menu = false; + var steps_menu = false; + var watch = false; + var fuzz = false; + var debounce_interval_ms: u16 = 50; + var listen_port: u16 = 0; + var remaining_args: ?[]const []const u8 = null; + + var build_file: ?[]const u8 = null; + var reference_trace: ?u32 = null; + var debug_compile_errors = false; + var verbose_link = (native_os != .wasi or builtin.link_libc) and std.zig.EnvVar.ZIG_VERBOSE_LINK.isSet(); + var verbose_cc = (native_os != .wasi or builtin.link_libc) and std.zig.EnvVar.ZIG_VERBOSE_CC.isSet(); + var verbose_air = false; + var verbose_intern_pool = false; + var verbose_generic_instances = false; + var verbose_llvm_ir: ?[]const u8 = null; + var verbose_llvm_bc: ?[]const u8 = null; + var verbose_cimport = false; + var verbose_llvm_cpu_features = false; + var fetch_only = false; + + while (nextArg(args, &arg_idx)) |arg| { + if (mem.startsWith(u8, arg, "-D")) { + try options_args.append(arena, arg); + } else if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "--verbose")) { + graph.verbose = true; + } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + help_menu = true; + } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { + install_prefix = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { + steps_menu = true; + } else if (mem.startsWith(u8, arg, "-fsys=")) { + const name = arg["-fsys=".len..]; + graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); + } else if (mem.startsWith(u8, arg, "-fno-sys=")) { + const name = arg["-fno-sys=".len..]; + graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); + } else if (mem.eql(u8, arg, "--release")) { + graph.release_mode = .any; + } else if (mem.startsWith(u8, arg, "--release=")) { + const text = arg["--release=".len..]; + graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { + fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ + arg, text, + }); + }; + } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { + install_paths.lib_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { + install_paths.exe_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-include-dir")) { + install_paths.include_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--sysroot")) { + graph.sysroot = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--maxrss")) { + const max_rss_text = nextArgOrFatal(args, &arg_idx); + max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { + std.debug.print("invalid byte size: '{s}': {s}\n", .{ + max_rss_text, @errorName(err), + }); + process.exit(1); + }; + } else if (mem.eql(u8, arg, "--skip-oom-steps")) { + skip_oom_steps = true; + } else if (mem.eql(u8, arg, "--search-prefix")) { + const search_prefix = nextArgOrFatal(args, &arg_idx); + graph.addSearchPrefix(search_prefix); + } else if (mem.eql(u8, arg, "--libc")) { + graph.libc_file = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--build-file")) { + build_file = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--color")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + color = std.meta.stringToEnum(Color, next_arg) orelse { + fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--summary")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [all|new|failures|none] after '{s}'", .{arg}); + summary = std.meta.stringToEnum(Summary, next_arg) orelse { + fatalWithHint("expected [all|new|failures|none] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--seed")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u32 after '{s}'", .{arg}); + graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--debounce")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u16 after '{s}'", .{arg}); + debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { + fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--port")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u16 after '{s}'", .{arg}); + listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| { + fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--debug-log")) { + const next_arg = nextArgOrFatal(args, &arg_idx); + try debug_log_scopes.append(next_arg); + } else if (mem.eql(u8, arg, "--debug-pkg-config")) { + graph.debug_pkg_config = true; + } else if (mem.eql(u8, arg, "--debug-rt")) { + graph.debug_compiler_runtime_libs = true; + } else if (mem.eql(u8, arg, "--debug-compile-errors")) { + graph.debug_compile_errors = true; + } else if (mem.eql(u8, arg, "--system")) { + graph.system_package_mode = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--glibc-runtimes")) { + graph.glibc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--verbose-link")) { + graph.verbose_link = true; + } else if (mem.eql(u8, arg, "--verbose-air")) { + graph.verbose_air = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { + graph.verbose_llvm_ir = "-"; + } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { + graph.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; + } else if (mem.eql(u8, arg, "--verbose-llvm-bc=")) { + graph.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; + } else if (mem.eql(u8, arg, "--verbose-cimport")) { + graph.verbose_cimport = true; + } else if (mem.eql(u8, arg, "--verbose-cc")) { + graph.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { + graph.verbose_llvm_cpu_features = true; + } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { + prominent_compile_errors = true; + } else if (mem.eql(u8, arg, "--watch")) { + watch = true; + } else if (mem.eql(u8, arg, "--fuzz")) { + fuzz = true; + } else if (mem.eql(u8, arg, "--fetch")) { + fetch_only = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; + } else if (mem.eql(u8, arg, "-fwine")) { + graph.enable_wine = true; + } else if (mem.eql(u8, arg, "-fno-wine")) { + graph.enable_wine = false; + } else if (mem.eql(u8, arg, "-fqemu")) { + graph.enable_qemu = true; + } else if (mem.eql(u8, arg, "-fno-qemu")) { + graph.enable_qemu = false; + } else if (mem.eql(u8, arg, "-fwasmtime")) { + graph.enable_wasmtime = true; + } else if (mem.eql(u8, arg, "-fno-wasmtime")) { + graph.enable_wasmtime = false; + } else if (mem.eql(u8, arg, "-frosetta")) { + graph.enable_rosetta = true; + } else if (mem.eql(u8, arg, "-fno-rosetta")) { + graph.enable_rosetta = false; + } else if (mem.eql(u8, arg, "-fdarling")) { + graph.enable_darling = true; + } else if (mem.eql(u8, arg, "-fno-darling")) { + graph.enable_darling = false; + } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { + graph.allow_so_scripts = true; + } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { + graph.allow_so_scripts = false; + } else if (mem.eql(u8, arg, "-freference-trace")) { + graph.reference_trace = 256; + } else if (mem.startsWith(u8, arg, "-freference-trace=")) { + const num = arg["-freference-trace=".len..]; + graph.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); + process.exit(1); + }; + } else if (mem.eql(u8, arg, "-fno-reference-trace")) { + graph.reference_trace = null; + } else if (mem.startsWith(u8, arg, "-j")) { + const num = arg["-j".len..]; + const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse jobs count '{s}': {s}", .{ + num, @errorName(err), + }); + process.exit(1); + }; + if (n_jobs < 1) { + std.debug.print("number of jobs must be at least 1\n", .{}); + process.exit(1); + } + thread_pool_options.n_jobs = n_jobs; + } else if (mem.eql(u8, arg, "--")) { + remaining_args = argsRest(args, arg_idx); + break; + } else { + fatalWithHint("unrecognized argument: '{s}'", .{arg}); + } + } else { + try targets.append(arg); + } + } + graph.debug_log_scopes = debug_log_scopes.items; + + const cwd_path = try process.getCwdAlloc(arena); + const build_root = try Package.findBuildRoot(arena, .{ + .cwd_path = cwd_path, + .build_file = build_file, + }); + + graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); + graph.cache.addPrefix(build_root.directory); + graph.cache.addPrefix(local_cache_directory); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + + const stderr = std.io.getStdErr(); + const ttyconf = get_tty_conf(color, stderr); + switch (ttyconf) { + .no_color => try graph.env_map.put("NO_COLOR", "1"), + .escape_codes => try graph.env_map.put("CLICOLOR_FORCE", "1"), + .windows_api => {}, + } + + const main_progress_node = std.Progress.start(.{ + .disable_printing = (color == .off), + }); + defer main_progress_node.end(); + + var thread_pool: std.Thread.Pool = undefined; + try thread_pool.init(thread_pool_options); + defer thread_pool.deinit(); + + + { + var compile_argv: std.ArrayListUnmanaged([]const u8) = .empty; + defer compile_argv.deinit(gpa); + + var run_argv: std.ArrayListUnmanaged([]const u8) = .empty; + defer run_argv.deinit(gpa); + + var cli_modules: std.StringArrayHashMapUnmanaged(CliModule) = .empty; + defer cli_modules.deinit(gpa); + + const configure_runner_module = try std.fmt.allocPrint(arena, "-Mroot={s}/lib/configure_runner.zig", .{ + zig_lib_dir, + }); + const build_zig_module = try std.fmt.allocPrint(arena, "-M@build={}/{s}", .{ + build_root.directory, build_root.build_zig_basename, + }); + + const exe_basename = try std.zig.binNameAlloc(arena, .{ + .root_name = "configure", + .target = graph.host.result, + .output_mode = .Exe, + }); + var http_client: std.http.Client = .{ .allocator = gpa }; + defer http_client.deinit(); + + var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; + + // This loop is re-evaluated when the build script exits with an indication that it + // could not continue due to missing lazy dependencies. + while (true) { + // We want to release all the locks before executing the child process, so we make a nice + // big block here to ensure the cleanup gets run when we extract out our argv. + { + { + cli_modules.clearRetainingCapacity(); + const root_mod = try addCliModule(gpa, arena, &cli_modules, configure_runner_module); + const build_mod = try addCliModule(gpa, arena, &cli_modules, build_zig_module); + + const fetch_prog_node = main_progress_node.start("Fetch Packages", 0); + defer fetch_prog_node.end(); + + const work_around_btrfs_bug = native_os == .linux and + std.zig.EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); + + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .read_only = false, + .recursive = true, + .debug_hash = false, + .work_around_btrfs_bug = work_around_btrfs_bug, + .unlazy_set = unlazy_set, + }; + defer job_queue.deinit(); + + if (graph.system_package_mode) |p| { + job_queue.global_cache = p; + job_queue.read_only = true; + } else { + try http_client.initDefaultProxies(arena); + } + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = build_root.directory }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .parent_package_root = build_root.directory, + .parent_manifest_ast = null, + .prog_node = fetch_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .allow_missing_fingerprint = false, + .allow_name_string = false, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .computed_hash = undefined, + .has_build_zig = true, + .oom_flag = false, + .latest_commit = null, + + .userdata = build_mod, + }; + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(build_root.directory, global_cache_directory), + &fetch, + ); + + job_queue.thread_pool.spawnWg(&job_queue.wait_group, Package.Fetch.workerRun, .{ + &fetch, "root", + }); + job_queue.wait_group.wait(); + + try job_queue.consolidateErrors(); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStdErr(color.renderOptions()); + process.exit(1); + } + + if (fetch_only) return std.process.cleanExit(); + + var source_buf = std.ArrayList(u8).init(gpa); + defer source_buf.deinit(); + try job_queue.createDependenciesSource(&source_buf); + const deps_mod = try createDependenciesModule( + arena, + source_buf.items, + root_mod, + global_cache_directory, + local_cache_directory, + builtin_mod, + config, + ); + + { + // We need a Module for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); + for (hashes, fetches) |*hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + if (!f.has_build_zig) + continue; + const hash_slice = hash.toSlice(); + const m = try Package.Module.create(arena, .{ + .global_cache_directory = global_cache_directory, + .paths = .{ + .root = try f.package_root.clone(arena), + .root_src_path = Package.build_zig_basename, + }, + .fully_qualified_name = try std.fmt.allocPrint( + arena, + "root.@dependencies.{s}", + .{hash_slice}, + ), + .cc_argv = &.{}, + .inherited = .{}, + .global = config, + .parent = root_mod, + .builtin_mod = builtin_mod, + .builtin_modules = null, // `builtin_mod` is specified + }); + const hash_cloned = try arena.dupe(u8, hash_slice); + deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); + f.module = m; + } + + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.module orelse continue; + const man = f.manifest orelse continue; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + global_cache_directory, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } + } + } + } + + try root_mod.deps.put(arena, "@build", Package.build_zig_basename); + + const keep_alive = false; + var prog_node = main_progress_node.start("Compile Build Script", 0); + defer prog_node.end(); + + try child_argv.appendSlice(gpa, &.{ + zig_exe, "build-exe", build_zig_module, + "--dep", "@build", configure_runner_module, + "--listen=-", + }); + const maybe_output_dir = try evalZigProcess(step, child_argv.items, prog_node, keep_alive); + const configure_exe_path = try maybe_output_dir.?.joinString(arena, exe_basename); + + prog_node.end(); + prog_node = main_progress_node.start("Run Build Script", 0); + + child_argv.clearRetainingCapacity(); + child_argv.appendSliceAssumeCapacity(&.{ + configure_exe_path, + zig_exe, + zig_lib_dir, + cache_root, + global_cache_root, + build_root, + }); + + + child_argv.items[argv_index_exe] = + try local_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?}); + } + + if (process.can_spawn) { + var child = std.process.Child.init(child_argv.items, gpa); + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + + const term = t: { + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + break :t child.spawnAndWait() catch |err| { + fatal("failed to spawn build runner {s}: {s}", .{ child_argv.items[0], @errorName(err) }); + }; + }; + + switch (term) { + .Exited => |code| { + if (code == 0) return cleanExit(); + // Indicates that the build runner has reported compile errors + // and this parent process does not need to report any further + // diagnostics. + if (code == 2) process.exit(2); + + if (code == 3) { + if (!dev.env.supports(.fetch_command)) process.exit(3); + // Indicates the configure phase failed due to missing lazy + // dependencies and stdout contains the hashes of the ones + // that are missing. + const s = fs.path.sep_str; + const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce; + const stdout = local_cache_directory.handle.readFileAlloc(arena, tmp_sub_path, 50 * 1024 * 1024) catch |err| { + fatal("unable to read results of configure phase from '{}{s}': {s}", .{ + local_cache_directory, tmp_sub_path, @errorName(err), + }); + }; + local_cache_directory.handle.deleteFile(tmp_sub_path) catch {}; + + var it = mem.splitScalar(u8, stdout, '\n'); + var any_errors = false; + while (it.next()) |hash| { + if (hash.len == 0) continue; + if (hash.len > Package.Hash.max_len) { + std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ + hash.len, hash, + }); + any_errors = true; + continue; + } + try unlazy_set.put(arena, .fromSlice(hash), {}); + } + if (any_errors) process.exit(3); + if (graph.system_package_mode) |p| { + // In this mode, the system needs to provide these packages; they + // cannot be fetched by Zig. + for (unlazy_set.keys()) |*hash| { + std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ + p, hash.toSlice(), + }); + } + std.log.info("remote package fetching disabled due to --system mode", .{}); + std.log.info("dependencies might be avoidable depending on build configuration", .{}); + process.exit(3); + } + continue; + } + + const cmd = try std.mem.join(arena, " ", child_argv.items); + fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); + }, + else => { + const cmd = try std.mem.join(arena, " ", child_argv.items); + fatal("the following build command crashed:\n{s}", .{cmd}); + }, + } + } else { + const cmd = try std.mem.join(arena, " ", child_argv.items); + fatal("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(native_os), cmd }); + } + } + } + + if (graph.needed_lazy_dependencies.entries.len != 0) { + var buffer: std.ArrayListUnmanaged(u8) = .empty; + for (graph.needed_lazy_dependencies.keys()) |k| { + try buffer.appendSlice(arena, k); + try buffer.append(arena, '\n'); + } + const s = std.fs.path.sep_str; + const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); + local_cache_directory.handle.writeFile(.{ + .sub_path = tmp_sub_path, + .data = buffer.items, + .flags = .{ .exclusive = true }, + }) catch |err| { + fatal("unable to write configuration results to '{}{s}': {s}", .{ + local_cache_directory, tmp_sub_path, @errorName(err), + }); + }; + process.exit(3); // Indicate configure phase failed with meaningful stdout. + } + + if (builder.validateUserInputDidItFail()) { + fatal(" access the help menu with 'zig build -h'", .{}); + } + + validateSystemLibraryOptions(builder); + + const stdout_writer = io.getStdOut().writer(); + + if (help_menu) + return usage(builder, stdout_writer); + + if (steps_menu) + return steps(builder, stdout_writer); + + var run: Run = .{ + .max_rss = max_rss, + .max_rss_is_default = false, + .max_rss_mutex = .{}, + .skip_oom_steps = skip_oom_steps, + .watch = watch, + .fuzz = fuzz, + .memory_blocked_steps = std.ArrayList(*Step).init(arena), + .step_stack = .{}, + .prominent_compile_errors = prominent_compile_errors, + + .claimed_rss = 0, + .summary = summary orelse if (watch) .new else .failures, + .ttyconf = ttyconf, + .stderr = stderr, + .thread_pool = thread_pool, + }; + + if (run.max_rss == 0) { + run.max_rss = process.totalSystemMemory() catch std.math.maxInt(u64); + run.max_rss_is_default = true; + } + + const gpa = arena; + prepare(gpa, arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { + error.UncleanExit => process.exit(1), + else => return err, + }; + + var w = if (watch) try Watch.init() else undefined; + + rebuild: while (true) { + runStepNames( + gpa, + builder, + targets.items, + main_progress_node, + &run, + ) catch |err| switch (err) { + error.UncleanExit => { + assert(!run.watch); + process.exit(1); + }, + else => return err, + }; + if (fuzz) { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, + // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case + // on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable; + try Fuzz.start( + gpa, + arena, + global_cache_directory, + zig_lib_directory, + zig_exe, + &run.thread_pool, + run.step_stack.keys(), + run.ttyconf, + listen_address, + main_progress_node, + ); + } + + if (!watch) return cleanExit(); + + if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); + + try w.update(gpa, run.step_stack.keys()); + + // Wait until a file system notification arrives. Read all such events + // until the buffer is empty. Then wait for a debounce interval, resetting + // if any more events come in. After the debounce interval has passed, + // trigger a rebuild on all steps with modified inputs, as well as their + // recursive dependants. + var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; + const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ + w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), + }) catch &caption_buf; + var debouncing_node = main_progress_node.start(caption, 0); + var debounce_timeout: Watch.Timeout = .none; + while (true) switch (try w.wait(gpa, debounce_timeout)) { + .timeout => { + debouncing_node.end(); + markFailedStepsDirty(gpa, run.step_stack.keys()); + continue :rebuild; + }, + .dirty => if (debounce_timeout == .none) { + debounce_timeout = .{ .ms = debounce_interval_ms }; + debouncing_node.end(); + debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + }, + .clean => {}, + }; + } +} + +const CliModule = struct { + deps: std.StringArrayHashMapUnmanaged(*CliModule), +}; + +fn addCliModule(gpa: Allocator, arena: Allocator, aoeu + const build_mod = try addCliModule(gpa, arena, &cli_modules, build_zig_module); + + +fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { + for (all_steps) |step| switch (step.state) { + .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), + else => continue, + }; + // Now that all dirty steps have been found, the remaining steps that + // succeeded from last run shall be marked "cached". + for (all_steps) |step| switch (step.state) { + .success => step.result_cached = true, + else => continue, + }; +} + +fn countSubProcesses(all_steps: []const *Step) usize { + var count: usize = 0; + for (all_steps) |s| { + count += @intFromBool(s.getZigProcess() != null); + } + return count; +} + +const Run = struct { + max_rss: u64, + max_rss_is_default: bool, + max_rss_mutex: std.Thread.Mutex, + skip_oom_steps: bool, + watch: bool, + fuzz: bool, + memory_blocked_steps: std.ArrayList(*Step), + step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), + prominent_compile_errors: bool, + thread_pool: *std.Thread.Pool, + + claimed_rss: usize, + summary: Summary, + ttyconf: std.io.tty.Config, + stderr: File, + + fn cleanExit(run: Run) void { + if (run.watch or run.fuzz) return; + return std.process.cleanExit(); + } +}; + +fn prepare( + gpa: Allocator, + arena: Allocator, + b: *std.Build, + step_names: []const []const u8, + run: *Run, + seed: u32, +) !void { + const step_stack = &run.step_stack; + + if (step_names.len == 0) { + try step_stack.put(gpa, b.default_step, {}); + } else { + try step_stack.ensureUnusedCapacity(gpa, step_names.len); + for (0..step_names.len) |i| { + const step_name = step_names[step_names.len - i - 1]; + const s = b.top_level_steps.get(step_name) orelse { + std.debug.print("no step named '{s}'\n access the help menu with 'zig build -h'\n", .{step_name}); + process.exit(1); + }; + step_stack.putAssumeCapacity(&s.step, {}); + } + } + + const starting_steps = try arena.dupe(*Step, step_stack.keys()); + + var rng = std.Random.DefaultPrng.init(seed); + const rand = rng.random(); + rand.shuffle(*Step, starting_steps); + + for (starting_steps) |s| { + constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) { + error.DependencyLoopDetected => return uncleanExit(), + else => |e| return e, + }; + } + + { + // Check that we have enough memory to complete the build. + var any_problems = false; + for (step_stack.keys()) |s| { + if (s.max_rss == 0) continue; + if (s.max_rss > run.max_rss) { + if (run.skip_oom_steps) { + s.state = .skipped_oom; + } else { + std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{ + s.owner.dep_prefix, s.name, s.max_rss, run.max_rss, + }); + any_problems = true; + } + } + } + if (any_problems) { + if (run.max_rss_is_default) { + std.debug.print("note: use --maxrss to override the default", .{}); + } + return uncleanExit(); + } + } +} + +fn runStepNames( + gpa: Allocator, + b: *std.Build, + step_names: []const []const u8, + parent_prog_node: std.Progress.Node, + run: *Run, +) !void { + const step_stack = &run.step_stack; + const thread_pool = &run.thread_pool; + + { + const step_prog = parent_prog_node.start("steps", step_stack.count()); + defer step_prog.end(); + + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + + // Here we spawn the initial set of tasks with a nice heuristic - + // dependency order. Each worker when it finishes a step will then + // check whether it should run any dependants. + const steps_slice = step_stack.keys(); + for (0..steps_slice.len) |i| { + const step = steps_slice[steps_slice.len - i - 1]; + if (step.state == .skipped_oom) continue; + + thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{ + &wait_group, b, step, step_prog, run, + }); + } + } + assert(run.memory_blocked_steps.items.len == 0); + + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_pass_count: usize = 0; + var test_leak_count: usize = 0; + var test_count: usize = 0; + + var success_count: usize = 0; + var skipped_count: usize = 0; + var failure_count: usize = 0; + var pending_count: usize = 0; + var total_compile_errors: usize = 0; + + for (step_stack.keys()) |s| { + test_fail_count += s.test_results.fail_count; + test_skip_count += s.test_results.skip_count; + test_leak_count += s.test_results.leak_count; + test_pass_count += s.test_results.passCount(); + test_count += s.test_results.test_count; + + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .running => unreachable, + .precheck_done => { + // precheck_done is equivalent to dependency_failure in the case of + // transitive dependencies. For example: + // A -> B -> C (failure) + // B will be marked as dependency_failure, while A may never be queued, and thus + // remain in the initial state of precheck_done. + s.state = .dependency_failure; + pending_count += 1; + }, + .dependency_failure => pending_count += 1, + .success => success_count += 1, + .skipped, .skipped_oom => skipped_count += 1, + .failure => { + failure_count += 1; + const compile_errors_len = s.result_error_bundle.errorMessageCount(); + if (compile_errors_len > 0) { + total_compile_errors += compile_errors_len; + } + }, + } + } + + // A proper command line application defaults to silently succeeding. + // The user may request verbose mode if they have a different preference. + const failures_only = switch (run.summary) { + .failures, .none => true, + else => false, + }; + if (failure_count == 0 and failures_only) { + return run.cleanExit(); + } + + const ttyconf = run.ttyconf; + + if (run.summary != .none) { + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + const stderr = run.stderr; + + const total_count = success_count + failure_count + pending_count + skipped_count; + ttyconf.setColor(stderr, .cyan) catch {}; + stderr.writeAll("Build Summary:") catch {}; + ttyconf.setColor(stderr, .reset) catch {}; + stderr.writer().print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; + if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {}; + if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {}; + + if (test_count > 0) stderr.writer().print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + if (test_skip_count > 0) stderr.writer().print("; {d} skipped", .{test_skip_count}) catch {}; + if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; + if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; + + stderr.writeAll("\n") catch {}; + + // Print a fancy tree with build results. + var step_stack_copy = try step_stack.clone(gpa); + defer step_stack_copy.deinit(gpa); + + var print_node: PrintNode = .{ .parent = null }; + if (step_names.len == 0) { + print_node.last = true; + printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; + } else { + const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { + var i: usize = step_names.len; + while (i > 0) { + i -= 1; + const step = b.top_level_steps.get(step_names[i]).?.step; + const found = switch (run.summary) { + .all, .none => unreachable, + .failures => step.state != .success, + .new => !step.result_cached, + }; + if (found) break :blk i; + } + break :blk b.top_level_steps.count(); + }; + for (step_names, 0..) |step_name, i| { + const tls = b.top_level_steps.get(step_name).?; + print_node.last = i + 1 == last_index; + printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; + } + } + } + + if (failure_count == 0) { + return run.cleanExit(); + } + + // Finally, render compile errors at the bottom of the terminal. + if (run.prominent_compile_errors and total_compile_errors > 0) { + for (step_stack.keys()) |s| { + if (s.result_error_bundle.errorMessageCount() > 0) { + s.result_error_bundle.renderToStdErr(.{ .ttyconf = ttyconf, .include_reference_trace = (b.reference_trace orelse 0) > 0 }); + } + } + + if (!run.watch) { + // Signal to parent process that we have printed compile errors. The + // parent process may choose to omit the "following command failed" + // line in this case. + std.debug.lockStdErr(); + process.exit(2); + } + } + + if (!run.watch) return uncleanExit(); +} + +const PrintNode = struct { + parent: ?*PrintNode, + last: bool = false, +}; + +fn printPrefix(node: *PrintNode, stderr: File, ttyconf: std.io.tty.Config) !void { + const parent = node.parent orelse return; + if (parent.parent == null) return; + try printPrefix(parent, stderr, ttyconf); + if (parent.last) { + try stderr.writeAll(" "); + } else { + try stderr.writeAll(switch (ttyconf) { + .no_color, .windows_api => "| ", + .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ + }); + } +} + +fn printChildNodePrefix(stderr: File, ttyconf: std.io.tty.Config) !void { + try stderr.writeAll(switch (ttyconf) { + .no_color, .windows_api => "+- ", + .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ + }); +} + +fn printStepStatus( + s: *Step, + stderr: File, + ttyconf: std.io.tty.Config, + run: *const Run, +) !void { + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + .running => unreachable, + + .dependency_failure => { + try ttyconf.setColor(stderr, .dim); + try stderr.writeAll(" transitive failure\n"); + try ttyconf.setColor(stderr, .reset); + }, + + .success => { + try ttyconf.setColor(stderr, .green); + if (s.result_cached) { + try stderr.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + try stderr.writer().print(" {d} passed", .{pass_count}); + if (s.test_results.skip_count > 0) { + try ttyconf.setColor(stderr, .yellow); + try stderr.writer().print(" {d} skipped", .{s.test_results.skip_count}); + } + } else { + try stderr.writeAll(" success"); + } + try ttyconf.setColor(stderr, .reset); + if (s.result_duration_ns) |ns| { + try ttyconf.setColor(stderr, .dim); + if (ns >= std.time.ns_per_min) { + try stderr.writer().print(" {d}m", .{ns / std.time.ns_per_min}); + } else if (ns >= std.time.ns_per_s) { + try stderr.writer().print(" {d}s", .{ns / std.time.ns_per_s}); + } else if (ns >= std.time.ns_per_ms) { + try stderr.writer().print(" {d}ms", .{ns / std.time.ns_per_ms}); + } else if (ns >= std.time.ns_per_us) { + try stderr.writer().print(" {d}us", .{ns / std.time.ns_per_us}); + } else { + try stderr.writer().print(" {d}ns", .{ns}); + } + try ttyconf.setColor(stderr, .reset); + } + if (s.result_peak_rss != 0) { + const rss = s.result_peak_rss; + try ttyconf.setColor(stderr, .dim); + if (rss >= 1000_000_000) { + try stderr.writer().print(" MaxRSS:{d}G", .{rss / 1000_000_000}); + } else if (rss >= 1000_000) { + try stderr.writer().print(" MaxRSS:{d}M", .{rss / 1000_000}); + } else if (rss >= 1000) { + try stderr.writer().print(" MaxRSS:{d}K", .{rss / 1000}); + } else { + try stderr.writer().print(" MaxRSS:{d}B", .{rss}); + } + try ttyconf.setColor(stderr, .reset); + } + try stderr.writeAll("\n"); + }, + .skipped, .skipped_oom => |skip| { + try ttyconf.setColor(stderr, .yellow); + try stderr.writeAll(" skipped"); + if (skip == .skipped_oom) { + try stderr.writeAll(" (not enough memory)"); + try ttyconf.setColor(stderr, .dim); + try stderr.writer().print(" upper bound of {d} exceeded runner limit ({d})", .{ s.max_rss, run.max_rss }); + try ttyconf.setColor(stderr, .yellow); + } + try stderr.writeAll("\n"); + try ttyconf.setColor(stderr, .reset); + }, + .failure => try printStepFailure(s, stderr, ttyconf), + } +} + +fn printStepFailure( + s: *Step, + stderr: File, + ttyconf: std.io.tty.Config, +) !void { + if (s.result_error_bundle.errorMessageCount() > 0) { + try ttyconf.setColor(stderr, .red); + try stderr.writer().print(" {d} errors\n", .{ + s.result_error_bundle.errorMessageCount(), + }); + try ttyconf.setColor(stderr, .reset); + } else if (!s.test_results.isSuccess()) { + try stderr.writer().print(" {d}/{d} passed", .{ + s.test_results.passCount(), s.test_results.test_count, + }); + if (s.test_results.fail_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .red); + try stderr.writer().print("{d} failed", .{ + s.test_results.fail_count, + }); + try ttyconf.setColor(stderr, .reset); + } + if (s.test_results.skip_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .yellow); + try stderr.writer().print("{d} skipped", .{ + s.test_results.skip_count, + }); + try ttyconf.setColor(stderr, .reset); + } + if (s.test_results.leak_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .red); + try stderr.writer().print("{d} leaked", .{ + s.test_results.leak_count, + }); + try ttyconf.setColor(stderr, .reset); + } + try stderr.writeAll("\n"); + } else if (s.result_error_msgs.items.len > 0) { + try ttyconf.setColor(stderr, .red); + try stderr.writeAll(" failure\n"); + try ttyconf.setColor(stderr, .reset); + } else { + assert(s.result_stderr.len > 0); + try ttyconf.setColor(stderr, .red); + try stderr.writeAll(" stderr\n"); + try ttyconf.setColor(stderr, .reset); + } +} + +fn printTreeStep( + b: *std.Build, + s: *Step, + run: *const Run, + stderr: File, + ttyconf: std.io.tty.Config, + parent_node: *PrintNode, + step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), +) !void { + const first = step_stack.swapRemove(s); + const summary = run.summary; + const skip = switch (summary) { + .none => unreachable, + .all => false, + .new => s.result_cached, + .failures => s.state == .success, + }; + if (skip) return; + try printPrefix(parent_node, stderr, ttyconf); + + if (!first) try ttyconf.setColor(stderr, .dim); + if (parent_node.parent != null) { + if (parent_node.last) { + try printChildNodePrefix(stderr, ttyconf); + } else { + try stderr.writeAll(switch (ttyconf) { + .no_color, .windows_api => "+- ", + .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ + }); + } + } + + // dep_prefix omitted here because it is redundant with the tree. + try stderr.writeAll(s.name); + + if (first) { + try printStepStatus(s, stderr, ttyconf, run); + + const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { + var i: usize = s.dependencies.items.len; + while (i > 0) { + i -= 1; + + const step = s.dependencies.items[i]; + const found = switch (summary) { + .all, .none => unreachable, + .failures => step.state != .success, + .new => !step.result_cached, + }; + if (found) break :blk i; + } + break :blk s.dependencies.items.len -| 1; + }; + for (s.dependencies.items, 0..) |dep, i| { + var print_node: PrintNode = .{ + .parent = parent_node, + .last = i == last_index, + }; + try printTreeStep(b, dep, run, stderr, ttyconf, &print_node, step_stack); + } + } else { + if (s.dependencies.items.len == 0) { + try stderr.writeAll(" (reused)\n"); + } else { + try stderr.writer().print(" (+{d} more reused dependencies)\n", .{ + s.dependencies.items.len, + }); + } + try ttyconf.setColor(stderr, .reset); + } +} + +/// Traverse the dependency graph depth-first and make it undirected by having +/// steps know their dependants (they only know dependencies at start). +/// Along the way, check that there is no dependency loop, and record the steps +/// in traversal order in `step_stack`. +/// Each step has its dependencies traversed in random order, this accomplishes +/// two things: +/// - `step_stack` will be in randomized-depth-first order, so the build runner +/// spawns steps in a random (but optimized) order +/// - each step's `dependants` list is also filled in a random order, so that +/// when it finishes executing in `workerMakeOneStep`, it spawns next steps +/// to run in random order +fn constructGraphAndCheckForDependencyLoop( + b: *std.Build, + s: *Step, + step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), + rand: std.Random, +) !void { + switch (s.state) { + .precheck_started => { + std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); + return error.DependencyLoopDetected; + }, + .precheck_unstarted => { + s.state = .precheck_started; + + try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len); + + // We dupe to avoid shuffling the steps in the summary, it depends + // on s.dependencies' order. + const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM"); + rand.shuffle(*Step, deps); + + for (deps) |dep| { + try step_stack.put(b.allocator, dep, {}); + try dep.dependants.append(b.allocator, s); + constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| { + if (err == error.DependencyLoopDetected) { + std.debug.print(" {s}\n", .{s.name}); + } + return err; + }; + } + + s.state = .precheck_done; + }, + .precheck_done => {}, + + // These don't happen until we actually run the step graph. + .dependency_failure => unreachable, + .running => unreachable, + .success => unreachable, + .failure => unreachable, + .skipped => unreachable, + .skipped_oom => unreachable, + } +} + +fn workerMakeOneStep( + wg: *std.Thread.WaitGroup, + b: *std.Build, + s: *Step, + prog_node: std.Progress.Node, + run: *Run, +) void { + const thread_pool = &run.thread_pool; + + // First, check the conditions for running this step. If they are not met, + // then we return without doing the step, relying on another worker to + // queue this step up again when dependencies are met. + for (s.dependencies.items) |dep| { + switch (@atomicLoad(Step.State, &dep.state, .seq_cst)) { + .success, .skipped => continue, + .failure, .dependency_failure, .skipped_oom => { + @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst); + return; + }, + .precheck_done, .running => { + // dependency is not finished yet. + return; + }, + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + } + } + + if (s.max_rss != 0) { + run.max_rss_mutex.lock(); + defer run.max_rss_mutex.unlock(); + + // Avoid running steps twice. + if (s.state != .precheck_done) { + // Another worker got the job. + return; + } + + const new_claimed_rss = run.claimed_rss + s.max_rss; + if (new_claimed_rss > run.max_rss) { + // Running this step right now could possibly exceed the allotted RSS. + // Add this step to the queue of memory-blocked steps. + run.memory_blocked_steps.append(s) catch @panic("OOM"); + return; + } + + run.claimed_rss = new_claimed_rss; + s.state = .running; + } else { + // Avoid running steps twice. + if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .seq_cst, .seq_cst) != null) { + // Another worker got the job. + return; + } + } + + const sub_prog_node = prog_node.start(s.name, 0); + defer sub_prog_node.end(); + + const make_result = s.make(.{ + .progress_node = sub_prog_node, + .thread_pool = thread_pool, + .watch = run.watch, + }); + + // No matter the result, we want to display error/warning messages. + const show_compile_errors = !run.prominent_compile_errors and + s.result_error_bundle.errorMessageCount() > 0; + const show_error_msgs = s.result_error_msgs.items.len > 0; + const show_stderr = s.result_stderr.len > 0; + + if (show_error_msgs or show_compile_errors or show_stderr) { + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + + const gpa = b.allocator; + const options: std.zig.ErrorBundle.RenderOptions = .{ + .ttyconf = run.ttyconf, + .include_reference_trace = (b.reference_trace orelse 0) > 0, + }; + printErrorMessages(gpa, s, options, run.stderr, run.prominent_compile_errors) catch {}; + } + + handle_result: { + if (make_result) |_| { + @atomicStore(Step.State, &s.state, .success, .seq_cst); + } else |err| switch (err) { + error.MakeFailed => { + @atomicStore(Step.State, &s.state, .failure, .seq_cst); + break :handle_result; + }, + error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .seq_cst), + } + + // Successful completion of a step, so we queue up its dependants as well. + for (s.dependants.items) |dep| { + thread_pool.spawnWg(wg, workerMakeOneStep, .{ + wg, b, dep, prog_node, run, + }); + } + } + + // If this is a step that claims resources, we must now queue up other + // steps that are waiting for resources. + if (s.max_rss != 0) { + run.max_rss_mutex.lock(); + defer run.max_rss_mutex.unlock(); + + // Give the memory back to the scheduler. + run.claimed_rss -= s.max_rss; + // Avoid kicking off too many tasks that we already know will not have + // enough resources. + var remaining = run.max_rss - run.claimed_rss; + var i: usize = 0; + var j: usize = 0; + while (j < run.memory_blocked_steps.items.len) : (j += 1) { + const dep = run.memory_blocked_steps.items[j]; + assert(dep.max_rss != 0); + if (dep.max_rss <= remaining) { + remaining -= dep.max_rss; + + thread_pool.spawnWg(wg, workerMakeOneStep, .{ + wg, b, dep, prog_node, run, + }); + } else { + run.memory_blocked_steps.items[i] = dep; + i += 1; + } + } + run.memory_blocked_steps.shrinkRetainingCapacity(i); + } +} + +pub fn printErrorMessages( + gpa: Allocator, + failing_step: *Step, + options: std.zig.ErrorBundle.RenderOptions, + stderr: File, + prominent_compile_errors: bool, +) !void { + // Provide context for where these error messages are coming from by + // printing the corresponding Step subtree. + + var step_stack: std.ArrayListUnmanaged(*Step) = .empty; + defer step_stack.deinit(gpa); + try step_stack.append(gpa, failing_step); + while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { + try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); + } + + // Now, `step_stack` has the subtree that we want to print, in reverse order. + const ttyconf = options.ttyconf; + try ttyconf.setColor(stderr, .dim); + var indent: usize = 0; + while (step_stack.pop()) |s| : (indent += 1) { + if (indent > 0) { + try stderr.writer().writeByteNTimes(' ', (indent - 1) * 3); + try printChildNodePrefix(stderr, ttyconf); + } + + try stderr.writeAll(s.name); + + if (s == failing_step) { + try printStepFailure(s, stderr, ttyconf); + } else { + try stderr.writeAll("\n"); + } + } + try ttyconf.setColor(stderr, .reset); + + if (failing_step.result_stderr.len > 0) { + try stderr.writeAll(failing_step.result_stderr); + if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { + try stderr.writeAll("\n"); + } + } + + if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) { + try failing_step.result_error_bundle.renderToWriter(options, stderr.writer()); + } + + for (failing_step.result_error_msgs.items) |msg| { + try ttyconf.setColor(stderr, .red); + try stderr.writeAll("error: "); + try ttyconf.setColor(stderr, .reset); + try stderr.writeAll(msg); + try stderr.writeAll("\n"); + } +} + +fn steps(builder: *std.Build, out_stream: anytype) !void { + const allocator = builder.allocator; + for (builder.top_level_steps.values()) |top_level_step| { + const name = if (&top_level_step.step == builder.default_step) + try fmt.allocPrint(allocator, "{s} (default)", .{top_level_step.step.name}) + else + top_level_step.step.name; + try out_stream.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); + } +} + +fn usage(b: *std.Build, out_stream: anytype) !void { + try out_stream.print( + \\Usage: {s} build [steps] [options] + \\ + \\Steps: + \\ + , .{b.graph.zig_exe}); + try steps(b, out_stream); + + try out_stream.writeAll( + \\ + \\General Options: + \\ -p, --prefix [path] Where to install files (default: zig-out) + \\ --prefix-lib-dir [path] Where to install libraries + \\ --prefix-exe-dir [path] Where to install executables + \\ --prefix-include-dir [path] Where to install C header files + \\ + \\ --release[=mode] Request release mode, optionally specifying a + \\ preferred optimization mode: fast, safe, small + \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --glibc-runtimes [path] Enhances QEMU integration by providing glibc built + \\ for multiple foreign architectures, allowing + \\ execution of non-native programs that link with glibc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ + \\ -h, --help Print this help and exit + \\ -l, --list-steps Print available steps + \\ --verbose Print commands before executing them + \\ --color [auto|off|on] Enable or disable colored error messages + \\ --prominent-compile-errors Buffer compile errors and display at end + \\ --summary [mode] Control the printing of the build summary + \\ all Print the build summary in its entirety + \\ new Omit cached steps + \\ failures (Default) Only print failed steps + \\ none Do not print the build summary + \\ -j Limit concurrent jobs (default is to use all CPU cores) + \\ --maxrss Limit memory usage (default is to use available memory) + \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --fetch Exit after fetching dependency tree + \\ --watch Continuously rebuild when source files are modified + \\ --fuzz Continuously search for unit test failures + \\ --debounce Delay before rebuilding after changed file detected + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation + \\ + \\Project-Specific Options: + \\ + ); + + const arena = b.allocator; + if (b.available_options_list.items.len == 0) { + try out_stream.print(" (none)\n", .{}); + } else { + for (b.available_options_list.items) |option| { + const name = try fmt.allocPrint(arena, " -D{s}=[{s}]", .{ + option.name, + @tagName(option.type_id), + }); + try out_stream.print("{s:<30} {s}\n", .{ name, option.description }); + if (option.enum_options) |enum_options| { + const padding = " " ** 33; + try out_stream.writeAll(padding ++ "Supported Values:\n"); + for (enum_options) |enum_option| { + try out_stream.print(padding ++ " {s}\n", .{enum_option}); + } + } + } + } + + try out_stream.writeAll( + \\ + \\System Integration Options: + \\ --search-prefix [path] Add a path to look for binaries, libraries, headers + \\ --sysroot [path] Set the system root directory (usually /) + \\ --libc [file] Provide a file which specifies libc paths + \\ + \\ --system [pkgdir] Disable package fetching; enable all integrations + \\ -fsys=[name] Enable a system integration + \\ -fno-sys=[name] Disable a system integration + \\ + \\ Available System Integrations: Enabled: + \\ + ); + if (b.graph.system_library_options.entries.len == 0) { + try out_stream.writeAll(" (none) -\n"); + } else { + for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { + const status = switch (v) { + .declared_enabled => "yes", + .declared_disabled => "no", + .user_enabled, .user_disabled => unreachable, // already emitted error + }; + try out_stream.print(" {s:<43} {s}\n", .{ k, status }); + } + } + + try out_stream.writeAll( + \\ + \\Advanced Options: + \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error + \\ -fno-reference-trace Disable reference trace + \\ -fallow-so-scripts Allows .so files to be GNU ld scripts + \\ -fno-allow-so-scripts (default) .so files must be ELF files + \\ --build-file [file] Override path to build.zig + \\ --cache-dir [path] Override path to local Zig cache directory + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner + \\ --seed [integer] For shuffling dependency traversal order (default: random) + \\ --debug-log [scope] Enable debugging the compiler + \\ --debug-pkg-config Fail if unknown pkg-config flags encountered + \\ --debug-rt Debug compiler runtime libraries + \\ --verbose-link Enable compiler debug output for linking + \\ --verbose-air Enable compiler debug output for Zig AIR + \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR + \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC + \\ --verbose-cimport Enable compiler debug output for C imports + \\ --verbose-cc Enable compiler debug output for C compilation + \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features + \\ + ); +} + +fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse { + std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); + process.exit(1); + }; +} + +fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { + if (idx >= args.len) return null; + return args[idx..]; +} + +/// Perhaps in the future there could be an Advanced Options flag such as +/// --debug-build-runner-leaks which would make this function return instead of +/// calling exit. +fn uncleanExit() error{UncleanExit} { + std.debug.lockStdErr(); + process.exit(1); +} + +const Color = std.zig.Color; +const Summary = enum { all, new, failures, none }; + +fn get_tty_conf(color: Color, stderr: File) std.io.tty.Config { + return switch (color) { + .auto => std.io.tty.detectConfig(stderr), + .on => .escape_codes, + .off => .no_color, + }; +} + +fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { + std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); + process.exit(1); +} + +fn validateSystemLibraryOptions(b: *std.Build) void { + var bad = false; + for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { + switch (v) { + .user_disabled, .user_enabled => { + // The user tried to enable or disable a system library integration, but + // the build script did not recognize that option. + std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); + bad = true; + }, + .declared_disabled, .declared_enabled => {}, + } + } + if (bad) { + std.debug.print(" access the help menu with 'zig build -h'\n", .{}); + process.exit(1); + } +} + +/// Creates the dependencies.zig file and corresponding `Module` for the +/// build runner to obtain via `@import("@dependencies")`. +fn createDependenciesModule( + arena: Allocator, + source: []const u8, + main_mod: *CliModule, + global_cache_directory: Directory, + local_cache_directory: Directory, + builtin_mod: *Module, + global_options: Compilation.Config, +) !*CliModule { + // Atomically create the file in a directory named after the hash of its contents. + const basename = "dependencies.zig"; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + { + var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(.{ .sub_path = basename, .data = source }); + } + + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(source); + const hex_digest = hh.final(); + + const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); + try Package.Fetch.renameTmpIntoCache( + local_cache_directory.handle, + tmp_dir_sub_path, + o_dir_sub_path, + ); + + const deps_mod = try Module.create(arena, .{ + .global_cache_directory = global_cache_directory, + .paths = .{ + .root = .{ + .root_dir = local_cache_directory, + .sub_path = o_dir_sub_path, + }, + .root_src_path = basename, + }, + .fully_qualified_name = "root.@dependencies", + .parent = main_mod, + .cc_argv = &.{}, + .inherited = .{}, + .global = global_options, + .builtin_mod = builtin_mod, + .builtin_modules = null, // `builtin_mod` is specified + }); + try main_mod.deps.put(arena, "@dependencies", deps_mod); + return deps_mod; +} diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig deleted file mode 100644 index 8702acb329779b504b9b5623deee4663195ae590..0000000000000000000000000000000000000000 --- a/lib/compiler/build_runner.zig +++ /dev/null @@ -1,1525 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const assert = std.debug.assert; -const io = std.io; -const fmt = std.fmt; -const mem = std.mem; -const process = std.process; -const ArrayList = std.ArrayList; -const File = std.fs.File; -const Step = std.Build.Step; -const Watch = std.Build.Watch; -const Fuzz = std.Build.Fuzz; -const Allocator = std.mem.Allocator; -const fatal = std.process.fatal; -const runner = @This(); - -pub const root = @import("@build"); -pub const dependencies = @import("@dependencies"); - -pub const std_options: std.Options = .{ - .side_channels_mitigations = .none, - .http_disable_tls = true, - .crypto_fork_safety = false, -}; - -pub fn main() !void { - // Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived, - // one shot program. We don't need to waste time freeing memory and finding places to squish - // bytes into. So we free everything all at once at the very end. - var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer single_threaded_arena.deinit(); - - var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ - .child_allocator = single_threaded_arena.allocator(), - }; - const arena = thread_safe_arena.allocator(); - - const args = try process.argsAlloc(arena); - - // skip my own exe name - var arg_idx: usize = 1; - - const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); - const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); - const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); - const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); - const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); - - const zig_lib_directory: std.Build.Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), - }; - - const build_root_directory: std.Build.Cache.Directory = .{ - .path = build_root, - .handle = try std.fs.cwd().openDir(build_root, .{}), - }; - - const local_cache_directory: std.Build.Cache.Directory = .{ - .path = cache_root, - .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), - }; - - const global_cache_directory: std.Build.Cache.Directory = .{ - .path = global_cache_root, - .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), - }; - - var graph: std.Build.Graph = .{ - .arena = arena, - .cache = .{ - .gpa = arena, - .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), - }, - .zig_exe = zig_exe, - .env_map = try process.getEnvMap(arena), - .global_cache_root = global_cache_directory, - .zig_lib_directory = zig_lib_directory, - .host = .{ - .query = .{}, - .result = try std.zig.system.resolveTargetQuery(.{}), - }, - }; - - graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); - - const builder = try std.Build.create( - &graph, - build_root_directory, - local_cache_directory, - dependencies.root_deps, - ); - - var targets = ArrayList([]const u8).init(arena); - var debug_log_scopes = ArrayList([]const u8).init(arena); - var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena }; - - var install_prefix: ?[]const u8 = null; - var dir_list = std.Build.DirList{}; - var summary: ?Summary = null; - var max_rss: u64 = 0; - var skip_oom_steps = false; - var color: Color = .auto; - var prominent_compile_errors = false; - var help_menu = false; - var steps_menu = false; - var output_tmp_nonce: ?[16]u8 = null; - var watch = false; - var fuzz = false; - var debounce_interval_ms: u16 = 50; - var listen_port: u16 = 0; - - while (nextArg(args, &arg_idx)) |arg| { - if (mem.startsWith(u8, arg, "-Z")) { - if (arg.len != 18) fatalWithHint("bad argument: '{s}'", .{arg}); - output_tmp_nonce = arg[2..18].*; - } else if (mem.startsWith(u8, arg, "-D")) { - const option_contents = arg[2..]; - if (option_contents.len == 0) - fatalWithHint("expected option name after '-D'", .{}); - if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { - const option_name = option_contents[0..name_end]; - const option_value = option_contents[name_end + 1 ..]; - if (try builder.addUserInputOption(option_name, option_value)) - fatal(" access the help menu with 'zig build -h'", .{}); - } else { - if (try builder.addUserInputFlag(option_contents)) - fatal(" access the help menu with 'zig build -h'", .{}); - } - } else if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "--verbose")) { - builder.verbose = true; - } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - help_menu = true; - } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - install_prefix = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { - steps_menu = true; - } else if (mem.startsWith(u8, arg, "-fsys=")) { - const name = arg["-fsys=".len..]; - graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); - } else if (mem.startsWith(u8, arg, "-fno-sys=")) { - const name = arg["-fno-sys=".len..]; - graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); - } else if (mem.eql(u8, arg, "--release")) { - builder.release_mode = .any; - } else if (mem.startsWith(u8, arg, "--release=")) { - const text = arg["--release=".len..]; - builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { - fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ - arg, text, - }); - }; - } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - dir_list.exe_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - dir_list.include_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--sysroot")) { - builder.sysroot = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--maxrss")) { - const max_rss_text = nextArgOrFatal(args, &arg_idx); - max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { - std.debug.print("invalid byte size: '{s}': {s}\n", .{ - max_rss_text, @errorName(err), - }); - process.exit(1); - }; - } else if (mem.eql(u8, arg, "--skip-oom-steps")) { - skip_oom_steps = true; - } else if (mem.eql(u8, arg, "--search-prefix")) { - const search_prefix = nextArgOrFatal(args, &arg_idx); - builder.addSearchPrefix(search_prefix); - } else if (mem.eql(u8, arg, "--libc")) { - builder.libc_file = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [all|new|failures|none] after '{s}'", .{arg}); - summary = std.meta.stringToEnum(Summary, next_arg) orelse { - fatalWithHint("expected [all|new|failures|none] after '{s}', found '{s}'", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after '{s}'", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ - next_arg, @errorName(err), - }); - }; - } else if (mem.eql(u8, arg, "--debounce")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u16 after '{s}'", .{arg}); - debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { - fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {s}\n", .{ - next_arg, @errorName(err), - }); - }; - } else if (mem.eql(u8, arg, "--port")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u16 after '{s}'", .{arg}); - listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| { - fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{ - next_arg, @errorName(err), - }); - }; - } else if (mem.eql(u8, arg, "--debug-log")) { - const next_arg = nextArgOrFatal(args, &arg_idx); - try debug_log_scopes.append(next_arg); - } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - builder.debug_pkg_config = true; - } else if (mem.eql(u8, arg, "--debug-rt")) { - graph.debug_compiler_runtime_libs = true; - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - builder.debug_compile_errors = true; - } else if (mem.eql(u8, arg, "--system")) { - // The usage text shows another argument after this parameter - // but it is handled by the parent process. The build runner - // only sees this flag. - graph.system_package_mode = true; - } else if (mem.eql(u8, arg, "--glibc-runtimes")) { - builder.glibc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--verbose-link")) { - builder.verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - builder.verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - builder.verbose_llvm_ir = "-"; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { - builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; - } else if (mem.eql(u8, arg, "--verbose-llvm-bc=")) { - builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; - } else if (mem.eql(u8, arg, "--verbose-cimport")) { - builder.verbose_cimport = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - builder.verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - builder.verbose_llvm_cpu_features = true; - } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { - prominent_compile_errors = true; - } else if (mem.eql(u8, arg, "--watch")) { - watch = true; - } else if (mem.eql(u8, arg, "--fuzz")) { - fuzz = true; - } else if (mem.eql(u8, arg, "-fincremental")) { - graph.incremental = true; - } else if (mem.eql(u8, arg, "-fno-incremental")) { - graph.incremental = false; - } else if (mem.eql(u8, arg, "-fwine")) { - builder.enable_wine = true; - } else if (mem.eql(u8, arg, "-fno-wine")) { - builder.enable_wine = false; - } else if (mem.eql(u8, arg, "-fqemu")) { - builder.enable_qemu = true; - } else if (mem.eql(u8, arg, "-fno-qemu")) { - builder.enable_qemu = false; - } else if (mem.eql(u8, arg, "-fwasmtime")) { - builder.enable_wasmtime = true; - } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - builder.enable_wasmtime = false; - } else if (mem.eql(u8, arg, "-frosetta")) { - builder.enable_rosetta = true; - } else if (mem.eql(u8, arg, "-fno-rosetta")) { - builder.enable_rosetta = false; - } else if (mem.eql(u8, arg, "-fdarling")) { - builder.enable_darling = true; - } else if (mem.eql(u8, arg, "-fno-darling")) { - builder.enable_darling = false; - } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { - graph.allow_so_scripts = true; - } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { - graph.allow_so_scripts = false; - } else if (mem.eql(u8, arg, "-freference-trace")) { - builder.reference_trace = 256; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); - process.exit(1); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - builder.reference_trace = null; - } else if (mem.startsWith(u8, arg, "-j")) { - const num = arg["-j".len..]; - const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse jobs count '{s}': {s}", .{ - num, @errorName(err), - }); - process.exit(1); - }; - if (n_jobs < 1) { - std.debug.print("number of jobs must be at least 1\n", .{}); - process.exit(1); - } - thread_pool_options.n_jobs = n_jobs; - } else if (mem.eql(u8, arg, "--")) { - builder.args = argsRest(args, arg_idx); - break; - } else { - fatalWithHint("unrecognized argument: '{s}'", .{arg}); - } - } else { - try targets.append(arg); - } - } - - const stderr = std.io.getStdErr(); - const ttyconf = get_tty_conf(color, stderr); - switch (ttyconf) { - .no_color => try graph.env_map.put("NO_COLOR", "1"), - .escape_codes => try graph.env_map.put("CLICOLOR_FORCE", "1"), - .windows_api => {}, - } - - const main_progress_node = std.Progress.start(.{ - .disable_printing = (color == .off), - }); - defer main_progress_node.end(); - - builder.debug_log_scopes = debug_log_scopes.items; - builder.resolveInstallPrefix(install_prefix, dir_list); - { - var prog_node = main_progress_node.start("Configure", 0); - defer prog_node.end(); - try builder.runBuild(root); - createModuleDependencies(builder) catch @panic("OOM"); - } - - if (graph.needed_lazy_dependencies.entries.len != 0) { - var buffer: std.ArrayListUnmanaged(u8) = .empty; - for (graph.needed_lazy_dependencies.keys()) |k| { - try buffer.appendSlice(arena, k); - try buffer.append(arena, '\n'); - } - const s = std.fs.path.sep_str; - const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); - local_cache_directory.handle.writeFile(.{ - .sub_path = tmp_sub_path, - .data = buffer.items, - .flags = .{ .exclusive = true }, - }) catch |err| { - fatal("unable to write configuration results to '{}{s}': {s}", .{ - local_cache_directory, tmp_sub_path, @errorName(err), - }); - }; - process.exit(3); // Indicate configure phase failed with meaningful stdout. - } - - if (builder.validateUserInputDidItFail()) { - fatal(" access the help menu with 'zig build -h'", .{}); - } - - validateSystemLibraryOptions(builder); - - const stdout_writer = io.getStdOut().writer(); - - if (help_menu) - return usage(builder, stdout_writer); - - if (steps_menu) - return steps(builder, stdout_writer); - - var run: Run = .{ - .max_rss = max_rss, - .max_rss_is_default = false, - .max_rss_mutex = .{}, - .skip_oom_steps = skip_oom_steps, - .watch = watch, - .fuzz = fuzz, - .memory_blocked_steps = std.ArrayList(*Step).init(arena), - .step_stack = .{}, - .prominent_compile_errors = prominent_compile_errors, - - .claimed_rss = 0, - .summary = summary orelse if (watch) .new else .failures, - .ttyconf = ttyconf, - .stderr = stderr, - .thread_pool = undefined, - }; - - if (run.max_rss == 0) { - run.max_rss = process.totalSystemMemory() catch std.math.maxInt(u64); - run.max_rss_is_default = true; - } - - const gpa = arena; - prepare(gpa, arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { - error.UncleanExit => process.exit(1), - else => return err, - }; - - var w = if (watch) try Watch.init() else undefined; - - try run.thread_pool.init(thread_pool_options); - defer run.thread_pool.deinit(); - - rebuild: while (true) { - runStepNames( - gpa, - builder, - targets.items, - main_progress_node, - &run, - ) catch |err| switch (err) { - error.UncleanExit => { - assert(!run.watch); - process.exit(1); - }, - else => return err, - }; - if (fuzz) { - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, - // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case - // on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable; - try Fuzz.start( - gpa, - arena, - global_cache_directory, - zig_lib_directory, - zig_exe, - &run.thread_pool, - run.step_stack.keys(), - run.ttyconf, - listen_address, - main_progress_node, - ); - } - - if (!watch) return cleanExit(); - - if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); - - try w.update(gpa, run.step_stack.keys()); - - // Wait until a file system notification arrives. Read all such events - // until the buffer is empty. Then wait for a debounce interval, resetting - // if any more events come in. After the debounce interval has passed, - // trigger a rebuild on all steps with modified inputs, as well as their - // recursive dependants. - var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; - const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), - }) catch &caption_buf; - var debouncing_node = main_progress_node.start(caption, 0); - var debounce_timeout: Watch.Timeout = .none; - while (true) switch (try w.wait(gpa, debounce_timeout)) { - .timeout => { - debouncing_node.end(); - markFailedStepsDirty(gpa, run.step_stack.keys()); - continue :rebuild; - }, - .dirty => if (debounce_timeout == .none) { - debounce_timeout = .{ .ms = debounce_interval_ms }; - debouncing_node.end(); - debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); - }, - .clean => {}, - }; - } -} - -fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { - for (all_steps) |step| switch (step.state) { - .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), - else => continue, - }; - // Now that all dirty steps have been found, the remaining steps that - // succeeded from last run shall be marked "cached". - for (all_steps) |step| switch (step.state) { - .success => step.result_cached = true, - else => continue, - }; -} - -fn countSubProcesses(all_steps: []const *Step) usize { - var count: usize = 0; - for (all_steps) |s| { - count += @intFromBool(s.getZigProcess() != null); - } - return count; -} - -const Run = struct { - max_rss: u64, - max_rss_is_default: bool, - max_rss_mutex: std.Thread.Mutex, - skip_oom_steps: bool, - watch: bool, - fuzz: bool, - memory_blocked_steps: std.ArrayList(*Step), - step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), - prominent_compile_errors: bool, - thread_pool: std.Thread.Pool, - - claimed_rss: usize, - summary: Summary, - ttyconf: std.io.tty.Config, - stderr: File, - - fn cleanExit(run: Run) void { - if (run.watch or run.fuzz) return; - return runner.cleanExit(); - } -}; - -fn prepare( - gpa: Allocator, - arena: Allocator, - b: *std.Build, - step_names: []const []const u8, - run: *Run, - seed: u32, -) !void { - const step_stack = &run.step_stack; - - if (step_names.len == 0) { - try step_stack.put(gpa, b.default_step, {}); - } else { - try step_stack.ensureUnusedCapacity(gpa, step_names.len); - for (0..step_names.len) |i| { - const step_name = step_names[step_names.len - i - 1]; - const s = b.top_level_steps.get(step_name) orelse { - std.debug.print("no step named '{s}'\n access the help menu with 'zig build -h'\n", .{step_name}); - process.exit(1); - }; - step_stack.putAssumeCapacity(&s.step, {}); - } - } - - const starting_steps = try arena.dupe(*Step, step_stack.keys()); - - var rng = std.Random.DefaultPrng.init(seed); - const rand = rng.random(); - rand.shuffle(*Step, starting_steps); - - for (starting_steps) |s| { - constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) { - error.DependencyLoopDetected => return uncleanExit(), - else => |e| return e, - }; - } - - { - // Check that we have enough memory to complete the build. - var any_problems = false; - for (step_stack.keys()) |s| { - if (s.max_rss == 0) continue; - if (s.max_rss > run.max_rss) { - if (run.skip_oom_steps) { - s.state = .skipped_oom; - } else { - std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{ - s.owner.dep_prefix, s.name, s.max_rss, run.max_rss, - }); - any_problems = true; - } - } - } - if (any_problems) { - if (run.max_rss_is_default) { - std.debug.print("note: use --maxrss to override the default", .{}); - } - return uncleanExit(); - } - } -} - -fn runStepNames( - gpa: Allocator, - b: *std.Build, - step_names: []const []const u8, - parent_prog_node: std.Progress.Node, - run: *Run, -) !void { - const step_stack = &run.step_stack; - const thread_pool = &run.thread_pool; - - { - const step_prog = parent_prog_node.start("steps", step_stack.count()); - defer step_prog.end(); - - var wait_group: std.Thread.WaitGroup = .{}; - defer wait_group.wait(); - - // Here we spawn the initial set of tasks with a nice heuristic - - // dependency order. Each worker when it finishes a step will then - // check whether it should run any dependants. - const steps_slice = step_stack.keys(); - for (0..steps_slice.len) |i| { - const step = steps_slice[steps_slice.len - i - 1]; - if (step.state == .skipped_oom) continue; - - thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{ - &wait_group, b, step, step_prog, run, - }); - } - } - assert(run.memory_blocked_steps.items.len == 0); - - var test_skip_count: usize = 0; - var test_fail_count: usize = 0; - var test_pass_count: usize = 0; - var test_leak_count: usize = 0; - var test_count: usize = 0; - - var success_count: usize = 0; - var skipped_count: usize = 0; - var failure_count: usize = 0; - var pending_count: usize = 0; - var total_compile_errors: usize = 0; - - for (step_stack.keys()) |s| { - test_fail_count += s.test_results.fail_count; - test_skip_count += s.test_results.skip_count; - test_leak_count += s.test_results.leak_count; - test_pass_count += s.test_results.passCount(); - test_count += s.test_results.test_count; - - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .running => unreachable, - .precheck_done => { - // precheck_done is equivalent to dependency_failure in the case of - // transitive dependencies. For example: - // A -> B -> C (failure) - // B will be marked as dependency_failure, while A may never be queued, and thus - // remain in the initial state of precheck_done. - s.state = .dependency_failure; - pending_count += 1; - }, - .dependency_failure => pending_count += 1, - .success => success_count += 1, - .skipped, .skipped_oom => skipped_count += 1, - .failure => { - failure_count += 1; - const compile_errors_len = s.result_error_bundle.errorMessageCount(); - if (compile_errors_len > 0) { - total_compile_errors += compile_errors_len; - } - }, - } - } - - // A proper command line application defaults to silently succeeding. - // The user may request verbose mode if they have a different preference. - const failures_only = switch (run.summary) { - .failures, .none => true, - else => false, - }; - if (failure_count == 0 and failures_only) { - return run.cleanExit(); - } - - const ttyconf = run.ttyconf; - - if (run.summary != .none) { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - const stderr = run.stderr; - - const total_count = success_count + failure_count + pending_count + skipped_count; - ttyconf.setColor(stderr, .cyan) catch {}; - stderr.writeAll("Build Summary:") catch {}; - ttyconf.setColor(stderr, .reset) catch {}; - stderr.writer().print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; - if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {}; - if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {}; - - if (test_count > 0) stderr.writer().print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; - if (test_skip_count > 0) stderr.writer().print("; {d} skipped", .{test_skip_count}) catch {}; - if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; - if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; - - stderr.writeAll("\n") catch {}; - - // Print a fancy tree with build results. - var step_stack_copy = try step_stack.clone(gpa); - defer step_stack_copy.deinit(gpa); - - var print_node: PrintNode = .{ .parent = null }; - if (step_names.len == 0) { - print_node.last = true; - printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; - } else { - const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { - var i: usize = step_names.len; - while (i > 0) { - i -= 1; - const step = b.top_level_steps.get(step_names[i]).?.step; - const found = switch (run.summary) { - .all, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk b.top_level_steps.count(); - }; - for (step_names, 0..) |step_name, i| { - const tls = b.top_level_steps.get(step_name).?; - print_node.last = i + 1 == last_index; - printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; - } - } - } - - if (failure_count == 0) { - return run.cleanExit(); - } - - // Finally, render compile errors at the bottom of the terminal. - if (run.prominent_compile_errors and total_compile_errors > 0) { - for (step_stack.keys()) |s| { - if (s.result_error_bundle.errorMessageCount() > 0) { - s.result_error_bundle.renderToStdErr(.{ .ttyconf = ttyconf, .include_reference_trace = (b.reference_trace orelse 0) > 0 }); - } - } - - if (!run.watch) { - // Signal to parent process that we have printed compile errors. The - // parent process may choose to omit the "following command failed" - // line in this case. - std.debug.lockStdErr(); - process.exit(2); - } - } - - if (!run.watch) return uncleanExit(); -} - -const PrintNode = struct { - parent: ?*PrintNode, - last: bool = false, -}; - -fn printPrefix(node: *PrintNode, stderr: File, ttyconf: std.io.tty.Config) !void { - const parent = node.parent orelse return; - if (parent.parent == null) return; - try printPrefix(parent, stderr, ttyconf); - if (parent.last) { - try stderr.writeAll(" "); - } else { - try stderr.writeAll(switch (ttyconf) { - .no_color, .windows_api => "| ", - .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ - }); - } -} - -fn printChildNodePrefix(stderr: File, ttyconf: std.io.tty.Config) !void { - try stderr.writeAll(switch (ttyconf) { - .no_color, .windows_api => "+- ", - .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ - }); -} - -fn printStepStatus( - s: *Step, - stderr: File, - ttyconf: std.io.tty.Config, - run: *const Run, -) !void { - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - .running => unreachable, - - .dependency_failure => { - try ttyconf.setColor(stderr, .dim); - try stderr.writeAll(" transitive failure\n"); - try ttyconf.setColor(stderr, .reset); - }, - - .success => { - try ttyconf.setColor(stderr, .green); - if (s.result_cached) { - try stderr.writeAll(" cached"); - } else if (s.test_results.test_count > 0) { - const pass_count = s.test_results.passCount(); - try stderr.writer().print(" {d} passed", .{pass_count}); - if (s.test_results.skip_count > 0) { - try ttyconf.setColor(stderr, .yellow); - try stderr.writer().print(" {d} skipped", .{s.test_results.skip_count}); - } - } else { - try stderr.writeAll(" success"); - } - try ttyconf.setColor(stderr, .reset); - if (s.result_duration_ns) |ns| { - try ttyconf.setColor(stderr, .dim); - if (ns >= std.time.ns_per_min) { - try stderr.writer().print(" {d}m", .{ns / std.time.ns_per_min}); - } else if (ns >= std.time.ns_per_s) { - try stderr.writer().print(" {d}s", .{ns / std.time.ns_per_s}); - } else if (ns >= std.time.ns_per_ms) { - try stderr.writer().print(" {d}ms", .{ns / std.time.ns_per_ms}); - } else if (ns >= std.time.ns_per_us) { - try stderr.writer().print(" {d}us", .{ns / std.time.ns_per_us}); - } else { - try stderr.writer().print(" {d}ns", .{ns}); - } - try ttyconf.setColor(stderr, .reset); - } - if (s.result_peak_rss != 0) { - const rss = s.result_peak_rss; - try ttyconf.setColor(stderr, .dim); - if (rss >= 1000_000_000) { - try stderr.writer().print(" MaxRSS:{d}G", .{rss / 1000_000_000}); - } else if (rss >= 1000_000) { - try stderr.writer().print(" MaxRSS:{d}M", .{rss / 1000_000}); - } else if (rss >= 1000) { - try stderr.writer().print(" MaxRSS:{d}K", .{rss / 1000}); - } else { - try stderr.writer().print(" MaxRSS:{d}B", .{rss}); - } - try ttyconf.setColor(stderr, .reset); - } - try stderr.writeAll("\n"); - }, - .skipped, .skipped_oom => |skip| { - try ttyconf.setColor(stderr, .yellow); - try stderr.writeAll(" skipped"); - if (skip == .skipped_oom) { - try stderr.writeAll(" (not enough memory)"); - try ttyconf.setColor(stderr, .dim); - try stderr.writer().print(" upper bound of {d} exceeded runner limit ({d})", .{ s.max_rss, run.max_rss }); - try ttyconf.setColor(stderr, .yellow); - } - try stderr.writeAll("\n"); - try ttyconf.setColor(stderr, .reset); - }, - .failure => try printStepFailure(s, stderr, ttyconf), - } -} - -fn printStepFailure( - s: *Step, - stderr: File, - ttyconf: std.io.tty.Config, -) !void { - if (s.result_error_bundle.errorMessageCount() > 0) { - try ttyconf.setColor(stderr, .red); - try stderr.writer().print(" {d} errors\n", .{ - s.result_error_bundle.errorMessageCount(), - }); - try ttyconf.setColor(stderr, .reset); - } else if (!s.test_results.isSuccess()) { - try stderr.writer().print(" {d}/{d} passed", .{ - s.test_results.passCount(), s.test_results.test_count, - }); - if (s.test_results.fail_count > 0) { - try stderr.writeAll(", "); - try ttyconf.setColor(stderr, .red); - try stderr.writer().print("{d} failed", .{ - s.test_results.fail_count, - }); - try ttyconf.setColor(stderr, .reset); - } - if (s.test_results.skip_count > 0) { - try stderr.writeAll(", "); - try ttyconf.setColor(stderr, .yellow); - try stderr.writer().print("{d} skipped", .{ - s.test_results.skip_count, - }); - try ttyconf.setColor(stderr, .reset); - } - if (s.test_results.leak_count > 0) { - try stderr.writeAll(", "); - try ttyconf.setColor(stderr, .red); - try stderr.writer().print("{d} leaked", .{ - s.test_results.leak_count, - }); - try ttyconf.setColor(stderr, .reset); - } - try stderr.writeAll("\n"); - } else if (s.result_error_msgs.items.len > 0) { - try ttyconf.setColor(stderr, .red); - try stderr.writeAll(" failure\n"); - try ttyconf.setColor(stderr, .reset); - } else { - assert(s.result_stderr.len > 0); - try ttyconf.setColor(stderr, .red); - try stderr.writeAll(" stderr\n"); - try ttyconf.setColor(stderr, .reset); - } -} - -fn printTreeStep( - b: *std.Build, - s: *Step, - run: *const Run, - stderr: File, - ttyconf: std.io.tty.Config, - parent_node: *PrintNode, - step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), -) !void { - const first = step_stack.swapRemove(s); - const summary = run.summary; - const skip = switch (summary) { - .none => unreachable, - .all => false, - .new => s.result_cached, - .failures => s.state == .success, - }; - if (skip) return; - try printPrefix(parent_node, stderr, ttyconf); - - if (!first) try ttyconf.setColor(stderr, .dim); - if (parent_node.parent != null) { - if (parent_node.last) { - try printChildNodePrefix(stderr, ttyconf); - } else { - try stderr.writeAll(switch (ttyconf) { - .no_color, .windows_api => "+- ", - .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ - }); - } - } - - // dep_prefix omitted here because it is redundant with the tree. - try stderr.writeAll(s.name); - - if (first) { - try printStepStatus(s, stderr, ttyconf, run); - - const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { - var i: usize = s.dependencies.items.len; - while (i > 0) { - i -= 1; - - const step = s.dependencies.items[i]; - const found = switch (summary) { - .all, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk s.dependencies.items.len -| 1; - }; - for (s.dependencies.items, 0..) |dep, i| { - var print_node: PrintNode = .{ - .parent = parent_node, - .last = i == last_index, - }; - try printTreeStep(b, dep, run, stderr, ttyconf, &print_node, step_stack); - } - } else { - if (s.dependencies.items.len == 0) { - try stderr.writeAll(" (reused)\n"); - } else { - try stderr.writer().print(" (+{d} more reused dependencies)\n", .{ - s.dependencies.items.len, - }); - } - try ttyconf.setColor(stderr, .reset); - } -} - -/// Traverse the dependency graph depth-first and make it undirected by having -/// steps know their dependants (they only know dependencies at start). -/// Along the way, check that there is no dependency loop, and record the steps -/// in traversal order in `step_stack`. -/// Each step has its dependencies traversed in random order, this accomplishes -/// two things: -/// - `step_stack` will be in randomized-depth-first order, so the build runner -/// spawns steps in a random (but optimized) order -/// - each step's `dependants` list is also filled in a random order, so that -/// when it finishes executing in `workerMakeOneStep`, it spawns next steps -/// to run in random order -fn constructGraphAndCheckForDependencyLoop( - b: *std.Build, - s: *Step, - step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), - rand: std.Random, -) !void { - switch (s.state) { - .precheck_started => { - std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); - return error.DependencyLoopDetected; - }, - .precheck_unstarted => { - s.state = .precheck_started; - - try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len); - - // We dupe to avoid shuffling the steps in the summary, it depends - // on s.dependencies' order. - const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM"); - rand.shuffle(*Step, deps); - - for (deps) |dep| { - try step_stack.put(b.allocator, dep, {}); - try dep.dependants.append(b.allocator, s); - constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| { - if (err == error.DependencyLoopDetected) { - std.debug.print(" {s}\n", .{s.name}); - } - return err; - }; - } - - s.state = .precheck_done; - }, - .precheck_done => {}, - - // These don't happen until we actually run the step graph. - .dependency_failure => unreachable, - .running => unreachable, - .success => unreachable, - .failure => unreachable, - .skipped => unreachable, - .skipped_oom => unreachable, - } -} - -fn workerMakeOneStep( - wg: *std.Thread.WaitGroup, - b: *std.Build, - s: *Step, - prog_node: std.Progress.Node, - run: *Run, -) void { - const thread_pool = &run.thread_pool; - - // First, check the conditions for running this step. If they are not met, - // then we return without doing the step, relying on another worker to - // queue this step up again when dependencies are met. - for (s.dependencies.items) |dep| { - switch (@atomicLoad(Step.State, &dep.state, .seq_cst)) { - .success, .skipped => continue, - .failure, .dependency_failure, .skipped_oom => { - @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst); - return; - }, - .precheck_done, .running => { - // dependency is not finished yet. - return; - }, - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - } - } - - if (s.max_rss != 0) { - run.max_rss_mutex.lock(); - defer run.max_rss_mutex.unlock(); - - // Avoid running steps twice. - if (s.state != .precheck_done) { - // Another worker got the job. - return; - } - - const new_claimed_rss = run.claimed_rss + s.max_rss; - if (new_claimed_rss > run.max_rss) { - // Running this step right now could possibly exceed the allotted RSS. - // Add this step to the queue of memory-blocked steps. - run.memory_blocked_steps.append(s) catch @panic("OOM"); - return; - } - - run.claimed_rss = new_claimed_rss; - s.state = .running; - } else { - // Avoid running steps twice. - if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .seq_cst, .seq_cst) != null) { - // Another worker got the job. - return; - } - } - - const sub_prog_node = prog_node.start(s.name, 0); - defer sub_prog_node.end(); - - const make_result = s.make(.{ - .progress_node = sub_prog_node, - .thread_pool = thread_pool, - .watch = run.watch, - }); - - // No matter the result, we want to display error/warning messages. - const show_compile_errors = !run.prominent_compile_errors and - s.result_error_bundle.errorMessageCount() > 0; - const show_error_msgs = s.result_error_msgs.items.len > 0; - const show_stderr = s.result_stderr.len > 0; - - if (show_error_msgs or show_compile_errors or show_stderr) { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - - const gpa = b.allocator; - const options: std.zig.ErrorBundle.RenderOptions = .{ - .ttyconf = run.ttyconf, - .include_reference_trace = (b.reference_trace orelse 0) > 0, - }; - printErrorMessages(gpa, s, options, run.stderr, run.prominent_compile_errors) catch {}; - } - - handle_result: { - if (make_result) |_| { - @atomicStore(Step.State, &s.state, .success, .seq_cst); - } else |err| switch (err) { - error.MakeFailed => { - @atomicStore(Step.State, &s.state, .failure, .seq_cst); - break :handle_result; - }, - error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .seq_cst), - } - - // Successful completion of a step, so we queue up its dependants as well. - for (s.dependants.items) |dep| { - thread_pool.spawnWg(wg, workerMakeOneStep, .{ - wg, b, dep, prog_node, run, - }); - } - } - - // If this is a step that claims resources, we must now queue up other - // steps that are waiting for resources. - if (s.max_rss != 0) { - run.max_rss_mutex.lock(); - defer run.max_rss_mutex.unlock(); - - // Give the memory back to the scheduler. - run.claimed_rss -= s.max_rss; - // Avoid kicking off too many tasks that we already know will not have - // enough resources. - var remaining = run.max_rss - run.claimed_rss; - var i: usize = 0; - var j: usize = 0; - while (j < run.memory_blocked_steps.items.len) : (j += 1) { - const dep = run.memory_blocked_steps.items[j]; - assert(dep.max_rss != 0); - if (dep.max_rss <= remaining) { - remaining -= dep.max_rss; - - thread_pool.spawnWg(wg, workerMakeOneStep, .{ - wg, b, dep, prog_node, run, - }); - } else { - run.memory_blocked_steps.items[i] = dep; - i += 1; - } - } - run.memory_blocked_steps.shrinkRetainingCapacity(i); - } -} - -pub fn printErrorMessages( - gpa: Allocator, - failing_step: *Step, - options: std.zig.ErrorBundle.RenderOptions, - stderr: File, - prominent_compile_errors: bool, -) !void { - // Provide context for where these error messages are coming from by - // printing the corresponding Step subtree. - - var step_stack: std.ArrayListUnmanaged(*Step) = .empty; - defer step_stack.deinit(gpa); - try step_stack.append(gpa, failing_step); - while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { - try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); - } - - // Now, `step_stack` has the subtree that we want to print, in reverse order. - const ttyconf = options.ttyconf; - try ttyconf.setColor(stderr, .dim); - var indent: usize = 0; - while (step_stack.pop()) |s| : (indent += 1) { - if (indent > 0) { - try stderr.writer().writeByteNTimes(' ', (indent - 1) * 3); - try printChildNodePrefix(stderr, ttyconf); - } - - try stderr.writeAll(s.name); - - if (s == failing_step) { - try printStepFailure(s, stderr, ttyconf); - } else { - try stderr.writeAll("\n"); - } - } - try ttyconf.setColor(stderr, .reset); - - if (failing_step.result_stderr.len > 0) { - try stderr.writeAll(failing_step.result_stderr); - if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { - try stderr.writeAll("\n"); - } - } - - if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) { - try failing_step.result_error_bundle.renderToWriter(options, stderr.writer()); - } - - for (failing_step.result_error_msgs.items) |msg| { - try ttyconf.setColor(stderr, .red); - try stderr.writeAll("error: "); - try ttyconf.setColor(stderr, .reset); - try stderr.writeAll(msg); - try stderr.writeAll("\n"); - } -} - -fn steps(builder: *std.Build, out_stream: anytype) !void { - const allocator = builder.allocator; - for (builder.top_level_steps.values()) |top_level_step| { - const name = if (&top_level_step.step == builder.default_step) - try fmt.allocPrint(allocator, "{s} (default)", .{top_level_step.step.name}) - else - top_level_step.step.name; - try out_stream.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); - } -} - -fn usage(b: *std.Build, out_stream: anytype) !void { - try out_stream.print( - \\Usage: {s} build [steps] [options] - \\ - \\Steps: - \\ - , .{b.graph.zig_exe}); - try steps(b, out_stream); - - try out_stream.writeAll( - \\ - \\General Options: - \\ -p, --prefix [path] Where to install files (default: zig-out) - \\ --prefix-lib-dir [path] Where to install libraries - \\ --prefix-exe-dir [path] Where to install executables - \\ --prefix-include-dir [path] Where to install C header files - \\ - \\ --release[=mode] Request release mode, optionally specifying a - \\ preferred optimization mode: fast, safe, small - \\ - \\ -fdarling, -fno-darling Integration with system-installed Darling to - \\ execute macOS programs on Linux hosts - \\ (default: no) - \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute - \\ foreign-architecture programs on Linux hosts - \\ (default: no) - \\ --glibc-runtimes [path] Enhances QEMU integration by providing glibc built - \\ for multiple foreign architectures, allowing - \\ execution of non-native programs that link with glibc. - \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on - \\ ARM64 macOS hosts. (default: no) - \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to - \\ execute WASI binaries. (default: no) - \\ -fwine, -fno-wine Integration with system-installed Wine to execute - \\ Windows programs on Linux hosts. (default: no) - \\ - \\ -h, --help Print this help and exit - \\ -l, --list-steps Print available steps - \\ --verbose Print commands before executing them - \\ --color [auto|off|on] Enable or disable colored error messages - \\ --prominent-compile-errors Buffer compile errors and display at end - \\ --summary [mode] Control the printing of the build summary - \\ all Print the build summary in its entirety - \\ new Omit cached steps - \\ failures (Default) Only print failed steps - \\ none Do not print the build summary - \\ -j Limit concurrent jobs (default is to use all CPU cores) - \\ --maxrss Limit memory usage (default is to use available memory) - \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss - \\ --fetch Exit after fetching dependency tree - \\ --watch Continuously rebuild when source files are modified - \\ --fuzz Continuously search for unit test failures - \\ --debounce Delay before rebuilding after changed file detected - \\ -fincremental Enable incremental compilation - \\ -fno-incremental Disable incremental compilation - \\ - \\Project-Specific Options: - \\ - ); - - const arena = b.allocator; - if (b.available_options_list.items.len == 0) { - try out_stream.print(" (none)\n", .{}); - } else { - for (b.available_options_list.items) |option| { - const name = try fmt.allocPrint(arena, " -D{s}=[{s}]", .{ - option.name, - @tagName(option.type_id), - }); - try out_stream.print("{s:<30} {s}\n", .{ name, option.description }); - if (option.enum_options) |enum_options| { - const padding = " " ** 33; - try out_stream.writeAll(padding ++ "Supported Values:\n"); - for (enum_options) |enum_option| { - try out_stream.print(padding ++ " {s}\n", .{enum_option}); - } - } - } - } - - try out_stream.writeAll( - \\ - \\System Integration Options: - \\ --search-prefix [path] Add a path to look for binaries, libraries, headers - \\ --sysroot [path] Set the system root directory (usually /) - \\ --libc [file] Provide a file which specifies libc paths - \\ - \\ --system [pkgdir] Disable package fetching; enable all integrations - \\ -fsys=[name] Enable a system integration - \\ -fno-sys=[name] Disable a system integration - \\ - \\ Available System Integrations: Enabled: - \\ - ); - if (b.graph.system_library_options.entries.len == 0) { - try out_stream.writeAll(" (none) -\n"); - } else { - for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { - const status = switch (v) { - .declared_enabled => "yes", - .declared_disabled => "no", - .user_enabled, .user_disabled => unreachable, // already emitted error - }; - try out_stream.print(" {s:<43} {s}\n", .{ k, status }); - } - } - - try out_stream.writeAll( - \\ - \\Advanced Options: - \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error - \\ -fno-reference-trace Disable reference trace - \\ -fallow-so-scripts Allows .so files to be GNU ld scripts - \\ -fno-allow-so-scripts (default) .so files must be ELF files - \\ --build-file [file] Override path to build.zig - \\ --cache-dir [path] Override path to local Zig cache directory - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner - \\ --seed [integer] For shuffling dependency traversal order (default: random) - \\ --debug-log [scope] Enable debugging the compiler - \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-rt Debug compiler runtime libraries - \\ --verbose-link Enable compiler debug output for linking - \\ --verbose-air Enable compiler debug output for Zig AIR - \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR - \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC - \\ --verbose-cimport Enable compiler debug output for C imports - \\ --verbose-cc Enable compiler debug output for C compilation - \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features - \\ - ); -} - -fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { - if (idx.* >= args.len) return null; - defer idx.* += 1; - return args[idx.*]; -} - -fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); - process.exit(1); - }; -} - -fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { - if (idx >= args.len) return null; - return args[idx..]; -} - -/// Perhaps in the future there could be an Advanced Options flag such as -/// --debug-build-runner-leaks which would make this function return instead of -/// calling exit. -fn cleanExit() void { - std.debug.lockStdErr(); - process.exit(0); -} - -/// Perhaps in the future there could be an Advanced Options flag such as -/// --debug-build-runner-leaks which would make this function return instead of -/// calling exit. -fn uncleanExit() error{UncleanExit} { - std.debug.lockStdErr(); - process.exit(1); -} - -const Color = std.zig.Color; -const Summary = enum { all, new, failures, none }; - -fn get_tty_conf(color: Color, stderr: File) std.io.tty.Config { - return switch (color) { - .auto => std.io.tty.detectConfig(stderr), - .on => .escape_codes, - .off => .no_color, - }; -} - -fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); - process.exit(1); -} - -fn validateSystemLibraryOptions(b: *std.Build) void { - var bad = false; - for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { - switch (v) { - .user_disabled, .user_enabled => { - // The user tried to enable or disable a system library integration, but - // the build script did not recognize that option. - std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); - bad = true; - }, - .declared_disabled, .declared_enabled => {}, - } - } - if (bad) { - std.debug.print(" access the help menu with 'zig build -h'\n", .{}); - process.exit(1); - } -} - -/// Starting from all top-level steps in `b`, traverses the entire step graph -/// and adds all step dependencies implied by module graphs. -fn createModuleDependencies(b: *std.Build) Allocator.Error!void { - const arena = b.graph.arena; - - var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; - var next_step_idx: usize = 0; - - try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); - for (b.top_level_steps.values()) |tls| { - all_steps.putAssumeCapacityNoClobber(&tls.step, {}); - } - - while (next_step_idx < all_steps.count()) { - const step = all_steps.keys()[next_step_idx]; - next_step_idx += 1; - - // Set up any implied dependencies for this step. It's important that we do this first, so - // that the loop below discovers steps implied by the module graph. - try createModuleDependenciesForStep(step); - - try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); - for (step.dependencies.items) |other_step| { - all_steps.putAssumeCapacity(other_step, {}); - } - } -} - -/// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which -/// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. -fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { - const root_module = if (step.cast(Step.Compile)) |cs| root: { - break :root cs.root_module; - } else return; // not a compile step so no module dependencies - - // Starting from `root_module`, discover all modules in this graph. - const modules = root_module.getGraph().modules; - - // For each of those modules, set up the implied step dependencies. - for (modules) |mod| { - if (mod.root_source_file) |lp| lp.addStepDependencies(step); - for (mod.include_dirs.items) |include_dir| switch (include_dir) { - .path, - .path_system, - .path_after, - .framework_path, - .framework_path_system, - => |lp| lp.addStepDependencies(step), - - .other_step => |other| { - other.getEmittedIncludeTree().addStepDependencies(step); - step.dependOn(&other.step); - }, - - .config_header_step => |other| step.dependOn(&other.step), - }; - for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); - for (mod.rpaths.items) |rpath| switch (rpath) { - .lazy_path => |lp| lp.addStepDependencies(step), - .special => {}, - }; - for (mod.link_objects.items) |link_object| switch (link_object) { - .static_path, - .assembly_file, - => |lp| lp.addStepDependencies(step), - .other_step => |other| step.dependOn(&other.step), - .system_lib => {}, - .c_source_file => |source| source.file.addStepDependencies(step), - .c_source_files => |source_files| source_files.root.addStepDependencies(step), - .win32_resource_file => |rc_source| { - rc_source.file.addStepDependencies(step); - for (rc_source.include_paths) |lp| lp.addStepDependencies(step); - }, - }; - } -} diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig new file mode 100644 index 0000000000000000000000000000000000000000..3a80732dd42a1cbee87020be9ce2b0378486422c --- /dev/null +++ b/lib/compiler/configure_runner.zig @@ -0,0 +1,214 @@ +const builtin = @import("builtin"); + +const std = @import("std"); +const mem = std.mem; +const fatal = std.process.fatal; +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const Step = std.Build.Step; + +pub const root = @import("@build"); +pub const dependencies = @import("@dependencies"); + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .http_disable_tls = true, + .crypto_fork_safety = false, +}; + +comptime { + assert(builtin.single_threaded); +} + +pub fn main() !void { + var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer single_threaded_arena.deinit(); + const arena = single_threaded_arena.allocator(); + + const args = try std.process.argsAlloc(arena); + + // skip my own exe name + var arg_idx: usize = 1; + + const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); + const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); + const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); + const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); + + const zig_lib_directory: std.Build.Cache.Directory = .{ + .path = zig_lib_dir, + .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), + }; + + const build_root_directory: std.Build.Cache.Directory = .{ + .path = build_root, + .handle = try std.fs.cwd().openDir(build_root, .{}), + }; + + const local_cache_directory: std.Build.Cache.Directory = .{ + .path = cache_root, + .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), + }; + + const global_cache_directory: std.Build.Cache.Directory = .{ + .path = global_cache_root, + .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), + }; + + var graph: std.Build.Graph = .{ + .arena = arena, + .cache = .{ + .gpa = arena, + .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), + }, + .zig_exe = zig_exe, + .env_map = try std.process.getEnvMap(arena), + .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + .host = .{ + .query = .{}, + .result = try std.zig.system.resolveTargetQuery(.{}), + }, + }; + + graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); + graph.cache.addPrefix(build_root_directory); + graph.cache.addPrefix(local_cache_directory); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + const builder = try std.Build.create( + &graph, + build_root_directory, + local_cache_directory, + dependencies.root_deps, + ); + + var install_prefix: ?std.Build.Cache.Path = null; + var install_paths: std.Build.InstallPaths = .{}; + + while (nextArg(args, &arg_idx)) |arg| { + if (mem.startsWith(u8, arg, "-D")) { + const option_contents = arg[2..]; + if (option_contents.len == 0) + fatal("expected option name after '-D'", .{}); + if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { + const option_name = option_contents[0..name_end]; + const option_value = option_contents[name_end + 1 ..]; + if (try builder.addUserInputOption(option_name, option_value)) + fatal(" access the help menu with 'zig build -h'", .{}); + } else { + if (try builder.addUserInputFlag(option_contents)) + fatal(" access the help menu with 'zig build -h'", .{}); + } + } else if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { + install_prefix = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { + install_paths.lib_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { + install_paths.exe_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-include-dir")) { + install_paths.include_dir = nextArgOrFatal(args, &arg_idx); + } else { + fatal("unrecognized argument: '{s}'", .{arg}); + } + } else { + fatal("unrecognized argument: '{s}'", .{arg}); + } + } + + builder.resolveInstallPrefix(install_prefix, install_paths); + try builder.runBuild(root); + createModuleDependencies(builder) catch @panic("OOM"); + + try std.io.getStdOut().writeAll("TODO\n"); +} + +fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse fatal("expected argument after '{s}'", .{args[idx.* - 1]}); +} + +/// Starting from all top-level steps in `b`, traverses the entire step graph +/// and adds all step dependencies implied by module graphs. +fn createModuleDependencies(b: *std.Build) Allocator.Error!void { + const arena = b.graph.arena; + + var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; + var next_step_idx: usize = 0; + + try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); + for (b.top_level_steps.values()) |tls| { + all_steps.putAssumeCapacityNoClobber(&tls.step, {}); + } + + while (next_step_idx < all_steps.count()) { + const step = all_steps.keys()[next_step_idx]; + next_step_idx += 1; + + // Set up any implied dependencies for this step. It's important that we do this first, so + // that the loop below discovers steps implied by the module graph. + try createModuleDependenciesForStep(step); + + try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); + for (step.dependencies.items) |other_step| { + all_steps.putAssumeCapacity(other_step, {}); + } + } +} + +/// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which +/// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. +fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { + const root_module = if (step.cast(Step.Compile)) |cs| root: { + break :root cs.root_module; + } else return; // not a compile step so no module dependencies + + // Starting from `root_module`, discover all modules in this graph. + const modules = root_module.getGraph().modules; + + // For each of those modules, set up the implied step dependencies. + for (modules) |mod| { + if (mod.root_source_file) |lp| lp.addStepDependencies(step); + for (mod.include_dirs.items) |include_dir| switch (include_dir) { + .path, + .path_system, + .path_after, + .framework_path, + .framework_path_system, + => |lp| lp.addStepDependencies(step), + + .other_step => |other| { + other.getEmittedIncludeTree().addStepDependencies(step); + step.dependOn(&other.step); + }, + + .config_header_step => |other| step.dependOn(&other.step), + }; + for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); + for (mod.rpaths.items) |rpath| switch (rpath) { + .lazy_path => |lp| lp.addStepDependencies(step), + .special => {}, + }; + for (mod.link_objects.items) |link_object| switch (link_object) { + .static_path, + .assembly_file, + => |lp| lp.addStepDependencies(step), + .other_step => |other| step.dependOn(&other.step), + .system_lib => {}, + .c_source_file => |source| source.file.addStepDependencies(step), + .c_source_files => |source_files| source_files.root.addStepDependencies(step), + .win32_resource_file => |rc_source| { + rc_source.file.addStepDependencies(step); + for (rc_source.include_paths) |lp| lp.addStepDependencies(step); + }, + }; + } +} diff --git a/lib/compiler/fetch.zig b/lib/compiler/fetch.zig new file mode 100644 index 0000000000000000000000000000000000000000..3fc035c1144f0f2d920fdd812b3294db38b67d24 --- /dev/null +++ b/lib/compiler/fetch.zig @@ -0,0 +1,382 @@ +const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("std"); +const mem = std.mem; +const fs = std.fs; +const process = std.process; +const fatal = std.process.fatal; +const Path = std.Build.Cache.Path; +const Directory = std.Build.Cache.Directory; +const Package = std.zig.Package; +const Allocator = std.mem.Allocator; + +const usage = + \\Usage: zig fetch [options] + \\Usage: zig fetch [options] + \\ + \\ Copy a package into the global cache and print its hash. + \\ must point to one of the following: + \\ - A git+http / git+https server for the package + \\ - A tarball file (with or without compression) containing + \\ package source + \\ - A git bundle file containing package source + \\ + \\Examples: + \\ + \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git + \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --debug-hash Print verbose hash information to stdout + \\ --save Add the fetched package to build.zig.zon + \\ --save=[name] Add the fetched package to build.zig.zon as name + \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim + \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim + \\ +; + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .crypto_fork_safety = false, +}; + +pub fn main() !void { + var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + const gpa = arena; + + const args = try process.argsAlloc(arena); + + var zig_lib_directory: Directory = .{ + .handle = try std.fs.cwd().openDir(args[1], .{}), + }; + defer zig_lib_directory.handle.close(); + + var global_cache_directory: Directory = .{ + .handle = try std.fs.cwd().openDir(args[2], .{}), + }; + defer global_cache_directory.handle.close(); + + const color: std.zig.Color = .auto; + const work_around_btrfs_bug = native_os == .linux and std.zig.EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); + var opt_path_or_url: ?[]const u8 = null; + var debug_hash: bool = false; + var save: union(enum) { + no, + yes: ?[]const u8, + exact: ?[]const u8, + } = .no; + + { + var i: usize = 3; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + const stdout = std.io.getStdOut().writer(); + try stdout.writeAll(usage); + return process.cleanExit(); + } else if (mem.eql(u8, arg, "--debug-hash")) { + debug_hash = true; + } else if (mem.eql(u8, arg, "--save")) { + save = .{ .yes = null }; + } else if (mem.startsWith(u8, arg, "--save=")) { + save = .{ .yes = arg["--save=".len..] }; + } else if (mem.eql(u8, arg, "--save-exact")) { + save = .{ .exact = null }; + } else if (mem.startsWith(u8, arg, "--save-exact=")) { + save = .{ .exact = arg["--save-exact=".len..] }; + } else { + fatal("unrecognized parameter: '{s}'", .{arg}); + } + } else if (opt_path_or_url != null) { + fatal("unexpected extra parameter: '{s}'", .{arg}); + } else { + opt_path_or_url = arg; + } + } + } + + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); + + var thread_pool: std.Thread.Pool = undefined; + try thread_pool.init(.{ .allocator = gpa }); + defer thread_pool.deinit(); + + var http_client: std.http.Client = .{ .allocator = gpa }; + defer http_client.deinit(); + + try http_client.initDefaultProxies(arena); + + var root_prog_node = std.Progress.start(.{ + .root_name = "Fetch", + }); + defer root_prog_node.end(); + + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .recursive = false, + .read_only = false, + .debug_hash = debug_hash, + .work_around_btrfs_bug = work_around_btrfs_bug, + }; + defer job_queue.deinit(); + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .allow_missing_fingerprint = true, + .allow_name_string = true, + .use_latest_commit = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + }; + defer fetch.deinit(); + + fetch.run() catch |err| switch (err) { + error.OutOfMemory => fatal("out of memory", .{}), + error.FetchFailed => {}, // error bundle checked below + }; + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStdErr(color.renderOptions()); + process.exit(1); + } + + const package_hash = fetch.computedPackageHash(); + const package_hash_slice = package_hash.toSlice(); + + root_prog_node.end(); + root_prog_node = .{ .index = .none }; + + const name = switch (save) { + .no => { + try std.io.getStdOut().writer().print("{s}\n", .{package_hash_slice}); + return process.cleanExit(); + }, + .yes, .exact => |name| name: { + if (name) |n| break :name n; + const fetched_manifest = fetch.manifest orelse + fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); + break :name fetched_manifest.name; + }, + }; + + const cwd_path = try process.getCwdAlloc(arena); + + var build_root = try Package.findBuildRoot(arena, .{ + .cwd_path = cwd_path, + }); + defer build_root.deinit(); + + // The name to use in case the manifest file needs to be created now. + const init_root_name = std.fs.path.basename(build_root.directory.path orelse cwd_path); + var manifest, var ast = try loadManifest(gpa, arena, zig_lib_directory, .{ + .root_name = try Package.sanitizeExampleName(arena, init_root_name), + .dir = build_root.directory.handle, + .color = color, + }); + defer { + manifest.deinit(gpa); + ast.deinit(gpa); + } + + var fixups: std.zig.Ast.Fixups = .{}; + defer fixups.deinit(gpa); + + var saved_path_or_url = path_or_url; + + if (fetch.latest_commit) |latest_commit| resolved: { + const latest_commit_hex = try std.fmt.allocPrint(arena, "{}", .{latest_commit}); + + var uri = try std.Uri.parse(path_or_url); + + if (uri.fragment) |fragment| { + const target_ref = try fragment.toRawMaybeAlloc(arena); + + // the refspec may already be fully resolved + if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; + + std.log.info("resolved ref '{s}' to commit {s}", .{ target_ref, latest_commit_hex }); + + // include the original refspec in a query parameter, could be used to check for updates + uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={%}", .{fragment}) }; + } else { + std.log.info("resolved to commit {s}", .{latest_commit_hex}); + } + + // replace the refspec with the resolved commit SHA + uri.fragment = .{ .raw = latest_commit_hex }; + + switch (save) { + .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{}", .{uri}), + .no, .exact => {}, // keep the original URL + } + } + + const new_node_init = try std.fmt.allocPrint(arena, + \\.{{ + \\ .url = "{}", + \\ .hash = "{}", + \\ }} + , .{ + std.zig.fmtEscapes(saved_path_or_url), + std.zig.fmtEscapes(package_hash_slice), + }); + + const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{ + std.zig.fmtId(name), new_node_init, + }); + + const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ + new_node_text, + }); + + const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ + dependencies_init, + }); + + if (manifest.dependencies.get(name)) |dep| { + if (dep.hash) |h| { + switch (dep.location) { + .url => |u| { + if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { + std.log.info("existing dependency named '{s}' is up-to-date", .{name}); + process.exit(0); + } + }, + .path => {}, + } + } + + const location_replace = try std.fmt.allocPrint( + arena, + "\"{}\"", + .{std.zig.fmtEscapes(saved_path_or_url)}, + ); + const hash_replace = try std.fmt.allocPrint( + arena, + "\"{}\"", + .{std.zig.fmtEscapes(package_hash_slice)}, + ); + + std.log.warn("overwriting existing dependency named '{s}'", .{name}); + try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); + if (dep.hash_node.unwrap()) |hash_node| { + try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); + } else { + // https://github.com/ziglang/zig/issues/21690 + } + } else if (manifest.dependencies.count() > 0) { + // Add fixup for adding another dependency. + const deps = manifest.dependencies.values(); + const last_dep_node = deps[deps.len - 1].node; + try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); + } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { + // Add fixup for replacing the entire dependencies struct. + try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); + } else { + // Add fixup for adding dependencies struct. + try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); + } + + var rendered = std.ArrayList(u8).init(gpa); + defer rendered.deinit(); + try ast.renderToArrayList(&rendered, fixups); + + build_root.directory.handle.writeFile(.{ .sub_path = Package.Manifest.basename, .data = rendered.items }) catch |err| { + fatal("unable to write {s} file: {s}", .{ Package.Manifest.basename, @errorName(err) }); + }; + + return process.cleanExit(); +} + +const LoadManifestOptions = struct { + root_name: []const u8, + dir: fs.Dir, + color: std.zig.Color, +}; + +fn loadManifest( + gpa: Allocator, + arena: Allocator, + zig_lib_directory: Directory, + options: LoadManifestOptions, +) !struct { Package.Manifest, std.zig.Ast } { + const manifest_bytes = while (true) { + break options.dir.readFileAllocOptions( + arena, + Package.Manifest.basename, + Package.Manifest.max_bytes, + null, + 1, + 0, + ) catch |err| switch (err) { + error.FileNotFound => { + const fingerprint: Package.Fingerprint = .generate(options.root_name); + var templates = Package.Templates.find(gpa, zig_lib_directory); + defer templates.deinit(gpa); + templates.write(arena, options.dir, options.root_name, Package.Manifest.basename, fingerprint) catch |e| { + fatal("unable to write {s}: {s}", .{ + Package.Manifest.basename, @errorName(e), + }); + }; + continue; + }, + else => |e| fatal("unable to load {s}: {s}", .{ + Package.Manifest.basename, @errorName(e), + }), + }; + }; + var ast = try std.zig.Ast.parse(gpa, manifest_bytes, .zon); + errdefer ast.deinit(gpa); + + if (ast.errors.len > 0) { + try std.zig.printAstErrorsToStderr(gpa, ast, Package.Manifest.basename, options.color); + process.exit(2); + } + + var manifest = try Package.Manifest.parse(gpa, ast, .{}); + errdefer manifest.deinit(gpa); + + if (manifest.errors.len > 0) { + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + + const src_path = try wip_errors.addString(Package.Manifest.basename); + try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); + + var error_bundle = try wip_errors.toOwnedBundle(""); + defer error_bundle.deinit(gpa); + error_bundle.renderToStdErr(options.color.renderOptions()); + + process.exit(2); + } + return .{ manifest, ast }; +} diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 4afdcd2bff8c03dc2fe060b0509e054ed730ae73..b5b489da9a57dbaf3815138204f1fe66cf946615 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -31,35 +31,17 @@ allocator: Allocator, user_input_options: UserInputOptionsMap, available_options_map: AvailableOptionsMap, available_options_list: ArrayList(AvailableOption), -verbose: bool, -verbose_link: bool, -verbose_cc: bool, -verbose_air: bool, -verbose_llvm_ir: ?[]const u8, -verbose_llvm_bc: ?[]const u8, -verbose_cimport: bool, -verbose_llvm_cpu_features: bool, -reference_trace: ?u32 = null, invalid_user_input: bool, default_step: *Step, top_level_steps: std.StringArrayHashMapUnmanaged(*TopLevelStep), -install_prefix: []const u8, -dest_dir: ?[]const u8, -lib_dir: []const u8, -exe_dir: []const u8, -h_dir: []const u8, -install_path: []const u8, -sysroot: ?[]const u8 = null, -search_prefixes: std.ArrayListUnmanaged([]const u8), -libc_file: ?[]const u8 = null, +install_prefix: Cache.Path, +install_lib_path: Cache.Path, +install_exe_path: Cache.Path, +install_include_path: Cache.Path, /// Path to the directory containing build.zig. build_root: Cache.Directory, cache_root: Cache.Directory, pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, -args: ?[]const []const u8 = null, -debug_log_scopes: []const []const u8 = &.{}, -debug_compile_errors: bool = false, -debug_pkg_config: bool = false, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. /// Set to 0 to disable stack collection. @@ -75,11 +57,6 @@ enable_rosetta: bool = false, enable_wasmtime: bool = false, /// Use system Wine installation to run cross compiled Windows build artifacts. enable_wine: bool = false, -/// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc, -/// this will be the directory $glibc-build-dir/install/glibcs -/// Given the example of the aarch64 target, this is the directory -/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. -glibc_runtimes_dir: ?[]const u8 = null, dep_prefix: []const u8 = "", @@ -92,8 +69,6 @@ pkg_hash: []const u8, /// A mapping from dependency names to package hashes. available_deps: AvailableDeps, -release_mode: ReleaseMode, - pub const ReleaseMode = enum { off, any, @@ -107,7 +82,7 @@ pub const ReleaseMode = enum { pub const Graph = struct { arena: Allocator, system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, - system_package_mode: bool = false, + system_package_mode: ?Cache.Directory = null, debug_compiler_runtime_libs: bool = false, cache: Cache, zig_exe: [:0]const u8, @@ -121,6 +96,31 @@ pub const Graph = struct { random_seed: u32 = 0, dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, + + release_mode: ReleaseMode, + sysroot: ?[]const u8 = null, + search_prefixes: std.ArrayListUnmanaged([]const u8), + libc_file: ?[]const u8 = null, + debug_compile_errors: bool = false, + /// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc, + /// this will be the directory $glibc-build-dir/install/glibcs + /// Given the example of the aarch64 target, this is the directory + /// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. + glibc_runtimes_dir: ?[]const u8 = null, + verbose: bool, + verbose_link: bool, + verbose_cc: bool, + verbose_air: bool, + verbose_llvm_ir: ?[]const u8, + verbose_llvm_bc: ?[]const u8, + verbose_cimport: bool, + verbose_llvm_cpu_features: bool, + reference_trace: ?u32 = null, + debug_log_scopes: []const []const u8 = &.{}, + + pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { + b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); + } }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -239,10 +239,10 @@ const TopLevelStep = struct { description: []const u8, }; -pub const DirList = struct { - lib_dir: ?[]const u8 = null, - exe_dir: ?[]const u8 = null, - include_dir: ?[]const u8 = null, +pub const InstallPaths = struct { + lib_path: ?Cache.Path = null, + exe_path: ?Cache.Path = null, + include_path: ?Cache.Path = null, }; pub fn create( @@ -259,13 +259,6 @@ pub fn create( .build_root = build_root, .cache_root = cache_root, .verbose = false, - .verbose_link = false, - .verbose_cc = false, - .verbose_air = false, - .verbose_llvm_ir = null, - .verbose_llvm_bc = null, - .verbose_cimport = false, - .verbose_llvm_cpu_features = false, .invalid_user_input = false, .allocator = arena, .user_input_options = UserInputOptionsMap.init(arena), @@ -273,12 +266,10 @@ pub fn create( .available_options_list = ArrayList(AvailableOption).init(arena), .top_level_steps = .{}, .default_step = undefined, - .search_prefixes = .{}, .install_prefix = undefined, .lib_dir = undefined, .exe_dir = undefined, .h_dir = undefined, - .dest_dir = graph.env_map.get("DESTDIR"), .install_tls = .{ .step = Step.init(.{ .id = TopLevelStep.base_id, @@ -297,7 +288,6 @@ pub fn create( .description = "Remove build artifacts from prefix path", }, .install_path = undefined, - .args = null, .modules = .init(arena), .named_writefiles = .init(arena), .named_lazy_paths = .init(arena), @@ -358,37 +348,22 @@ fn createChildOnly( .available_options_map = AvailableOptionsMap.init(allocator), .available_options_list = ArrayList(AvailableOption).init(allocator), .verbose = parent.verbose, - .verbose_link = parent.verbose_link, - .verbose_cc = parent.verbose_cc, - .verbose_air = parent.verbose_air, - .verbose_llvm_ir = parent.verbose_llvm_ir, - .verbose_llvm_bc = parent.verbose_llvm_bc, - .verbose_cimport = parent.verbose_cimport, - .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, - .reference_trace = parent.reference_trace, .invalid_user_input = false, .default_step = undefined, .top_level_steps = .{}, .install_prefix = undefined, - .dest_dir = parent.dest_dir, .lib_dir = parent.lib_dir, .exe_dir = parent.exe_dir, .h_dir = parent.h_dir, .install_path = parent.install_path, .sysroot = parent.sysroot, - .search_prefixes = parent.search_prefixes, - .libc_file = parent.libc_file, .build_root = build_root, .cache_root = parent.cache_root, - .debug_log_scopes = parent.debug_log_scopes, - .debug_compile_errors = parent.debug_compile_errors, - .debug_pkg_config = parent.debug_pkg_config, .enable_darling = parent.enable_darling, .enable_qemu = parent.enable_qemu, .enable_rosetta = parent.enable_rosetta, .enable_wasmtime = parent.enable_wasmtime, .enable_wine = parent.enable_wine, - .glibc_runtimes_dir = parent.glibc_runtimes_dir, .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), .modules = .init(allocator), .named_writefiles = .init(allocator), @@ -638,42 +613,16 @@ fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { const digest = hash.final(); const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); - b.resolveInstallPrefix(install_prefix, .{}); + try b.resolveInstallPrefix(install_prefix, .{}); } -/// This function is intended to be called by lib/build_runner.zig, not a build.zig file. -pub fn resolveInstallPrefix(b: *Build, install_prefix: ?[]const u8, dir_list: DirList) void { - if (b.dest_dir) |dest_dir| { - b.install_prefix = install_prefix orelse "/usr"; - b.install_path = b.pathJoin(&.{ dest_dir, b.install_prefix }); - } else { - b.install_prefix = install_prefix orelse - (b.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); - b.install_path = b.install_prefix; - } +fn resolveInstallPrefix(b: *Build, install_prefix: Cache.Path, paths: InstallPaths) !void { + const arena = b.allocator; - var lib_list = [_][]const u8{ b.install_path, "lib" }; - var exe_list = [_][]const u8{ b.install_path, "bin" }; - var h_list = [_][]const u8{ b.install_path, "include" }; - - if (dir_list.lib_dir) |dir| { - if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; - lib_list[1] = dir; - } - - if (dir_list.exe_dir) |dir| { - if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; - exe_list[1] = dir; - } - - if (dir_list.include_dir) |dir| { - if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; - h_list[1] = dir; - } - - b.lib_dir = b.pathJoin(&lib_list); - b.exe_dir = b.pathJoin(&exe_list); - b.h_dir = b.pathJoin(&h_list); + b.install_prefix = install_prefix; + b.install_lib_path = paths.lib_path orelse try install_prefix.join(arena, "lib"); + b.install_exe_path = paths.exe_path orelse try install_prefix.join(arena, "bin"); + b.install_include_path = paths.include_path orelse try install_prefix.join(arena, "include"); } /// Create a set of key-value pairs that can be converted into a Zig source @@ -1990,38 +1939,6 @@ fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { return null; } -pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) error{FileNotFound}![]const u8 { - // TODO report error for ambiguous situations - for (b.search_prefixes.items) |search_prefix| { - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - return tryFindProgram(b, b.pathJoin(&.{ search_prefix, "bin", name })) orelse continue; - } - } - if (b.graph.env_map.get("PATH")) |PATH| { - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter); - while (it.next()) |p| { - return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; - } - } - } - for (names) |name| { - if (fs.path.isAbsolute(name)) { - return name; - } - for (paths) |p| { - return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; - } - } - return error.FileNotFound; -} - pub fn runAllowFail( b: *Build, argv: []const []const u8, @@ -2085,10 +2002,6 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { }; } -pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { - b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); -} - pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix const base_dir = switch (dir) { diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 23001554744df403883aba81d69b04954b5c40ee..b346138e9aad44b64a7766003e2782405965bb5e 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -25,6 +25,7 @@ pub const WindowsSdk = @import("zig/WindowsSdk.zig"); pub const LibCDirs = @import("zig/LibCDirs.zig"); pub const target = @import("zig/target.zig"); pub const llvm = @import("zig/llvm.zig"); +pub const Package = @import("zig/Package.zig"); // Character literal parsing pub const ParsedCharLiteral = string_literal.ParsedCharLiteral; diff --git a/lib/std/zig/Package.zig b/lib/std/zig/Package.zig new file mode 100644 index 0000000000000000000000000000000000000000..a65ef0d08e57404442d26977c7e396fde3d7a7ec --- /dev/null +++ b/lib/std/zig/Package.zig @@ -0,0 +1,307 @@ +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +pub const Fetch = @import("Package/Fetch.zig"); +pub const build_zig_basename = "build.zig"; +pub const Manifest = @import("Package/Manifest.zig"); + +pub const multihash_len = 1 + 1 + Hash.Algo.digest_length; +pub const multihash_hex_digest_len = 2 * multihash_len; +pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; + +pub const Fingerprint = packed struct(u64) { + id: u32, + checksum: u32, + + pub fn generate(name: []const u8) Fingerprint { + return .{ + .id = std.crypto.random.intRangeLessThan(u32, 1, 0xffffffff), + .checksum = std.hash.Crc32.hash(name), + }; + } + + pub fn validate(n: Fingerprint, name: []const u8) bool { + switch (n.id) { + 0x00000000, 0xffffffff => return false, + else => return std.hash.Crc32.hash(name) == n.checksum, + } + } + + pub fn int(n: Fingerprint) u64 { + return @bitCast(n); + } +}; + +/// A user-readable, file system safe hash that identifies an exact package +/// snapshot, including file contents. +/// +/// The hash is not only to prevent collisions but must resist attacks where +/// the adversary fully controls the contents being hashed. Thus, it contains +/// a full SHA-256 digest. +/// +/// This data structure can be used to store the legacy hash format too. Legacy +/// hash format is scheduled to be removed after 0.14.0 is tagged. +/// +/// There's also a third way this structure is used. When using path rather than +/// hash, a unique hash is still needed, so one is computed based on the path. +pub const Hash = struct { + /// Maximum size of a package hash. Unused bytes at the end are + /// filled with zeroes. + bytes: [max_len]u8, + + pub const Algo = std.crypto.hash.sha2.Sha256; + pub const Digest = [Algo.digest_length]u8; + + /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" + pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; + + pub fn fromSlice(s: []const u8) Hash { + assert(s.len <= max_len); + var result: Hash = undefined; + @memcpy(result.bytes[0..s.len], s); + @memset(result.bytes[s.len..], 0); + return result; + } + + pub fn toSlice(ph: *const Hash) []const u8 { + var end: usize = ph.bytes.len; + while (true) { + end -= 1; + if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; + } + } + + pub fn eql(a: *const Hash, b: *const Hash) bool { + return std.mem.eql(u8, &a.bytes, &b.bytes); + } + + /// Distinguishes whether the legacy multihash format is being stored here. + pub fn isOld(h: *const Hash) bool { + if (h.bytes.len < 2) return false; + const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false; + if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false; + if (h.toSlice().len != multihash_hex_digest_len) return false; + return std.mem.indexOfScalar(u8, &h.bytes, '-') == null; + } + + test isOld { + const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7"); + try std.testing.expect(h.isOld()); + } + + /// Produces "$name-$semver-$hashplus". + /// * name is the name field from build.zig.zon, asserted to be at most 32 + /// bytes and assumed be a valid zig identifier + /// * semver is the version field from build.zig.zon, asserted to be at + /// most 32 bytes + /// * hashplus is the following 33-byte array, base64 encoded using -_ to make + /// it filesystem safe: + /// - (4 bytes) LE u32 Package ID + /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated + /// - (25 bytes) truncated SHA-256 digest of hashed files of the package + pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { + assert(name.len <= 32); + assert(ver.len <= 32); + var result: Hash = undefined; + var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes); + buf.appendSliceAssumeCapacity(name); + buf.appendAssumeCapacity('-'); + buf.appendSliceAssumeCapacity(ver); + buf.appendAssumeCapacity('-'); + var hashplus: [33]u8 = undefined; + std.mem.writeInt(u32, hashplus[0..4], id, .little); + std.mem.writeInt(u32, hashplus[4..8], size, .little); + hashplus[8..].* = digest[0..25].*; + _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); + @memset(buf.unusedCapacitySlice(), 0); + return result; + } + + /// Produces a unique hash based on the path provided. The result should + /// not be user-visible. + pub fn initPath(sub_path: []const u8, is_global: bool) Hash { + var result: Hash = .{ .bytes = @splat(0) }; + var i: usize = 0; + if (is_global) { + result.bytes[0] = '/'; + i += 1; + } + if (i + sub_path.len <= result.bytes.len) { + @memcpy(result.bytes[i..][0..sub_path.len], sub_path); + return result; + } + var bin_digest: [Algo.digest_length]u8 = undefined; + Algo.hash(sub_path, &bin_digest, .{}); + _ = std.fmt.bufPrint(result.bytes[i..], "{}", .{std.fmt.fmtSliceHexLower(&bin_digest)}) catch unreachable; + return result; + } +}; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub const multihash_function: MultihashFunction = switch (Hash.Algo) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => unreachable, +}; + +pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest { + const hex_charset = std.fmt.hex_charset; + + var result: MultiHashHexDigest = undefined; + + result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; + result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; + + result[2] = hex_charset[Hash.Algo.digest_length >> 4]; + result[3] = hex_charset[Hash.Algo.digest_length & 15]; + + for (digest, 0..) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +comptime { + // We avoid unnecessary uleb128 code in hexDigest by asserting here the + // values are small enough to be contained in the one-byte encoding. + assert(@intFromEnum(multihash_function) < 127); + assert(Hash.Algo.digest_length < 127); +} + +test Hash { + const example_digest: Hash.Digest = .{ + 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, + 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, + }; + const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); + try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); +} + +pub fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 { + var result: std.ArrayListUnmanaged(u8) = .empty; + for (bytes, 0..) |byte, i| switch (byte) { + '0'...'9' => { + if (i == 0) try result.append(arena, '_'); + try result.append(arena, byte); + }, + '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte), + '-', '.', ' ' => try result.append(arena, '_'), + else => continue, + }; + if (!std.zig.isValidId(result.items)) return "foo"; + if (result.items.len > Manifest.max_name_len) + result.shrinkRetainingCapacity(Manifest.max_name_len); + + return result.toOwnedSlice(arena); +} + +test sanitizeExampleName { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!")); + try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a")); + try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!")); + try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test")); + try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests")); + try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); +} + +pub const BuildRoot = struct { + directory: std.Build.Cache.Directory, + build_zig_basename: []const u8, + cleanup_build_dir: ?std.fs.Dir, + + fn deinit(br: *BuildRoot) void { + if (br.cleanup_build_dir) |*dir| dir.close(); + br.* = undefined; + } +}; + +pub const FindBuildRootOptions = struct { + build_file: ?[]const u8 = null, + cwd_path: ?[]const u8 = null, +}; + +pub fn findBuildRoot(arena: Allocator, options: FindBuildRootOptions) !BuildRoot { + const cwd_path = options.cwd_path orelse try std.process.getCwdAlloc(arena); + const basename = if (options.build_file) |bf| std.fs.path.basename(bf) else build_zig_basename; + + if (options.build_file) |bf| { + if (std.fs.path.dirname(bf)) |dirname| { + const dir = std.fs.cwd().openDir(dirname, .{}) catch |err| { + std.process.fatal("unable to open directory to build file from argument 'build-file', '{s}': {s}", .{ dirname, @errorName(err) }); + }; + return .{ + .build_zig_basename = basename, + .directory = .{ .path = dirname, .handle = dir }, + .cleanup_build_dir = dir, + }; + } + + return .{ + .build_zig_basename = basename, + .directory = .{ .path = null, .handle = std.fs.cwd() }, + .cleanup_build_dir = null, + }; + } + // Search up parent directories until we find build.zig. + var dirname: []const u8 = cwd_path; + while (true) { + const joined_path = try std.fs.path.join(arena, &[_][]const u8{ dirname, basename }); + if (std.fs.cwd().access(joined_path, .{})) |_| { + const dir = std.fs.cwd().openDir(dirname, .{}) catch |err| { + std.process.fatal("unable to open directory while searching for {s} file, '{s}': {s}", .{ + basename, dirname, @errorName(err), + }); + }; + return .{ + .build_zig_basename = basename, + .directory = .{ + .path = dirname, + .handle = dir, + }, + .cleanup_build_dir = dir, + }; + } else |err| switch (err) { + error.FileNotFound => { + dirname = std.fs.path.dirname(dirname) orelse { + std.log.info("initialize {s} template file with 'zig init'", .{basename}); + std.log.info("see 'zig --help' for more options", .{}); + std.process.fatal("no {s} file found, in the current directory or any parent directories", .{ + basename, + }); + }; + continue; + }, + else => |e| return e, + } + } +} + +test { + _ = Fetch; +} diff --git a/lib/std/zig/Package/Fetch.zig b/lib/std/zig/Package/Fetch.zig new file mode 100644 index 0000000000000000000000000000000000000000..593e8178c5f2e6b6df907f5369a4b251920300f2 --- /dev/null +++ b/lib/std/zig/Package/Fetch.zig @@ -0,0 +1,2413 @@ +//! Represents one independent job whose responsibility is to: +//! +//! 1. Check the global zig package cache to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 8. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. +//! 2. Fetch and unpack a URL into a temporary directory. +//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed +//! for the file to be missing, in which case this fetched package is considered +//! to be a "naked" package. +//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by +//! deleting excluded files. If any files had errors for files that were +//! ultimately excluded, those errors should be ignored, such as failure to +//! create symlinks that weren't supposed to be included anyway. +//! 5. Compute the package hash based on the remaining files in the temporary +//! directory. +//! 6. Rename the temporary directory into the global zig package cache +//! directory. If the hash already exists, delete the temporary directory and +//! leave the zig package cache directory untouched as it may be in use by the +//! system. This is done even if the hash is invalid, in case the package with +//! the different hash is used in the future. +//! 7. Validate the computed hash against the expected hash. If invalid, +//! this job is done. +//! 8. Spawn a new fetch job for each dependency in the manifest file. Use +//! a mutex and a hash map so that redundant jobs do not get queued up. +//! +//! All of this must be done with only referring to the state inside this struct +//! because this work will be done in a dedicated thread. + +arena: std.heap.ArenaAllocator, +location: Location, +location_tok: std.zig.Ast.TokenIndex, +hash_tok: std.zig.Ast.OptionalTokenIndex, +name_tok: std.zig.Ast.TokenIndex, +lazy_status: LazyStatus, +parent_package_root: Cache.Path, +parent_manifest_ast: ?*const std.zig.Ast, +prog_node: std.Progress.Node, +job_queue: *JobQueue, +/// If true, don't add an error for a missing hash. This flag is not passed +/// down to recursive dependencies. It's intended to be used only be the CLI. +omit_missing_hash_error: bool, +/// If true, don't fail when a manifest file is missing the `paths` field, +/// which specifies inclusion rules. This is intended to be true for the first +/// fetch task and false for the recursive dependencies. +allow_missing_paths_field: bool, +allow_missing_fingerprint: bool, +allow_name_string: bool, +/// If true and URL points to a Git repository, will use the latest commit. +use_latest_commit: bool, + +// Above this are fields provided as inputs to `run`. +// Below this are fields populated by `run`. + +/// This will either be relative to `global_cache`, or to the build root of +/// the root package. +package_root: Cache.Path, +error_bundle: ErrorBundle.Wip, +manifest: ?Manifest, +manifest_ast: std.zig.Ast, +computed_hash: ComputedHash, +/// Fetch logic notices whether a package has a build.zig file and sets this flag. +has_build_zig: bool, +/// Indicates whether the task aborted due to an out-of-memory condition. +oom_flag: bool, +/// If `use_latest_commit` was true, this will be set to the commit that was used. +/// If the resource pointed to by the location is not a Git-repository, this +/// will be left unchanged. +latest_commit: ?git.Oid, + +userdata: ?*anyopaque = null, + +pub const LazyStatus = enum { + /// Not lazy. + eager, + /// Lazy, found. + available, + /// Lazy, not found. + unavailable, +}; + +/// Contains shared state among all `Fetch` tasks. +pub const JobQueue = struct { + mutex: std.Thread.Mutex = .{}, + /// It's an array hash map so that it can be sorted before rendering the + /// dependencies.zig source file. + /// Protected by `mutex`. + table: Table = .{}, + /// `table` may be missing some tasks such as ones that failed, so this + /// field contains references to all of them. + /// Protected by `mutex`. + all_fetches: std.ArrayListUnmanaged(*Fetch) = .empty, + + http_client: *std.http.Client, + thread_pool: *ThreadPool, + wait_group: WaitGroup = .{}, + global_cache: Cache.Directory, + /// If true then, no fetching occurs, and: + /// * The `global_cache` directory is assumed to be the direct parent + /// directory of on-disk packages rather than having the "p/" directory + /// prefix inside of it. + /// * An error occurs if any non-lazy packages are not already present in + /// the package cache directory. + /// * Missing hash field causes an error, and no fetching occurs so it does + /// not print the correct hash like usual. + read_only: bool, + recursive: bool, + /// Dumps hash information to stdout which can be used to troubleshoot why + /// two hashes of the same package do not match. + /// If this is true, `recursive` must be false. + debug_hash: bool, + work_around_btrfs_bug: bool, + /// Set of hashes that will be additionally fetched even if they are marked + /// as lazy. + unlazy_set: UnlazySet = .{}, + + pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch); + pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void); + + pub fn deinit(jq: *JobQueue) void { + if (jq.all_fetches.items.len == 0) return; + const gpa = jq.all_fetches.items[0].arena.child_allocator; + jq.table.deinit(gpa); + // These must be deinitialized in reverse order because subsequent + // `Fetch` instances are allocated in prior ones' arenas. + // Sorry, I know it's a bit weird, but it slightly simplifies the + // critical section. + while (jq.all_fetches.pop()) |f| f.deinit(); + jq.all_fetches.deinit(gpa); + jq.* = undefined; + } + + /// Dumps all subsequent error bundles into the first one. + pub fn consolidateErrors(jq: *JobQueue) !void { + const root = &jq.all_fetches.items[0].error_bundle; + const gpa = root.gpa; + for (jq.all_fetches.items[1..]) |fetch| { + if (fetch.error_bundle.root_list.items.len > 0) { + var bundle = try fetch.error_bundle.toOwnedBundle(""); + defer bundle.deinit(gpa); + try root.addBundleAsRoots(bundle); + } + } + } + + /// Creates the dependencies.zig source code for the build runner to obtain + /// via `@import("@dependencies")`. + pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { + const keys = jq.table.keys(); + + assert(keys.len != 0); // caller should have added the first one + if (keys.len == 1) { + // This is the first one. It must have no dependencies. + return createEmptyDependenciesSource(buf); + } + + try buf.appendSlice("pub const packages = struct {\n"); + + // Ensure the generated .zig file is deterministic. + jq.table.sortUnstable(@as(struct { + keys: []const Package.Hash, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); + } + }, .{ .keys = keys })); + + for (keys, jq.table.values()) |*hash, fetch| { + if (fetch == jq.all_fetches.items[0]) { + // The first one is a dummy package for the current project. + continue; + } + + const hash_slice = hash.toSlice(); + + try buf.writer().print( + \\ pub const {} = struct {{ + \\ + , .{std.zig.fmtId(hash_slice)}); + + lazy: { + switch (fetch.lazy_status) { + .eager => break :lazy, + .available => { + try buf.appendSlice( + \\ pub const available = true; + \\ + ); + break :lazy; + }, + .unavailable => { + try buf.appendSlice( + \\ pub const available = false; + \\ }; + \\ + ); + continue; + }, + } + } + + try buf.writer().print( + \\ pub const build_root = "{q}"; + \\ + , .{fetch.package_root}); + + if (fetch.has_build_zig) { + try buf.writer().print( + \\ pub const build_zig = @import("{}"); + \\ + , .{std.zig.fmtEscapes(hash_slice)}); + } + + if (fetch.manifest) |*manifest| { + try buf.appendSlice( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; + try buf.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, + ); + } + + try buf.appendSlice( + \\ }; + \\ }; + \\ + ); + } else { + try buf.appendSlice( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; + \\ }; + \\ + ); + } + } + + try buf.appendSlice( + \\}; + \\ + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + + const root_fetch = jq.all_fetches.items[0]; + const root_manifest = &root_fetch.manifest.?; + + for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { + const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; + try buf.writer().print( + " .{{ \"{}\", \"{}\" }},\n", + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, + ); + } + try buf.appendSlice("};\n"); + } + + pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void { + try buf.appendSlice( + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + } +}; + +pub const Location = union(enum) { + remote: Remote, + /// A directory found inside the parent package. + relative_path: Cache.Path, + /// Recursive Fetch tasks will never use this Location, but it may be + /// passed in by the CLI. Indicates the file contents here should be copied + /// into the global package cache. It may be a file relative to the cwd or + /// absolute, in which case it should be treated exactly like a `file://` + /// URL, or a directory, in which case it should be treated as an + /// already-unpacked directory (but still needs to be copied into the + /// global package cache and have inclusion rules applied). + path_or_url: []const u8, + + pub const Remote = struct { + url: []const u8, + /// If this is null it means the user omitted the hash field from a dependency. + /// It will be an error but the logic should still fetch and print the discovered hash. + hash: ?Package.Hash, + }; +}; + +pub const RunError = error{ + OutOfMemory, + /// This error code is intended to be handled by inspecting the + /// `error_bundle` field. + FetchFailed, +}; + +pub fn run(f: *Fetch) RunError!void { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + + try eb.init(gpa); + + // Check the global zig package cache to see if the hash already exists. If + // so, load, parse, and validate the build.zig.zon file therein, and skip + // ahead to queuing up jobs for dependencies. Likewise if the location is a + // relative path, treat this the same as a cache hit. Otherwise, proceed. + + const remote = switch (f.location) { + .relative_path => |pkg_root| { + if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( + f.location_tok, + try eb.addString("expected path relative to build root; found absolute path"), + ); + if (f.hash_tok.unwrap()) |hash_tok| return f.fail( + hash_tok, + try eb.addString("path-based dependencies are not hashed"), + ); + // Packages fetched by URL may not use relative paths to escape outside the + // fetched package directory from within the package cache. + if (pkg_root.root_dir.eql(cache_root)) { + // `parent_package_root.sub_path` contains a path like this: + // "p/$hash", or + // "p/$hash/foo", with possibly more directories after "foo". + // We want to fail unless the resolved relative path has a + // prefix of "p/$hash/". + const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; + const parent_sub_path = f.parent_package_root.sub_path; + const end = find_end: { + if (parent_sub_path.len > prefix_len) { + // Use `isSep` instead of `indexOfScalarPos` to account for + // Windows accepting both `\` and `/` as path separators. + for (parent_sub_path[prefix_len..], prefix_len..) |c, i| { + if (std.fs.path.isSep(c)) break :find_end i; + } + } + break :find_end parent_sub_path.len; + }; + const expected_prefix = parent_sub_path[0..end]; + if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) { + return f.fail( + f.location_tok, + try eb.printString("dependency path outside project: '{}'", .{pkg_root}), + ); + } + } + f.package_root = pkg_root; + try loadManifest(f, pkg_root); + if (!f.has_build_zig) try checkBuildFileExistence(f); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); + }, + .remote => |remote| remote, + .path_or_url => |path_or_url| { + if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| { + var resource: Resource = .{ .dir = dir }; + return f.runResource(path_or_url, &resource, null); + } else |dir_err| { + const file_err = if (dir_err == error.NotDir) e: { + if (fs.cwd().openFile(path_or_url, .{})) |file| { + var resource: Resource = .{ .file = file }; + return f.runResource(path_or_url, &resource, null); + } else |err| break :e err; + } else dir_err; + + const uri = std.Uri.parse(path_or_url) catch |uri_err| { + return f.fail(0, try eb.printString( + "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", + .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, + )); + }; + var server_header_buffer: [header_buffer_size]u8 = undefined; + var resource = try f.initResource(uri, &server_header_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); + } + }, + }; + + if (remote.hash) |expected_hash| { + var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined; + prefixed_pkg_sub_path_buffer[0] = 'p'; + prefixed_pkg_sub_path_buffer[1] = fs.path.sep; + const hash_slice = expected_hash.toSlice(); + @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); + const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; + const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; + const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; + if (cache_root.handle.access(pkg_sub_path, .{})) |_| { + assert(f.lazy_status != .unavailable); + f.package_root = .{ + .root_dir = cache_root, + .sub_path = try arena.dupe(u8, pkg_sub_path), + }; + try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); + } else |err| switch (err) { + error.FileNotFound => { + switch (f.lazy_status) { + .eager => {}, + .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { + f.lazy_status = .unavailable; + return; + }, + .unavailable => unreachable, + } + if (f.job_queue.read_only) return f.fail( + f.name_tok, + try eb.printString("package not found at '{}{s}'", .{ + cache_root, pkg_sub_path, + }), + ); + }, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ + cache_root, pkg_sub_path, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } + } else if (f.job_queue.read_only) { + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + }); + return error.FetchFailed; + } + + // Fetch and unpack the remote into a temporary directory. + + const uri = std.Uri.parse(remote.url) catch |err| return f.fail( + f.location_tok, + try eb.printString("invalid URI: {s}", .{@errorName(err)}), + ); + var server_header_buffer: [header_buffer_size]u8 = undefined; + var resource = try f.initResource(uri, &server_header_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); +} + +pub fn deinit(f: *Fetch) void { + f.error_bundle.deinit(); + f.arena.deinit(); +} + +/// Consumes `resource`, even if an error is returned. +fn runResource( + f: *Fetch, + uri_path: []const u8, + resource: *Resource, + remote_hash: ?Package.Hash, +) RunError!void { + defer resource.deinit(); + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + const s = fs.path.sep_str; + const cache_root = f.job_queue.global_cache; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); + + const package_sub_path = blk: { + const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); + var tmp_directory: Cache.Directory = .{ + .path = tmp_directory_path, + .handle = handle: { + const dir = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ + .iterate = true, + }) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ + tmp_directory_path, @errorName(err), + }), + }); + return error.FetchFailed; + }; + break :handle dir; + }, + }; + defer tmp_directory.handle.close(); + + // Fetch and unpack a resource into a temporary directory. + var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); + + var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; + + // Apply btrfs workaround if needed. Reopen tmp_directory. + if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { + // https://github.com/ziglang/zig/issues/17095 + pkg_path.root_dir.handle.close(); + pkg_path.root_dir.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ + .iterate = true, + }) catch @panic("btrfs workaround failed"); + } + + // Load, parse, and validate the unpacked build.zig.zon file. It is allowed + // for the file to be missing, in which case this fetched package is + // considered to be a "naked" package. + try loadManifest(f, pkg_path); + + const filter: Filter = .{ + .include_paths = if (f.manifest) |m| m.paths else .{}, + }; + + // Ignore errors that were excluded by manifest, such as failure to + // create symlinks that weren't supposed to be included anyway. + try unpack_result.validate(f, filter); + + // Apply the manifest's inclusion rules to the temporary directory by + // deleting excluded files. + // Empty directories have already been omitted by `unpackResource`. + // Compute the package hash based on the remaining files in the temporary + // directory. + f.computed_hash = try computeHash(f, pkg_path, filter); + + break :blk if (unpack_result.root_dir.len > 0) + try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) + else + tmp_dir_sub_path; + }; + + const computed_package_hash = computedPackageHash(f); + + // Rename the temporary directory into the global zig package cache + // directory. If the hash already exists, delete the temporary directory + // and leave the zig package cache directory untouched as it may be in use + // by the system. This is done even if the hash is invalid, in case the + // package with the different hash is used in the future. + + f.package_root = .{ + .root_dir = cache_root, + .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), + }; + renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { + const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); + const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", + .{ src, dest, @errorName(err) }, + ) }); + return error.FetchFailed; + }; + // Remove temporary directory root if not already renamed to global cache. + if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { + cache_root.handle.deleteDir(tmp_dir_sub_path) catch {}; + } + + // Validate the computed hash against the expected hash. If invalid, this + // job is done. + + if (remote_hash) |declared_hash| { + const hash_tok = f.hash_tok.unwrap().?; + if (declared_hash.isOld()) { + const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest); + if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) { + return f.fail(hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash.toSlice(), actual_hex }, + )); + } + } else { + if (!computed_package_hash.eql(&declared_hash)) { + return f.fail(hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, + )); + } + } + } else if (!f.omit_missing_hash_error) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), + })); + return error.FetchFailed; + } + + // Spawn a new fetch job for each dependency in the manifest file. Use + // a mutex and a hash map so that redundant jobs do not get queued up. + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); +} + +pub fn computedPackageHash(f: *const Fetch) Package.Hash { + const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); + if (f.manifest) |man| { + var version_buffer: [32]u8 = undefined; + const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer; + return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); + } + // In the future build.zig.zon fields will be added to allow overriding these values + // for naked tarballs. + return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); +} + +/// `computeHash` gets a free check for the existence of `build.zig`, but when +/// not computing a hash, we need to do a syscall to check for it. +fn checkBuildFileExistence(f: *Fetch) RunError!void { + const eb = &f.error_bundle; + if (f.package_root.access(Package.build_zig_basename, .{})) |_| { + f.has_build_zig = true; + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to access '{}{s}': {s}", .{ + f.package_root, Package.build_zig_basename, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } +} + +/// This function populates `f.manifest` or leaves it `null`. +fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( + arena, + try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), + Manifest.max_bytes, + null, + 1, + 0, + ) catch |err| switch (err) { + error.FileNotFound => return, + else => |e| { + const file_path = try pkg_root.join(arena, Manifest.basename); + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ + file_path, @errorName(e), + }), + }); + return error.FetchFailed; + }, + }; + + const ast = &f.manifest_ast; + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); + + if (ast.errors.len > 0) { + const file_path = try std.fmt.allocPrint(arena, "{}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root}); + try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); + return error.FetchFailed; + } + + f.manifest = try Manifest.parse(arena, ast.*, .{ + .allow_missing_paths_field = f.allow_missing_paths_field, + .allow_missing_fingerprint = f.allow_missing_fingerprint, + .allow_name_string = f.allow_name_string, + }); + const manifest = &f.manifest.?; + + if (manifest.errors.len > 0) { + const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename }); + try manifest.copyErrorsIntoBundle(ast.*, src_path, eb); + return error.FetchFailed; + } +} + +fn queueJobsForDeps(f: *Fetch) RunError!void { + assert(f.job_queue.recursive); + + // If the package does not have a build.zig.zon file then there are no dependencies. + const manifest = f.manifest orelse return; + + const new_fetches, const prog_names = nf: { + const parent_arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + const dep_names = manifest.dependencies.keys(); + const deps = manifest.dependencies.values(); + // Grab the new tasks into a temporary buffer so we can unlock that mutex + // as fast as possible. + // This overallocates any fetches that get skipped by the `continue` in the + // loop below. + const new_fetches = try parent_arena.alloc(Fetch, deps.len); + const prog_names = try parent_arena.alloc([]const u8, deps.len); + var new_fetch_index: usize = 0; + + f.job_queue.mutex.lock(); + defer f.job_queue.mutex.unlock(); + + try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); + + // There are four cases here: + // * Correct hash is provided by manifest. + // - Hash map already has the entry, no need to add it again. + // * Incorrect hash is provided by manifest. + // - Hash mismatch error emitted; `queueJobsForDeps` is not called. + // * Hash is not provided by manifest. + // - Hash missing error emitted; `queueJobsForDeps` is not called. + // * path-based location is used without a hash. + // - Hash is added to the table based on the path alone before + // calling run(); no need to add it again. + // + // If we add a dep as lazy and then later try to add the same dep as eager, + // eagerness takes precedence and the existing entry is updated. + + for (dep_names, deps) |dep_name, dep| { + const new_fetch = &new_fetches[new_fetch_index]; + const location: Location = switch (dep.location) { + .url => |url| .{ .remote = .{ + .url = url, + .hash = h: { + const h = dep.hash orelse break :h null; + const pkg_hash: Package.Hash = .fromSlice(h); + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); + if (gop.found_existing) { + if (!dep.lazy) { + gop.value_ptr.*.lazy_status = .eager; + } + continue; + } + gop.value_ptr.* = new_fetch; + break :h pkg_hash; + }, + } }, + .path => |rel_path| l: { + // This might produce an invalid path, which is checked for + // at the beginning of run(). + const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); + const pkg_hash = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); + if (gop.found_existing) { + if (!dep.lazy) { + gop.value_ptr.*.lazy_status = .eager; + } + continue; + } + gop.value_ptr.* = new_fetch; + break :l .{ .relative_path = new_root }; + }, + }; + prog_names[new_fetch_index] = dep_name; + new_fetch_index += 1; + f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); + new_fetch.* = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = location, + .location_tok = dep.location_tok, + .hash_tok = dep.hash_tok, + .name_tok = dep.name_tok, + .lazy_status = if (dep.lazy) .available else .eager, + .parent_package_root = f.package_root, + .parent_manifest_ast = &f.manifest_ast, + .prog_node = f.prog_node, + .job_queue = f.job_queue, + .omit_missing_hash_error = false, + .allow_missing_paths_field = true, + .allow_missing_fingerprint = true, + .allow_name_string = true, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + }; + } + + f.prog_node.increaseEstimatedTotalItems(new_fetch_index); + + break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; + }; + + // Now it's time to give tasks to the thread pool. + const thread_pool = f.job_queue.thread_pool; + + for (new_fetches, prog_names) |*new_fetch, prog_name| { + thread_pool.spawnWg(&f.job_queue.wait_group, workerRun, .{ new_fetch, prog_name }); + } +} + +pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { + return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); +} + +pub fn workerRun(f: *Fetch, prog_name: []const u8) void { + const prog_node = f.prog_node.start(prog_name, 0); + defer prog_node.end(); + + run(f) catch |err| switch (err) { + error.OutOfMemory => f.oom_flag = true, + error.FetchFailed => { + // Nothing to do because the errors are already reported in `error_bundle`, + // and a reference is kept to the `Fetch` task inside `all_fetches`. + }, + }; +} + +fn srcLoc( + f: *Fetch, + tok: std.zig.Ast.TokenIndex, +) Allocator.Error!ErrorBundle.SourceLocationIndex { + const ast = f.parent_manifest_ast orelse return .none; + const eb = &f.error_bundle; + const start_loc = ast.tokenLocation(0, tok); + const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); + const msg_off = 0; + return eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = ast.tokenStart(tok), + .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), + .span_main = ast.tokenStart(tok) + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }); +} + +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = msg_str, + .src_loc = try f.srcLoc(msg_tok), + }); + return error.FetchFailed; +} + +const Resource = union(enum) { + file: fs.File, + http_request: std.http.Client.Request, + git: Git, + dir: fs.Dir, + + const Git = struct { + session: git.Session, + fetch_stream: git.Session.FetchStream, + want_oid: git.Oid, + }; + + fn deinit(resource: *Resource) void { + switch (resource.*) { + .file => |*file| file.close(), + .http_request => |*req| req.deinit(), + .git => |*git_resource| { + git_resource.fetch_stream.deinit(); + git_resource.session.deinit(); + }, + .dir => |*dir| dir.close(), + } + resource.* = undefined; + } + + fn reader(resource: *Resource) std.io.AnyReader { + return .{ + .context = resource, + .readFn = read, + }; + } + + fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { + const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); + switch (resource.*) { + .file => |*f| return f.read(buffer), + .http_request => |*r| return r.read(buffer), + .git => |*g| return g.fetch_stream.read(buffer), + .dir => unreachable, + } + } +}; + +const FileType = enum { + tar, + @"tar.gz", + @"tar.xz", + @"tar.zst", + git_pack, + zip, + + fn fromPath(file_path: []const u8) ?FileType { + if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; + if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; + if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; + return null; + } + + /// Parameter is a content-disposition header value. + fn fromContentDisposition(cd_header: []const u8) ?FileType { + const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse + return null; + + var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse + return null; + value_start += "filename".len; + if (cd_header[value_start] == '*') { + value_start += 1; + } + if (cd_header[value_start] != '=') return null; + value_start += 1; + + var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; + if (cd_header[value_end - 1] == '\"') { + value_end -= 1; + } + return fromPath(cd_header[value_start..value_end]); + } + + test fromContentDisposition { + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); + try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); + + try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; size=42") == null); + try std.testing.expect(fromContentDisposition("inline; size=42") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); + } +}; + +const header_buffer_size = 16 * 1024; + +fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource { + const gpa = f.arena.child_allocator; + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + + if (ascii.eqlIgnoreCase(uri.scheme, "file")) { + const path = try uri.path.toRawMaybeAlloc(arena); + return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ + f.parent_package_root, path, @errorName(err), + })); + } }; + } + + const http_client = f.job_queue.http_client; + + if (ascii.eqlIgnoreCase(uri.scheme, "http") or + ascii.eqlIgnoreCase(uri.scheme, "https")) + { + var req = http_client.open(.GET, uri, .{ + .server_header_buffer = server_header_buffer, + }) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to connect to server: {s}", + .{@errorName(err)}, + )); + }; + errdefer req.deinit(); // releases more than memory + + req.send() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "HTTP request failed: {s}", + .{@errorName(err)}, + )); + }; + req.wait() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "invalid HTTP response: {s}", + .{@errorName(err)}, + )); + }; + + if (req.response.status != .ok) { + return f.fail(f.location_tok, try eb.printString( + "bad HTTP response code: '{d} {s}'", + .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, + )); + } + + return .{ .http_request = req }; + } + + if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or + ascii.eqlIgnoreCase(uri.scheme, "git+https")) + { + var transport_uri = uri; + transport_uri.scheme = uri.scheme["git+".len..]; + var session = git.Session.init(gpa, http_client, transport_uri, server_header_buffer) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to discover remote git server capabilities: {s}", + .{@errorName(err)}, + )); + }; + errdefer session.deinit(); + + const want_oid = want_oid: { + const want_ref = + if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; + if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} + + const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); + const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); + + var ref_iterator = session.listRefs(.{ + .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, + .include_peeled = true, + .server_header_buffer = server_header_buffer, + }) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to list refs: {s}", + .{@errorName(err)}, + )); + }; + defer ref_iterator.deinit(); + while (ref_iterator.next() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to iterate refs: {s}", + .{@errorName(err)}, + )); + }) |ref| { + if (std.mem.eql(u8, ref.name, want_ref) or + std.mem.eql(u8, ref.name, want_ref_head) or + std.mem.eql(u8, ref.name, want_ref_tag)) + { + break :want_oid ref.peeled orelse ref.oid; + } + } + return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); + }; + if (f.use_latest_commit) { + f.latest_commit = want_oid; + } else if (uri.fragment == null) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("url field is missing an explicit ref"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("try .url = \"{;+/}#{}\",", .{ uri, want_oid }), + })); + return error.FetchFailed; + } + + var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; + _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{want_oid}) catch unreachable; + var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to create fetch stream: {s}", + .{@errorName(err)}, + )); + }; + errdefer fetch_stream.deinit(); + + return .{ .git = .{ + .session = session, + .fetch_stream = fetch_stream, + .want_oid = want_oid, + } }; + } + + return f.fail(f.location_tok, try eb.printString( + "unsupported URL scheme: {s}", + .{uri.scheme}, + )); +} + +fn unpackResource( + f: *Fetch, + resource: *Resource, + uri_path: []const u8, + tmp_directory: Cache.Directory, +) RunError!UnpackResult { + const eb = &f.error_bundle; + const file_type = switch (resource.*) { + .file => FileType.fromPath(uri_path) orelse + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), + + .http_request => |req| ft: { + // Content-Type takes first precedence. + const content_type = req.response.content_type orelse + return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); + + // Extract the MIME type, ignoring charset and boundary directives + const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; + const mime_type = content_type[0..mime_type_end]; + + if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) + break :ft .tar; + + if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or + ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or + ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or + ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or + ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) + { + break :ft .@"tar.gz"; + } + + if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) + break :ft .@"tar.xz"; + + if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) + break :ft .@"tar.zst"; + + if (ascii.eqlIgnoreCase(mime_type, "application/zip")) + break :ft .zip; + + if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and + !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) + { + return f.fail(f.location_tok, try eb.printString( + "unrecognized 'Content-Type' header: '{s}'", + .{content_type}, + )); + } + + // Next, the filename from 'content-disposition: attachment' takes precedence. + if (req.response.content_disposition) |cd_header| { + break :ft FileType.fromContentDisposition(cd_header) orelse { + return f.fail(f.location_tok, try eb.printString( + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + )); + }; + } + + // Finally, the path from the URI is used. + break :ft FileType.fromPath(uri_path) orelse { + return f.fail(f.location_tok, try eb.printString( + "unknown file type: '{s}'", + .{uri_path}, + )); + }; + }, + + .git => .git_pack, + + .dir => |dir| { + f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to copy directory '{s}': {s}", + .{ uri_path, @errorName(err) }, + )); + }; + return .{}; + }, + }; + + switch (file_type) { + .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()), + .@"tar.gz" => { + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + var dcp = std.compress.gzip.decompressor(br.reader()); + return try unpackTarball(f, tmp_directory.handle, dcp.reader()); + }, + .@"tar.xz" => { + const gpa = f.arena.child_allocator; + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to decompress tarball: {s}", + .{@errorName(err)}, + )); + }; + defer dcp.deinit(); + return try unpackTarball(f, tmp_directory.handle, dcp.reader()); + }, + .@"tar.zst" => { + const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; + const window_buffer = try f.arena.allocator().create([window_size]u8); + const reader = resource.reader(); + var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); + var dcp = std.compress.zstd.decompressor(br.reader(), .{ + .window_buffer = window_buffer, + }); + return try unpackTarball(f, tmp_directory.handle, dcp.reader()); + }, + .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { + error.FetchFailed => return error.FetchFailed, + error.OutOfMemory => return error.OutOfMemory, + else => |e| return f.fail(f.location_tok, try eb.printString( + "unable to unpack git files: {s}", + .{@errorName(e)}, + )), + }, + .zip => return try unzip(f, tmp_directory.handle, resource.reader()), + } +} + +fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + + var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; + + std.tar.pipeToFileSystem(out_dir, reader, .{ + .diagnostics = &diagnostics, + .strip_components = 0, + .mode_mode = .ignore, + .exclude_empty_directories = true, + }) catch |err| return f.fail(f.location_tok, try eb.printString( + "unable to unpack tarball to temporary directory: {s}", + .{@errorName(err)}, + )); + + var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; + if (diagnostics.errors.items.len > 0) { + try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); + for (diagnostics.errors.items) |item| { + switch (item) { + .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), + .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), + .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), + .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 + } + } + } + return res; +} + +fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { + // We write the entire contents to a file first because zip files + // must be processed back to front and they could be too large to + // load into memory. + + const cache_root = f.job_queue.global_cache; + + // TODO: the downside of this solution is if we get a failure/crash/oom/power out + // during this process, we leave behind a zip file that would be + // difficult to know if/when it can be cleaned up. + // Might be worth it to use a mechanism that enables other processes + // to see if the owning process of a file is still alive (on linux this + // can be done with file locks). + // Coupled with this mechansism, we could also use slots (i.e. zig-cache/tmp/0, + // zig-cache/tmp/1, etc) which would mean that subsequent runs would + // automatically clean up old dead files. + // This could all be done with a simple TmpFile abstraction. + const prefix = "tmp/"; + const suffix = ".zip"; + + const random_bytes_count = 20; + const random_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count); + var zip_path: [prefix.len + random_path_len + suffix.len]u8 = undefined; + @memcpy(zip_path[0..prefix.len], prefix); + @memcpy(zip_path[prefix.len + random_path_len ..], suffix); + { + var random_bytes: [random_bytes_count]u8 = undefined; + std.crypto.random.bytes(&random_bytes); + _ = std.fs.base64_encoder.encode( + zip_path[prefix.len..][0..random_path_len], + &random_bytes, + ); + } + + defer cache_root.handle.deleteFile(&zip_path) catch {}; + + const eb = &f.error_bundle; + + { + var zip_file = cache_root.handle.createFile( + &zip_path, + .{}, + ) catch |err| return f.fail(f.location_tok, try eb.printString( + "failed to create tmp zip file: {s}", + .{@errorName(err)}, + )); + defer zip_file.close(); + var buf: [4096]u8 = undefined; + while (true) { + const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( + "read zip stream failed: {s}", + .{@errorName(err)}, + )); + if (len == 0) break; + zip_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString( + "write temporary zip file failed: {s}", + .{@errorName(err)}, + )); + } + } + + var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; + // no need to deinit since we are using an arena allocator + + { + var zip_file = cache_root.handle.openFile( + &zip_path, + .{}, + ) catch |err| return f.fail(f.location_tok, try eb.printString( + "failed to open temporary zip file: {s}", + .{@errorName(err)}, + )); + defer zip_file.close(); + + std.zip.extract(out_dir, zip_file.seekableStream(), .{ + .allow_backslashes = true, + .diagnostics = &diagnostics, + }) catch |err| return f.fail(f.location_tok, try eb.printString( + "zip extract failed: {s}", + .{@errorName(err)}, + )); + } + + cache_root.handle.deleteFile(&zip_path) catch |err| return f.fail(f.location_tok, try eb.printString( + "delete temporary zip failed: {s}", + .{@errorName(err)}, + )); + + const res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; + return res; +} + +fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult { + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const object_format: git.Oid.Format = resource.want_oid; + + var res: UnpackResult = .{}; + // The .git directory is used to store the packfile and associated index, but + // we do not attempt to replicate the exact structure of a real .git + // directory, since that isn't relevant for fetching a package. + { + var pack_dir = try out_dir.makeOpenPath(".git", .{}); + defer pack_dir.close(); + var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); + defer pack_file.close(); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(resource.fetch_stream.reader(), pack_file.writer()); + try pack_file.sync(); + + var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); + defer index_file.close(); + { + const index_prog_node = f.prog_node.start("Index pack", 0); + defer index_prog_node.end(); + var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); + try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer()); + try index_buffered_writer.flush(); + try index_file.sync(); + } + + { + const checkout_prog_node = f.prog_node.start("Checkout", 0); + defer checkout_prog_node.end(); + var repository = try git.Repository.init(gpa, object_format, pack_file, index_file); + defer repository.deinit(); + var diagnostics: git.Diagnostics = .{ .allocator = arena }; + try repository.checkout(out_dir, resource.want_oid, &diagnostics); + + if (diagnostics.errors.items.len > 0) { + try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); + for (diagnostics.errors.items) |item| { + switch (item) { + .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), + .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), + } + } + } + } + } + + try out_dir.deleteTree(".git"); + return res; +} + +fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { + const gpa = f.arena.child_allocator; + // Recursive directory copy. + var it = try dir.walk(gpa); + defer it.deinit(); + while (try it.next()) |entry| { + switch (entry.kind) { + .directory => {}, // omit empty directories + .file => { + dir.copyFile( + entry.path, + tmp_dir, + entry.path, + .{}, + ) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); + try dir.copyFile(entry.path, tmp_dir, entry.path, .{}); + }, + else => |e| return e, + }; + }, + .sym_link => { + var buf: [fs.max_path_bytes]u8 = undefined; + const link_name = try dir.readLink(entry.path, &buf); + // TODO: if this would create a symlink to outside + // the destination directory, fail with an error instead. + tmp_dir.symLink(link_name, entry.path, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); + try tmp_dir.symLink(link_name, entry.path, .{}); + }, + else => |e| return e, + }; + }, + else => return error.IllegalFileTypeInPackage, + } + } +} + +pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { + assert(dest_dir_sub_path[1] == fs.path.sep); + var handled_missing_dir = false; + while (true) { + cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { + error.FileNotFound => { + if (handled_missing_dir) return err; + cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { + error.PathAlreadyExists => handled_missing_dir = true, + else => |e| return e, + }; + continue; + }, + error.PathAlreadyExists, error.AccessDenied => { + // Package has been already downloaded and may already be in use on the system. + cache_dir.deleteTree(tmp_dir_sub_path) catch { + // Garbage files leftover in zig-cache/tmp/ is, as they say + // on Star Trek, "operating within normal parameters". + }; + }, + else => |e| return e, + }; + break; + } +} + +const ComputedHash = struct { + digest: Package.Hash.Digest, + total_size: u64, +}; + +/// Assumes that files not included in the package have already been filtered +/// prior to calling this function. This ensures that files not protected by +/// the hash are not present on the file system. Empty directories are *not +/// hashed* and must not be present on the file system when calling this +/// function. +fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { + // All the path name strings need to be in memory for sorting. + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const thread_pool = f.job_queue.thread_pool; + const root_dir = pkg_path.root_dir.handle; + + // Collect all files, recursively, then sort. + var all_files = std.ArrayList(*HashedFile).init(gpa); + defer all_files.deinit(); + + var deleted_files = std.ArrayList(*DeletedFile).init(gpa); + defer deleted_files.deinit(); + + // Track directories which had any files deleted from them so that empty directories + // can be deleted. + var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; + defer sus_dirs.deinit(gpa); + + var walker = try root_dir.walk(gpa); + defer walker.deinit(); + + // Total number of bytes of file contents included in the package. + var total_size: u64 = 0; + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var wait_group: WaitGroup = .{}; + // `computeHash` is called from a worker thread so there must not be + // any waiting without working or a deadlock could occur. + defer thread_pool.waitAndWork(&wait_group); + + while (walker.next() catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to walk temporary directory '{}': {s}", + .{ pkg_path, @errorName(err) }, + ) }); + return error.FetchFailed; + }) |entry| { + if (entry.kind == .directory) continue; + + const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); + if (!filter.includePath(entry_pkg_path)) { + // Delete instead of including in hash calculation. + const fs_path = try arena.dupe(u8, entry.path); + + // Also track the parent directory in case it becomes empty. + if (fs.path.dirname(fs_path)) |parent| + try sus_dirs.put(gpa, parent, {}); + + const deleted_file = try arena.create(DeletedFile); + deleted_file.* = .{ + .fs_path = fs_path, + .failure = undefined, // to be populated by the worker + }; + thread_pool.spawnWg(&wait_group, workerDeleteFile, .{ root_dir, deleted_file }); + try deleted_files.append(deleted_file); + continue; + } + + const kind: HashedFile.Kind = switch (entry.kind) { + .directory => unreachable, + .file => .file, + .sym_link => .link, + else => return f.fail(f.location_tok, try eb.printString( + "package contains '{s}' which has illegal file type '{s}'", + .{ entry.path, @tagName(entry.kind) }, + )), + }; + + if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) + f.has_build_zig = true; + + const fs_path = try arena.dupe(u8, entry.path); + const hashed_file = try arena.create(HashedFile); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), + .kind = kind, + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + .size = undefined, // to be populated by the worker + }; + thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file }); + try all_files.append(hashed_file); + } + } + + { + // Sort by length, descending, so that child directories get removed first. + sus_dirs.sortUnstable(@as(struct { + keys: []const []const u8, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.keys[b_index].len < ctx.keys[a_index].len; + } + }, .{ .keys = sus_dirs.keys() })); + + // During this loop, more entries will be added, so we must loop by index. + var i: usize = 0; + while (i < sus_dirs.count()) : (i += 1) { + const sus_dir = sus_dirs.keys()[i]; + root_dir.deleteDir(sus_dir) catch |err| switch (err) { + error.DirNotEmpty => continue, + error.FileNotFound => continue, + else => |e| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to delete empty directory '{s}': {s}", + .{ sus_dir, @errorName(e) }, + ) }); + return error.FetchFailed; + }, + }; + if (fs.path.dirname(sus_dir)) |parent| { + try sus_dirs.put(gpa, parent, {}); + } + } + } + + std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Package.Hash.Algo.init(.{}); + var any_failures = false; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to hash '{s}': {s}", .{ + hashed_file.fs_path, @errorName(err), + }), + }); + }; + hasher.update(&hashed_file.hash); + total_size += hashed_file.size; + } + for (deleted_files.items) |deleted_file| { + deleted_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ + deleted_file.fs_path, @errorName(err), + }), + }); + }; + } + + if (any_failures) return error.FetchFailed; + + if (f.job_queue.debug_hash) { + assert(!f.job_queue.recursive); + // Print something to stdout that can be text diffed to figure out why + // the package hash is different. + dumpHashInfo(all_files.items) catch |err| { + std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)}); + std.process.exit(1); + }; + } + + return .{ + .digest = hasher.finalResult(), + .total_size = total_size, + }; +} + +fn dumpHashInfo(all_files: []const *const HashedFile) !void { + const stdout = std.io.getStdOut(); + var bw = std.io.bufferedWriter(stdout.writer()); + const w = bw.writer(); + + for (all_files) |hashed_file| { + try w.print("{s}: {s}: {s}\n", .{ + @tagName(hashed_file.kind), + std.fmt.fmtSliceHexLower(&hashed_file.hash), + hashed_file.normalized_path, + }); + } + + try bw.flush(); +} + +fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile) void { + hashed_file.failure = hashFileFallible(dir, hashed_file); +} + +fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void { + deleted_file.failure = deleteFileFallible(dir, deleted_file); +} + +fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { + var buf: [8000]u8 = undefined; + var hasher = Package.Hash.Algo.init(.{}); + hasher.update(hashed_file.normalized_path); + var file_size: u64 = 0; + + switch (hashed_file.kind) { + .file => { + var file = try dir.openFile(hashed_file.fs_path, .{}); + defer file.close(); + // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 + hasher.update(&.{ 0, 0 }); + var file_header: FileHeader = .{}; + while (true) { + const bytes_read = try file.read(&buf); + if (bytes_read == 0) break; + file_size += bytes_read; + hasher.update(buf[0..bytes_read]); + file_header.update(buf[0..bytes_read]); + } + if (file_header.isExecutable()) { + try setExecutable(file); + } + }, + .link => { + const link_name = try dir.readLink(hashed_file.fs_path, &buf); + if (fs.path.sep != canonical_sep) { + // Package hashes are intended to be consistent across + // platforms which means we must normalize path separators + // inside symlinks. + normalizePath(link_name); + } + hasher.update(link_name); + }, + } + hasher.final(&hashed_file.hash); + hashed_file.size = file_size; +} + +fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { + try dir.deleteFile(deleted_file.fs_path); +} + +fn setExecutable(file: fs.File) !void { + if (!std.fs.has_executable_bit) return; + + const S = std.posix.S; + const mode = fs.File.default_mode | S.IXUSR | S.IXGRP | S.IXOTH; + try file.chmod(mode); +} + +const DeletedFile = struct { + fs_path: []const u8, + failure: Error!void, + + const Error = + fs.Dir.DeleteFileError || + fs.Dir.DeleteDirError; +}; + +const HashedFile = struct { + fs_path: []const u8, + normalized_path: []const u8, + hash: Package.Hash.Digest, + failure: Error!void, + kind: Kind, + size: u64, + + const Error = + fs.File.OpenError || + fs.File.ReadError || + fs.File.StatError || + fs.File.ChmodError || + fs.Dir.ReadLinkError; + + const Kind = enum { file, link }; + + fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { + _ = context; + return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); + } +}; + +/// Strips root directory name from file system path. +fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { + if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; + + if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { + return fs_path[root_dir.len + 1 ..]; + } + + return fs_path; +} + +/// Make a file system path identical independently of operating system path inconsistencies. +/// This converts backslashes into forward slashes. +fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { + const normalized = try arena.dupe(u8, pkg_path); + if (fs.path.sep == canonical_sep) return normalized; + normalizePath(normalized); + return normalized; +} + +const canonical_sep = fs.path.sep_posix; + +fn normalizePath(bytes: []u8) void { + assert(fs.path.sep != canonical_sep); + std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); +} + +const Filter = struct { + include_paths: std.StringArrayHashMapUnmanaged(void) = .empty, + + /// sub_path is relative to the package root. + pub fn includePath(self: Filter, sub_path: []const u8) bool { + if (self.include_paths.count() == 0) return true; + if (self.include_paths.contains("")) return true; + if (self.include_paths.contains(".")) return true; + if (self.include_paths.contains(sub_path)) return true; + + // Check if any included paths are parent directories of sub_path. + var dirname = sub_path; + while (std.fs.path.dirname(dirname)) |next_dirname| { + if (self.include_paths.contains(next_dirname)) return true; + dirname = next_dirname; + } + + return false; + } + + test includePath { + const gpa = std.testing.allocator; + var filter: Filter = .{}; + defer filter.include_paths.deinit(gpa); + + try filter.include_paths.put(gpa, "src", {}); + try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); + try std.testing.expect(!filter.includePath(".gitignore")); + } +}; + +pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { + if (dep.hash) |h| return .fromSlice(h); + + switch (dep.location) { + .url => return null, + .path => |rel_path| { + var buf: [fs.max_path_bytes]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch + return null; + return relativePathDigest(new_root, cache_root); + }, + } +} + +const builtin = @import("builtin"); +const std = @import("std"); +const fs = std.fs; +const assert = std.debug.assert; +const ascii = std.ascii; +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const ThreadPool = std.Thread.Pool; +const WaitGroup = std.Thread.WaitGroup; +const Fetch = @This(); +const git = @import("Fetch/git.zig"); +const Package = @import("../Package.zig"); +const Manifest = Package.Manifest; +const ErrorBundle = std.zig.ErrorBundle; +const native_os = builtin.os.tag; + +test { + _ = Filter; + _ = FileType; + _ = UnpackResult; +} + +// Detects executable header: ELF or Macho-O magic header or shebang line. +const FileHeader = struct { + header: [4]u8 = undefined, + bytes_read: usize = 0, + + pub fn update(self: *FileHeader, buf: []const u8) void { + if (self.bytes_read >= self.header.len) return; + const n = @min(self.header.len - self.bytes_read, buf.len); + @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); + self.bytes_read += n; + } + + fn isScript(self: *FileHeader) bool { + const shebang = "#!"; + return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); + } + + fn isElf(self: *FileHeader) bool { + const elf_magic = std.elf.MAGIC; + return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); + } + + fn isMachO(self: *FileHeader) bool { + if (self.bytes_read < 4) return false; + const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); + return magic_number == std.macho.MH_MAGIC or + magic_number == std.macho.MH_MAGIC_64 or + magic_number == std.macho.FAT_MAGIC or + magic_number == std.macho.FAT_MAGIC_64 or + magic_number == std.macho.MH_CIGAM or + magic_number == std.macho.MH_CIGAM_64 or + magic_number == std.macho.FAT_CIGAM or + magic_number == std.macho.FAT_CIGAM_64; + } + + pub fn isExecutable(self: *FileHeader) bool { + return self.isScript() or self.isElf() or self.isMachO(); + } +}; + +test FileHeader { + var h: FileHeader = .{}; + try std.testing.expect(!h.isExecutable()); + + const elf_magic = std.elf.MAGIC; + h.update(elf_magic[0..2]); + try std.testing.expect(!h.isExecutable()); + h.update(elf_magic[2..4]); + try std.testing.expect(h.isExecutable()); + + h.update(elf_magic[2..4]); + try std.testing.expect(h.isExecutable()); + + const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; + h.bytes_read = 0; + h.update(&macho64_magic_bytes); + try std.testing.expect(h.isExecutable()); + + const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; + h.bytes_read = 0; + h.update(&macho64_cigam_bytes); + try std.testing.expect(h.isExecutable()); +} + +// Result of the `unpackResource` operation. Enables collecting errors from +// tar/git diagnostic, filtering that errors by manifest inclusion rules and +// emitting remaining errors to an `ErrorBundle`. +const UnpackResult = struct { + errors: []Error = undefined, + errors_count: usize = 0, + root_error_message: []const u8 = "", + + // A non empty value means that the package contents are inside a + // sub-directory indicated by the named path. + root_dir: []const u8 = "", + + const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, + unsupported_file_type: struct { + file_name: []const u8, + file_type: u8, + }, + + fn excluded(self: Error, filter: Filter) bool { + const file_name = switch (self) { + .unable_to_create_file => |info| info.file_name, + .unable_to_create_sym_link => |info| info.file_name, + .unsupported_file_type => |info| info.file_name, + }; + return !filter.includePath(file_name); + } + }; + + fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { + self.root_error_message = try arena.dupe(u8, root_error_message); + self.errors = try arena.alloc(UnpackResult.Error, n); + } + + fn hasErrors(self: *UnpackResult) bool { + return self.errors_count > 0; + } + + fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { + self.errors[self.errors_count] = .{ .unable_to_create_file = .{ + .code = err, + .file_name = file_name, + } }; + self.errors_count += 1; + } + + fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { + self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ + .code = err, + .file_name = file_name, + .link_name = link_name, + } }; + self.errors_count += 1; + } + + fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { + self.errors[self.errors_count] = .{ .unsupported_file_type = .{ + .file_name = file_name, + .file_type = file_type, + } }; + self.errors_count += 1; + } + + fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { + if (self.errors_count == 0) return; + + var unfiltered_errors: u32 = 0; + for (self.errors) |item| { + if (item.excluded(filter)) continue; + unfiltered_errors += 1; + } + if (unfiltered_errors == 0) return; + + // Emmit errors to an `ErrorBundle`. + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(self.root_error_message), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = unfiltered_errors, + }); + var note_i: u32 = try eb.reserveNotes(unfiltered_errors); + for (self.errors) |item| { + if (item.excluded(filter)) continue; + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + .unable_to_create_file => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create file '{s}': {s}", .{ + info.file_name, @errorName(info.code), + }), + })); + }, + .unsupported_file_type => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ + info.file_name, info.file_type, + }), + })); + }, + } + note_i += 1; + } + + return error.FetchFailed; + } + + test validate { + const gpa = std.testing.allocator; + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + // fill UnpackResult with errors + var res: UnpackResult = .{}; + try res.allocErrors(arena, 4, "unable to unpack"); + try std.testing.expectEqual(0, res.errors_count); + res.unableToCreateFile("dir1/file1", error.File1); + res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); + res.unableToCreateFile("dir1/file3", error.File3); + res.unsupportedFileType("dir2/file4", 'x'); + try std.testing.expectEqual(4, res.errors_count); + + // create filter, includes dir2, excludes dir1 + var filter: Filter = .{}; + try filter.include_paths.put(arena, "dir2", {}); + + // init Fetch + var fetch: Fetch = undefined; + fetch.parent_manifest_ast = null; + fetch.location_tok = 0; + try fetch.error_bundle.init(gpa); + defer fetch.error_bundle.deinit(); + + // validate errors with filter + try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); + + // output errors to string + var errors = try fetch.error_bundle.toOwnedBundle(""); + defer errors.deinit(gpa); + var out = std.ArrayList(u8).init(gpa); + defer out.deinit(); + try errors.renderToWriter(.{ .ttyconf = .no_color }, out.writer()); + try std.testing.expectEqualStrings( + \\error: unable to unpack + \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError + \\ note: file 'dir2/file4' has unsupported type 'x' + \\ + , out.items); + } +}; + +test "zip" { + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const test_files = [_]std.zip.testutil.File{ + .{ .name = "foo", .content = "this is just foo\n", .compression = .store }, + .{ .name = "bar", .content = "another file\n", .compression = .deflate }, + }; + { + var zip_file = try tmp.dir.createFile("test.zip", .{}); + defer zip_file.close(); + var bw = std.io.bufferedWriter(zip_file.writer()); + var store: [test_files.len]std.zip.testutil.FileStore = undefined; + try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); + try bw.flush(); + } + + const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); + defer gpa.free(zip_path); + + var fb: TestFetchBuilder = undefined; + var fetch = try fb.build(gpa, tmp.dir, zip_path); + defer fb.deinit(); + + try fetch.run(); + + var out = try fb.packageDir(); + defer out.close(); + + try std.zip.testutil.expectFiles(&test_files, out, .{}); +} + +test "zip with one root folder" { + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const test_files = [_]std.zip.testutil.File{ + .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store }, + .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store }, + }; + { + var zip_file = try tmp.dir.createFile("test.zip", .{}); + defer zip_file.close(); + var bw = std.io.bufferedWriter(zip_file.writer()); + var store: [test_files.len]std.zip.testutil.FileStore = undefined; + try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); + try bw.flush(); + } + + const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); + defer gpa.free(zip_path); + + var fb: TestFetchBuilder = undefined; + var fetch = try fb.build(gpa, tmp.dir, zip_path); + defer fb.deinit(); + + try fetch.run(); + + var out = try fb.packageDir(); + defer out.close(); + + try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" }); +} + +test "tarball with duplicate paths" { + // This tarball has duplicate path 'dir1/file1' to simulate case sensitve + // file system on any file sytstem. + // + // duplicate_paths/ + // duplicate_paths/dir1/ + // duplicate_paths/dir1/file1 + // duplicate_paths/dir1/file1 + // duplicate_paths/build.zig.zon + // duplicate_paths/src/ + // duplicate_paths/src/main.zig + // duplicate_paths/src/root.zig + // duplicate_paths/build.zig + // + + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const tarball_name = "duplicate_paths.tar.gz"; + try saveEmbedFile(tarball_name, tmp.dir); + const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); + defer gpa.free(tarball_path); + + // Run tarball fetch, expect to fail + var fb: TestFetchBuilder = undefined; + var fetch = try fb.build(gpa, tmp.dir, tarball_path); + defer fb.deinit(); + try std.testing.expectError(error.FetchFailed, fetch.run()); + + try fb.expectFetchErrors(1, + \\error: unable to unpack tarball + \\ note: unable to create file 'dir1/file1': PathAlreadyExists + \\ + ); +} + +test "tarball with excluded duplicate paths" { + // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. + // + // .paths = .{ + // "build.zig", + // "build.zig.zon", + // "src", + // } + // + + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const tarball_name = "duplicate_paths_excluded.tar.gz"; + try saveEmbedFile(tarball_name, tmp.dir); + const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); + defer gpa.free(tarball_path); + + // Run tarball fetch, should succeed + var fb: TestFetchBuilder = undefined; + var fetch = try fb.build(gpa, tmp.dir, tarball_path); + defer fb.deinit(); + try fetch.run(); + + const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); + try std.testing.expectEqualStrings( + "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", + &hex_digest, + ); + + const expected_files: []const []const u8 = &.{ + "build.zig", + "build.zig.zon", + "src/main.zig", + "src/root.zig", + }; + try fb.expectPackageFiles(expected_files); +} + +test "tarball without root folder" { + // Tarball with root folder. Manifest excludes dir1 and dir2. + // + // build.zig + // build.zig.zon + // dir1/ + // dir1/file2 + // dir1/file1 + // dir2/ + // dir2/file2 + // src/ + // src/main.zig + // + + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const tarball_name = "no_root.tar.gz"; + try saveEmbedFile(tarball_name, tmp.dir); + const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); + defer gpa.free(tarball_path); + + // Run tarball fetch, should succeed + var fb: TestFetchBuilder = undefined; + var fetch = try fb.build(gpa, tmp.dir, tarball_path); + defer fb.deinit(); + try fetch.run(); + + const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); + try std.testing.expectEqualStrings( + "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", + &hex_digest, + ); + + const expected_files: []const []const u8 = &.{ + "build.zig", + "build.zig.zon", + "src/main.zig", + }; + try fb.expectPackageFiles(expected_files); +} + +test "set executable bit based on file content" { + if (!std.fs.has_executable_bit) return error.SkipZigTest; + const gpa = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const tarball_name = "executables.tar.gz"; + try saveEmbedFile(tarball_name, tmp.dir); + const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); + defer gpa.free(tarball_path); + + // $ tar -tvf executables.tar.gz + // drwxrwxr-x 0 executables/ + // -rwxrwxr-x 170 executables/hello + // lrwxrwxrwx 0 executables/hello_ln -> hello + // -rw-rw-r-- 0 executables/file1 + // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit + // -rwxrwxr-x 7 executables/script_without_shebang + // -rwxrwxr-x 17 executables/script + + var fb: TestFetchBuilder = undefined; + var fetch = try fb.build(gpa, tmp.dir, tarball_path); + defer fb.deinit(); + + try fetch.run(); + try std.testing.expectEqualStrings( + "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", + &Package.multiHashHexDigest(fetch.computed_hash.digest), + ); + + var out = try fb.packageDir(); + defer out.close(); + const S = std.posix.S; + // expect executable bit not set + try std.testing.expect((try out.statFile("file1")).mode & S.IXUSR == 0); + try std.testing.expect((try out.statFile("script_without_shebang")).mode & S.IXUSR == 0); + // expect executable bit set + try std.testing.expect((try out.statFile("hello")).mode & S.IXUSR != 0); + try std.testing.expect((try out.statFile("script")).mode & S.IXUSR != 0); + try std.testing.expect((try out.statFile("script_with_shebang_without_exec_bit")).mode & S.IXUSR != 0); + try std.testing.expect((try out.statFile("hello_ln")).mode & S.IXUSR != 0); + + // + // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3 + // -rw-rw-r-- 1 0 Apr file1 + // -rwxrwxr-x 1 170 Apr hello + // lrwxrwxrwx 1 5 Apr hello_ln -> hello + // -rwxrwxr-x 1 17 Apr script + // -rw-rw-r-- 1 7 Apr script_without_shebang + // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit +} + +fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void { + //const tarball_name = "duplicate_paths_excluded.tar.gz"; + const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name); + var tmp_file = try dir.createFile(tarball_name, .{}); + defer tmp_file.close(); + try tmp_file.writeAll(tarball_content); +} + +// Builds Fetch with required dependencies, clears dependencies on deinit(). +const TestFetchBuilder = struct { + thread_pool: ThreadPool, + http_client: std.http.Client, + global_cache_directory: Cache.Directory, + job_queue: Fetch.JobQueue, + fetch: Fetch, + + fn build( + self: *TestFetchBuilder, + allocator: std.mem.Allocator, + cache_parent_dir: std.fs.Dir, + path_or_url: []const u8, + ) !*Fetch { + const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{}); + + try self.thread_pool.init(.{ .allocator = allocator }); + self.http_client = .{ .allocator = allocator }; + self.global_cache_directory = .{ .handle = cache_dir, .path = null }; + + self.job_queue = .{ + .http_client = &self.http_client, + .thread_pool = &self.thread_pool, + .global_cache = self.global_cache_directory, + .recursive = false, + .read_only = false, + .debug_hash = false, + .work_around_btrfs_bug = false, + }; + + self.fetch = .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, + .parent_manifest_ast = null, + .prog_node = std.Progress.Node.none, + .job_queue = &self.job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz + .allow_name_string = true, // so we can keep using the old testdata .tar.gz + .use_latest_commit = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + }; + return &self.fetch; + } + + fn deinit(self: *TestFetchBuilder) void { + self.fetch.deinit(); + self.job_queue.deinit(); + self.fetch.prog_node.end(); + self.global_cache_directory.handle.close(); + self.http_client.deinit(); + self.thread_pool.deinit(); + } + + fn packageDir(self: *TestFetchBuilder) !fs.Dir { + const root = self.fetch.package_root; + return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); + } + + // Test helper, asserts thet package dir constains expected_files. + // expected_files must be sorted. + fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { + var package_dir = try self.packageDir(); + defer package_dir.close(); + + var actual_files: std.ArrayListUnmanaged([]u8) = .empty; + defer actual_files.deinit(std.testing.allocator); + defer for (actual_files.items) |file| std.testing.allocator.free(file); + var walker = try package_dir.walk(std.testing.allocator); + defer walker.deinit(); + while (try walker.next()) |entry| { + if (entry.kind != .file) continue; + const path = try std.testing.allocator.dupe(u8, entry.path); + errdefer std.testing.allocator.free(path); + std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); + try actual_files.append(std.testing.allocator, path); + } + std.mem.sortUnstable([]u8, actual_files.items, {}, struct { + fn lessThan(_: void, a: []u8, b: []u8) bool { + return std.mem.lessThan(u8, a, b); + } + }.lessThan); + + try std.testing.expectEqual(expected_files.len, actual_files.items.len); + for (expected_files, 0..) |file_name, i| { + try std.testing.expectEqualStrings(file_name, actual_files.items[i]); + } + try std.testing.expectEqualDeep(expected_files, actual_files.items); + } + + // Test helper, asserts that fetch has failed with `msg` error message. + fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { + var errors = try self.fetch.error_bundle.toOwnedBundle(""); + defer errors.deinit(std.testing.allocator); + + const em = errors.getErrorMessage(errors.getMessages()[0]); + try std.testing.expectEqual(1, em.count); + if (notes_len > 0) { + try std.testing.expectEqual(notes_len, em.notes_len); + } + var al = std.ArrayList(u8).init(std.testing.allocator); + defer al.deinit(); + try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); + try std.testing.expectEqualStrings(msg, al.items); + } +}; diff --git a/lib/std/zig/Package/Fetch/git.zig b/lib/std/zig/Package/Fetch/git.zig new file mode 100644 index 0000000000000000000000000000000000000000..f6e3dc16152a0024cc6d546f655c51db282a4e6d --- /dev/null +++ b/lib/std/zig/Package/Fetch/git.zig @@ -0,0 +1,1689 @@ +//! Git support for package fetching. +//! +//! This is not intended to support all features of Git: it is limited to the +//! basic functionality needed to clone a repository for the purpose of fetching +//! a package. + +const std = @import("std"); +const mem = std.mem; +const testing = std.testing; +const Allocator = mem.Allocator; +const Sha1 = std.crypto.hash.Sha1; +const Sha256 = std.crypto.hash.sha2.Sha256; +const assert = std.debug.assert; + +/// The ID of a Git object. +pub const Oid = union(Format) { + sha1: [Sha1.digest_length]u8, + sha256: [Sha256.digest_length]u8, + + pub const max_formatted_length = len: { + var max: usize = 0; + for (std.enums.values(Format)) |f| { + max = @max(max, f.formattedLength()); + } + break :len max; + }; + + pub const Format = enum { + sha1, + sha256, + + pub fn byteLength(f: Format) usize { + return switch (f) { + .sha1 => Sha1.digest_length, + .sha256 => Sha256.digest_length, + }; + } + + pub fn formattedLength(f: Format) usize { + return 2 * f.byteLength(); + } + }; + + const Hasher = union(Format) { + sha1: Sha1, + sha256: Sha256, + + fn init(oid_format: Format) Hasher { + return switch (oid_format) { + .sha1 => .{ .sha1 = Sha1.init(.{}) }, + .sha256 => .{ .sha256 = Sha256.init(.{}) }, + }; + } + + // Must be public for use from HashedReader and HashedWriter. + pub fn update(hasher: *Hasher, b: []const u8) void { + switch (hasher.*) { + inline else => |*inner| inner.update(b), + } + } + + fn finalResult(hasher: *Hasher) Oid { + return switch (hasher.*) { + inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), + }; + } + }; + + pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { + assert(bytes.len == oid_format.byteLength()); + return switch (oid_format) { + inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), + }; + } + + pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { + return switch (oid_format) { + inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), + }; + } + + pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { + switch (oid_format) { + inline else => |tag| { + if (s.len != tag.formattedLength()) return error.InvalidOid; + var bytes: [tag.byteLength()]u8 = undefined; + for (&bytes, 0..) |*b, i| { + b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; + } + return @unionInit(Oid, @tagName(tag), bytes); + }, + } + } + + test parse { + try testing.expectEqualSlices( + u8, + &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, + &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, + ); + try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); + try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); + try testing.expectEqualSlices( + u8, + &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, + &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, + ); + try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); + try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); + try testing.expectError(error.InvalidOid, parse(.sha1, "master")); + try testing.expectError(error.InvalidOid, parse(.sha256, "master")); + try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); + try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); + } + + pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { + return for (std.enums.values(Format)) |f| { + if (s.len == f.formattedLength()) break parse(f, s); + } else error.InvalidOid; + } + + pub fn format( + oid: Oid, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + _ = fmt; + _ = options; + try writer.print("{}", .{std.fmt.fmtSliceHexLower(oid.slice())}); + } + + pub fn slice(oid: *const Oid) []const u8 { + return switch (oid.*) { + inline else => |*bytes| bytes, + }; + } +}; + +pub const Diagnostics = struct { + allocator: Allocator, + errors: std.ArrayListUnmanaged(Error) = .empty, + + pub const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, + }; + + pub fn deinit(d: *Diagnostics) void { + for (d.errors.items) |item| { + switch (item) { + .unable_to_create_sym_link => |info| { + d.allocator.free(info.file_name); + d.allocator.free(info.link_name); + }, + .unable_to_create_file => |info| { + d.allocator.free(info.file_name); + }, + } + } + d.errors.deinit(d.allocator); + d.* = undefined; + } +}; + +pub const Repository = struct { + odb: Odb, + + pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { + return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; + } + + pub fn deinit(repository: *Repository) void { + repository.odb.deinit(); + repository.* = undefined; + } + + /// Checks out the repository at `commit_oid` to `worktree`. + pub fn checkout( + repository: *Repository, + worktree: std.fs.Dir, + commit_oid: Oid, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(commit_oid); + const tree_oid = tree_oid: { + const commit_object = try repository.odb.readObject(); + if (commit_object.type != .commit) return error.NotACommit; + break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); + }; + try repository.checkoutTree(worktree, tree_oid, "", diagnostics); + } + + /// Checks out the tree at `tree_oid` to `worktree`. + fn checkoutTree( + repository: *Repository, + dir: std.fs.Dir, + tree_oid: Oid, + current_path: []const u8, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(tree_oid); + const tree_object = try repository.odb.readObject(); + if (tree_object.type != .tree) return error.NotATree; + // The tree object may be evicted from the object cache while we're + // iterating over it, so we can make a defensive copy here to make sure + // it remains valid until we're done with it + const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); + defer repository.odb.allocator.free(tree_data); + + var tree_iter: TreeIterator = .{ + .format = repository.odb.format, + .data = tree_data, + .pos = 0, + }; + while (try tree_iter.next()) |entry| { + switch (entry.type) { + .directory => { + try dir.makeDir(entry.name); + var subdir = try dir.openDir(entry.name, .{}); + defer subdir.close(); + const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); + defer repository.odb.allocator.free(sub_path); + try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics); + }, + .file => { + try repository.odb.seekOid(entry.oid); + const file_object = try repository.odb.readObject(); + if (file_object.type != .blob) return error.InvalidFile; + var file = dir.createFile(entry.name, .{ .exclusive = true }) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ + .code = e, + .file_name = file_name, + } }); + continue; + }; + defer file.close(); + try file.writeAll(file_object.data); + try file.sync(); + }, + .symlink => { + try repository.odb.seekOid(entry.oid); + const symlink_object = try repository.odb.readObject(); + if (symlink_object.type != .blob) return error.InvalidFile; + const link_name = symlink_object.data; + dir.symLink(link_name, entry.name, .{}) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); + errdefer diagnostics.allocator.free(link_name_dup); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ + .code = e, + .file_name = file_name, + .link_name = link_name_dup, + } }); + }; + }, + .gitlink => { + // Consistent with git archive behavior, create the directory but + // do nothing else + try dir.makeDir(entry.name); + }, + } + } + } + + /// Returns the ID of the tree associated with the given commit (provided as + /// raw object data). + fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { + if (!mem.startsWith(u8, commit_data, "tree ") or + commit_data.len < "tree ".len + format.formattedLength() + "\n".len or + commit_data["tree ".len + format.formattedLength()] != '\n') + { + return error.InvalidCommit; + } + return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); + } + + const TreeIterator = struct { + format: Oid.Format, + data: []const u8, + pos: usize, + + const Entry = struct { + type: Type, + executable: bool, + name: [:0]const u8, + oid: Oid, + + const Type = enum(u4) { + directory = 0o4, + file = 0o10, + symlink = 0o12, + gitlink = 0o16, + }; + }; + + fn next(iterator: *TreeIterator) !?Entry { + if (iterator.pos == iterator.data.len) return null; + + const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; + const mode: packed struct { + permission: u9, + unused: u3, + type: u4, + } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); + const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; + const executable = switch (mode.permission) { + 0 => if (@"type" == .file) return error.InvalidTree else false, + 0o644 => if (@"type" != .file) return error.InvalidTree else false, + 0o755 => if (@"type" != .file) return error.InvalidTree else true, + else => return error.InvalidTree, + }; + iterator.pos = mode_end + 1; + + const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; + const name = iterator.data[iterator.pos..name_end :0]; + iterator.pos = name_end + 1; + + const oid_length = iterator.format.byteLength(); + if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; + const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); + iterator.pos += oid_length; + + return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; + } + }; +}; + +/// A Git object database backed by a packfile. A packfile index is also used +/// for efficient access to objects in the packfile. +/// +/// The format of the packfile and its associated index are documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +const Odb = struct { + format: Oid.Format, + pack_file: std.fs.File, + index_header: IndexHeader, + index_file: std.fs.File, + cache: ObjectCache = .{}, + allocator: Allocator, + + /// Initializes the database from open pack and index files. + fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { + try pack_file.seekTo(0); + try index_file.seekTo(0); + const index_header = try IndexHeader.read(index_file.reader()); + return .{ + .format = format, + .pack_file = pack_file, + .index_header = index_header, + .index_file = index_file, + .allocator = allocator, + }; + } + + fn deinit(odb: *Odb) void { + odb.cache.deinit(odb.allocator); + odb.* = undefined; + } + + /// Reads the object at the current position in the database. + fn readObject(odb: *Odb) !Object { + var base_offset = try odb.pack_file.getPos(); + var base_header: EntryHeader = undefined; + var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; + defer delta_offsets.deinit(odb.allocator); + const base_object = while (true) { + if (odb.cache.get(base_offset)) |base_object| break base_object; + + base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(odb.allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; + try odb.pack_file.seekTo(base_offset); + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(odb.allocator, base_offset); + try odb.seekOid(ref_delta.base_object); + base_offset = try odb.pack_file.getPos(); + }, + else => { + const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); + errdefer odb.allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try odb.cache.put(odb.allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain( + odb.allocator, + odb.format, + odb.pack_file, + base_object, + delta_offsets.items, + &odb.cache, + ); + + return .{ .type = base_object.type, .data = base_data }; + } + + /// Seeks to the beginning of the object with the given ID. + fn seekOid(odb: *Odb, oid: Oid) !void { + const oid_length = odb.format.byteLength(); + const key = oid.slice()[0]; + var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; + var end_index = odb.index_header.fan_out_table[key]; + const found_index = while (start_index < end_index) { + const mid_index = start_index + (end_index - start_index) / 2; + try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); + const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); + switch (mem.order(u8, mid_oid.slice(), oid.slice())) { + .lt => start_index = mid_index + 1, + .gt => end_index = mid_index, + .eq => break mid_index, + } + } else return error.ObjectNotFound; + + const n_objects = odb.index_header.fan_out_table[255]; + const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); + try odb.index_file.seekTo(offset_values_start + found_index * 4); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big)); + const pack_offset = pack_offset: { + if (l1_offset.big) { + const l2_offset_values_start = offset_values_start + n_objects * 4; + try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); + break :pack_offset try odb.index_file.reader().readInt(u64, .big); + } else { + break :pack_offset l1_offset.value; + } + }; + + try odb.pack_file.seekTo(pack_offset); + } +}; + +const Object = struct { + type: Type, + data: []const u8, + + const Type = enum { + commit, + tree, + blob, + tag, + }; +}; + +/// A cache for object data. +/// +/// The purpose of this cache is to speed up resolution of deltas by caching the +/// results of resolving delta objects, while maintaining a maximum cache size +/// to avoid excessive memory usage. If the total size of the objects in the +/// cache exceeds the maximum, the cache will begin evicting the least recently +/// used objects: when resolving delta chains, the most recently used objects +/// will likely be more helpful as they will be further along in the chain +/// (skipping earlier reconstruction steps). +/// +/// Object data stored in the cache is managed by the cache. It should not be +/// freed by the caller at any point after inserting it into the cache. Any +/// objects remaining in the cache will be freed when the cache itself is freed. +const ObjectCache = struct { + objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, + lru_nodes: LruList = .{}, + byte_size: usize = 0, + + const max_byte_size = 128 * 1024 * 1024; // 128MiB + /// A list of offsets stored in the cache, with the most recently used + /// entries at the end. + const LruList = std.DoublyLinkedList(u64); + const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; + + fn deinit(cache: *ObjectCache, allocator: Allocator) void { + var object_iterator = cache.objects.iterator(); + while (object_iterator.next()) |object| { + allocator.free(object.value_ptr.object.data); + allocator.destroy(object.value_ptr.lru_node); + } + cache.objects.deinit(allocator); + cache.* = undefined; + } + + /// Gets an object from the cache, moving it to the most recently used + /// position if it is present. + fn get(cache: *ObjectCache, offset: u64) ?Object { + if (cache.objects.get(offset)) |entry| { + cache.lru_nodes.remove(entry.lru_node); + cache.lru_nodes.append(entry.lru_node); + return entry.object; + } else { + return null; + } + } + + /// Puts an object in the cache, possibly evicting older entries if the + /// cache exceeds its maximum size. Note that, although old objects may + /// be evicted, the object just added to the cache with this function + /// will not be evicted before the next call to `put` or `deinit` even if + /// it exceeds the maximum cache size. + fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { + const lru_node = try allocator.create(LruList.Node); + errdefer allocator.destroy(lru_node); + lru_node.data = offset; + + const gop = try cache.objects.getOrPut(allocator, offset); + if (gop.found_existing) { + cache.byte_size -= gop.value_ptr.object.data.len; + cache.lru_nodes.remove(gop.value_ptr.lru_node); + allocator.destroy(gop.value_ptr.lru_node); + allocator.free(gop.value_ptr.object.data); + } + gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; + cache.byte_size += object.data.len; + cache.lru_nodes.append(lru_node); + + while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { + // The > 1 check is to make sure that we don't evict the most + // recently added node, even if it by itself happens to exceed the + // maximum size of the cache. + const evict_node = cache.lru_nodes.popFirst().?; + const evict_offset = evict_node.data; + allocator.destroy(evict_node); + const evict_object = cache.objects.get(evict_offset).?.object; + cache.byte_size -= evict_object.data.len; + allocator.free(evict_object.data); + _ = cache.objects.remove(evict_offset); + } + } +}; + +/// A single pkt-line in the Git protocol. +/// +/// The format of a pkt-line is documented in +/// [protocol-common](https://git-scm.com/docs/protocol-common). The special +/// meanings of the delimiter and response-end packets are documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +const Packet = union(enum) { + flush, + delimiter, + response_end, + data: []const u8, + + const max_data_length = 65516; + + /// Reads a packet in pkt-line format. + fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { + const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; + switch (length) { + 0 => return .flush, + 1 => return .delimiter, + 2 => return .response_end, + 3 => return error.InvalidPacket, + else => if (length - 4 > max_data_length) return error.InvalidPacket, + } + const data = buf[0 .. length - 4]; + try reader.readNoEof(data); + return .{ .data = data }; + } + + /// Writes a packet in pkt-line format. + fn write(packet: Packet, writer: anytype) !void { + switch (packet) { + .flush => try writer.writeAll("0000"), + .delimiter => try writer.writeAll("0001"), + .response_end => try writer.writeAll("0002"), + .data => |data| { + assert(data.len <= max_data_length); + try writer.print("{x:0>4}", .{data.len + 4}); + try writer.writeAll(data); + }, + } + } + + /// Returns the normalized form of textual packet data, stripping any + /// trailing '\n'. + /// + /// As documented in + /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), + /// non-binary (textual) pkt-line data should contain a trailing '\n', but + /// is not required to do so (implementations must support both forms). + fn normalizeText(data: []const u8) []const u8 { + return if (mem.endsWith(u8, data, "\n")) + data[0 .. data.len - 1] + else + data; + } +}; + +/// A client session for the Git protocol, currently limited to an HTTP(S) +/// transport. Only protocol version 2 is supported, as documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +pub const Session = struct { + transport: *std.http.Client, + location: Location, + supports_agent: bool, + supports_shallow: bool, + object_format: Oid.Format, + allocator: Allocator, + + const agent = "zig/" ++ @import("builtin").zig_version_string; + const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); + + /// Initializes a client session and discovers the capabilities of the + /// server for optimal transport. + pub fn init( + allocator: Allocator, + transport: *std.http.Client, + uri: std.Uri, + http_headers_buffer: []u8, + ) !Session { + var session: Session = .{ + .transport = transport, + .location = try .init(allocator, uri), + .supports_agent = false, + .supports_shallow = false, + .object_format = .sha1, + .allocator = allocator, + }; + errdefer session.deinit(); + var capability_iterator = try session.getCapabilities(http_headers_buffer); + defer capability_iterator.deinit(); + while (try capability_iterator.next()) |capability| { + if (mem.eql(u8, capability.key, "agent")) { + session.supports_agent = true; + } else if (mem.eql(u8, capability.key, "fetch")) { + var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); + while (feature_iterator.next()) |feature| { + if (mem.eql(u8, feature, "shallow")) { + session.supports_shallow = true; + } + } + } else if (mem.eql(u8, capability.key, "object-format")) { + if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { + session.object_format = format; + } + } + } + return session; + } + + pub fn deinit(session: *Session) void { + session.location.deinit(session.allocator); + session.* = undefined; + } + + /// An owned `std.Uri` representing the location of the server (base URI). + const Location = struct { + uri: std.Uri, + + fn init(allocator: Allocator, uri: std.Uri) !Location { + const scheme = try allocator.dupe(u8, uri.scheme); + errdefer allocator.free(scheme); + const user = if (uri.user) |user| try std.fmt.allocPrint(allocator, "{user}", .{user}) else null; + errdefer if (user) |s| allocator.free(s); + const password = if (uri.password) |password| try std.fmt.allocPrint(allocator, "{password}", .{password}) else null; + errdefer if (password) |s| allocator.free(s); + const host = if (uri.host) |host| try std.fmt.allocPrint(allocator, "{host}", .{host}) else null; + errdefer if (host) |s| allocator.free(s); + const path = try std.fmt.allocPrint(allocator, "{path}", .{uri.path}); + errdefer allocator.free(path); + // The query and fragment are not used as part of the base server URI. + return .{ + .uri = .{ + .scheme = scheme, + .user = if (user) |s| .{ .percent_encoded = s } else null, + .password = if (password) |s| .{ .percent_encoded = s } else null, + .host = if (host) |s| .{ .percent_encoded = s } else null, + .port = uri.port, + .path = .{ .percent_encoded = path }, + }, + }; + } + + fn deinit(loc: *Location, allocator: Allocator) void { + allocator.free(loc.uri.scheme); + if (loc.uri.user) |user| allocator.free(user.percent_encoded); + if (loc.uri.password) |password| allocator.free(password.percent_encoded); + if (loc.uri.host) |host| allocator.free(host.percent_encoded); + allocator.free(loc.uri.path.percent_encoded); + } + }; + + /// Returns an iterator over capabilities supported by the server. + /// + /// The `session.location` is updated if the server returns a redirect, so + /// that subsequent session functions do not need to handle redirects. + fn getCapabilities(session: *Session, http_headers_buffer: []u8) !CapabilityIterator { + var info_refs_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); + defer session.allocator.free(session_uri_path); + info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "info/refs" }) }; + } + defer session.allocator.free(info_refs_uri.path.percent_encoded); + info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; + info_refs_uri.fragment = null; + + const max_redirects = 3; + var request = try session.transport.open(.GET, info_refs_uri, .{ + .redirect_behavior = @enumFromInt(max_redirects), + .server_header_buffer = http_headers_buffer, + .extra_headers = &.{ + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }); + errdefer request.deinit(); + try request.send(); + try request.finish(); + + try request.wait(); + if (request.response.status != .ok) return error.ProtocolError; + const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; + if (any_redirects_occurred) { + const request_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{request.uri.path}); + defer session.allocator.free(request_uri_path); + if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; + var new_uri = request.uri; + new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; + const new_location: Location = try .init(session.allocator, new_uri); + session.location.deinit(session.allocator); + session.location = new_location; + } + + const reader = request.reader(); + var buf: [Packet.max_data_length]u8 = undefined; + var state: enum { response_start, response_content } = .response_start; + while (true) { + // Some Git servers (at least GitHub) include an additional + // '# service=git-upload-pack' informative response before sending + // the expected 'version 2' packet and capability information. + // This is not universal: SourceHut, for example, does not do this. + // Thus, we need to skip any such useless additional responses + // before we get the one we're actually looking for. The responses + // will be delimited by flush packets. + const packet = Packet.read(reader, &buf) catch |e| switch (e) { + error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found + else => |other| return other, + }; + switch (packet) { + .flush => state = .response_start, + .data => |data| switch (state) { + .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { + return .{ .request = request }; + } else { + state = .response_content; + }, + else => {}, + }, + else => return error.UnexpectedPacket, + } + } + } + + const CapabilityIterator = struct { + request: std.http.Client.Request, + buf: [Packet.max_data_length]u8 = undefined, + + const Capability = struct { + key: []const u8, + value: ?[]const u8 = null, + + fn parse(data: []const u8) Capability { + return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| + .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } + else + .{ .key = data }; + } + }; + + fn deinit(iterator: *CapabilityIterator) void { + iterator.request.deinit(); + iterator.* = undefined; + } + + fn next(iterator: *CapabilityIterator) !?Capability { + switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { + .flush => return null, + .data => |data| return Capability.parse(Packet.normalizeText(data)), + else => return error.UnexpectedPacket, + } + } + }; + + const ListRefsOptions = struct { + /// The ref prefixes (if any) to use to filter the refs available on the + /// server. Note that the client must still check the returned refs + /// against its desired filters itself: the server is not required to + /// respect these prefix filters and may return other refs as well. + ref_prefixes: []const []const u8 = &.{}, + /// Whether to include symref targets for returned symbolic refs. + include_symrefs: bool = false, + /// Whether to include the peeled object ID for returned tag refs. + include_peeled: bool = false, + server_header_buffer: []u8, + }; + + /// Returns an iterator over refs known to the server. + pub fn listRefs(session: Session, options: ListRefsOptions) !RefIterator { + var upload_pack_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); + defer session.allocator.free(session_uri_path); + upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; + } + defer session.allocator.free(upload_pack_uri.path.percent_encoded); + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var body: std.ArrayListUnmanaged(u8) = .empty; + defer body.deinit(session.allocator); + const body_writer = body.writer(session.allocator); + try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, body_writer); + } + { + const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); + defer session.allocator.free(object_format_packet); + try Packet.write(.{ .data = object_format_packet }, body_writer); + } + try Packet.write(.delimiter, body_writer); + for (options.ref_prefixes) |ref_prefix| { + const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix}); + defer session.allocator.free(ref_prefix_packet); + try Packet.write(.{ .data = ref_prefix_packet }, body_writer); + } + if (options.include_symrefs) { + try Packet.write(.{ .data = "symrefs\n" }, body_writer); + } + if (options.include_peeled) { + try Packet.write(.{ .data = "peel\n" }, body_writer); + } + try Packet.write(.flush, body_writer); + + var request = try session.transport.open(.POST, upload_pack_uri, .{ + .redirect_behavior = .unhandled, + .server_header_buffer = options.server_header_buffer, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }); + errdefer request.deinit(); + request.transfer_encoding = .{ .content_length = body.items.len }; + try request.send(); + try request.writeAll(body.items); + try request.finish(); + + try request.wait(); + if (request.response.status != .ok) return error.ProtocolError; + + return .{ + .format = session.object_format, + .request = request, + }; + } + + pub const RefIterator = struct { + format: Oid.Format, + request: std.http.Client.Request, + buf: [Packet.max_data_length]u8 = undefined, + + pub const Ref = struct { + oid: Oid, + name: []const u8, + symref_target: ?[]const u8, + peeled: ?Oid, + }; + + pub fn deinit(iterator: *RefIterator) void { + iterator.request.deinit(); + iterator.* = undefined; + } + + pub fn next(iterator: *RefIterator) !?Ref { + switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { + .flush => return null, + .data => |data| { + const ref_data = Packet.normalizeText(data); + const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; + const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; + + const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; + const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; + + var symref_target: ?[]const u8 = null; + var peeled: ?Oid = null; + var last_sep_pos = name_sep_pos; + while (last_sep_pos < ref_data.len) { + const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; + const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; + if (mem.startsWith(u8, attribute, "symref-target:")) { + symref_target = attribute["symref-target:".len..]; + } else if (mem.startsWith(u8, attribute, "peeled:")) { + peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; + } + last_sep_pos = next_sep_pos; + } + + return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; + }, + else => return error.UnexpectedPacket, + } + } + }; + + /// Fetches the given refs from the server. A shallow fetch (depth 1) is + /// performed if the server supports it. + pub fn fetch( + session: Session, + wants: []const []const u8, + http_headers_buffer: []u8, + ) !FetchStream { + var upload_pack_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); + defer session.allocator.free(session_uri_path); + upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; + } + defer session.allocator.free(upload_pack_uri.path.percent_encoded); + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var body: std.ArrayListUnmanaged(u8) = .empty; + defer body.deinit(session.allocator); + const body_writer = body.writer(session.allocator); + try Packet.write(.{ .data = "command=fetch\n" }, body_writer); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, body_writer); + } + { + const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); + defer session.allocator.free(object_format_packet); + try Packet.write(.{ .data = object_format_packet }, body_writer); + } + try Packet.write(.delimiter, body_writer); + // Our packfile parser supports the OFS_DELTA object type + try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); + // We do not currently convey server progress information to the user + try Packet.write(.{ .data = "no-progress\n" }, body_writer); + if (session.supports_shallow) { + try Packet.write(.{ .data = "deepen 1\n" }, body_writer); + } + for (wants) |want| { + var buf: [Packet.max_data_length]u8 = undefined; + const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; + try Packet.write(.{ .data = arg }, body_writer); + } + try Packet.write(.{ .data = "done\n" }, body_writer); + try Packet.write(.flush, body_writer); + + var request = try session.transport.open(.POST, upload_pack_uri, .{ + .redirect_behavior = .not_allowed, + .server_header_buffer = http_headers_buffer, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }); + errdefer request.deinit(); + request.transfer_encoding = .{ .content_length = body.items.len }; + try request.send(); + try request.writeAll(body.items); + try request.finish(); + + try request.wait(); + if (request.response.status != .ok) return error.ProtocolError; + + const reader = request.reader(); + // We are not interested in any of the sections of the returned fetch + // data other than the packfile section, since we aren't doing anything + // complex like ref negotiation (this is a fresh clone). + var state: enum { section_start, section_content } = .section_start; + while (true) { + var buf: [Packet.max_data_length]u8 = undefined; + const packet = try Packet.read(reader, &buf); + switch (state) { + .section_start => switch (packet) { + .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { + return .{ .request = request }; + } else { + state = .section_content; + }, + else => return error.UnexpectedPacket, + }, + .section_content => switch (packet) { + .delimiter => state = .section_start, + .data => {}, + else => return error.UnexpectedPacket, + }, + } + } + } + + pub const FetchStream = struct { + request: std.http.Client.Request, + buf: [Packet.max_data_length]u8 = undefined, + pos: usize = 0, + len: usize = 0, + + pub fn deinit(stream: *FetchStream) void { + stream.request.deinit(); + } + + pub const ReadError = std.http.Client.Request.ReadError || error{ + InvalidPacket, + ProtocolError, + UnexpectedPacket, + }; + pub const Reader = std.io.Reader(*FetchStream, ReadError, read); + + const StreamCode = enum(u8) { + pack_data = 1, + progress = 2, + fatal_error = 3, + _, + }; + + pub fn reader(stream: *FetchStream) Reader { + return .{ .context = stream }; + } + + pub fn read(stream: *FetchStream, buf: []u8) !usize { + if (stream.pos == stream.len) { + while (true) { + switch (try Packet.read(stream.request.reader(), &stream.buf)) { + .flush => return 0, + .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { + .pack_data => { + stream.pos = 1; + stream.len = data.len; + break; + }, + .fatal_error => return error.ProtocolError, + else => {}, + }, + else => return error.UnexpectedPacket, + } + } + } + + const size = @min(buf.len, stream.len - stream.pos); + @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); + stream.pos += size; + return size; + } + }; +}; + +const PackHeader = struct { + total_objects: u32, + + const signature = "PACK"; + const supported_version = 2; + + fn read(reader: anytype) !PackHeader { + const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; + const version = reader.readInt(u32, .big) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (version != supported_version) return error.UnsupportedVersion; + const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + return .{ .total_objects = total_objects }; + } +}; + +const EntryHeader = union(Type) { + commit: Undeltified, + tree: Undeltified, + blob: Undeltified, + tag: Undeltified, + ofs_delta: OfsDelta, + ref_delta: RefDelta, + + const Type = enum(u3) { + commit = 1, + tree = 2, + blob = 3, + tag = 4, + ofs_delta = 6, + ref_delta = 7, + }; + + const Undeltified = struct { + uncompressed_length: u64, + }; + + const OfsDelta = struct { + offset: u64, + uncompressed_length: u64, + }; + + const RefDelta = struct { + base_object: Oid, + uncompressed_length: u64, + }; + + fn objectType(header: EntryHeader) Object.Type { + return switch (header) { + inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), + else => unreachable, + }; + } + + fn uncompressedLength(header: EntryHeader) u64 { + return switch (header) { + inline else => |entry| entry.uncompressed_length, + }; + } + + fn read(format: Oid.Format, reader: anytype) !EntryHeader { + const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; + const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }); + const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; + var uncompressed_length: u64 = initial.len; + uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; + const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; + return switch (@"type") { + inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ + .uncompressed_length = uncompressed_length, + }), + .ofs_delta => .{ .ofs_delta = .{ + .offset = try readOffsetVarInt(reader), + .uncompressed_length = uncompressed_length, + } }, + .ref_delta => .{ .ref_delta = .{ + .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }, + .uncompressed_length = uncompressed_length, + } }, + }; + } +}; + +fn readSizeVarInt(r: anytype) !u64 { + const Byte = packed struct { value: u7, has_next: bool }; + var b: Byte = @bitCast(try r.readByte()); + var value: u64 = b.value; + var shift: u6 = 0; + while (b.has_next) { + b = @bitCast(try r.readByte()); + shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; + value |= @as(u64, b.value) << shift; + } + return value; +} + +fn readOffsetVarInt(r: anytype) !u64 { + const Byte = packed struct { value: u7, has_next: bool }; + var b: Byte = @bitCast(try r.readByte()); + var value: u64 = b.value; + while (b.has_next) { + b = @bitCast(try r.readByte()); + value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; + value |= b.value; + } + return value; +} + +const IndexHeader = struct { + fan_out_table: [256]u32, + + const signature = "\xFFtOc"; + const supported_version = 2; + const size = 4 + 4 + @sizeOf([256]u32); + + fn read(reader: anytype) !IndexHeader { + var header_bytes = try reader.readBytesNoEof(size); + if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; + const version = mem.readInt(u32, header_bytes[4..8], .big); + if (version != supported_version) return error.UnsupportedVersion; + + var fan_out_table: [256]u32 = undefined; + var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); + const fan_out_table_reader = fan_out_table_stream.reader(); + for (&fan_out_table) |*entry| { + entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; + } + return .{ .fan_out_table = fan_out_table }; + } +}; + +const IndexEntry = struct { + offset: u64, + crc32: u32, +}; + +/// Writes out a version 2 index for the given packfile, as documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { + try pack.seekTo(0); + + var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; + defer index_entries.deinit(allocator); + var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty; + defer pending_deltas.deinit(allocator); + + const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); + + var cache: ObjectCache = .{}; + defer cache.deinit(allocator); + var remaining_deltas = pending_deltas.items.len; + while (remaining_deltas > 0) { + var i: usize = remaining_deltas; + while (i > 0) { + i -= 1; + const delta = pending_deltas.items[i]; + if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { + try index_entries.put(allocator, oid, delta); + _ = pending_deltas.swapRemove(i); + } + } + if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; + remaining_deltas = pending_deltas.items.len; + } + + var oids: std.ArrayListUnmanaged(Oid) = .empty; + defer oids.deinit(allocator); + try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); + var index_entries_iter = index_entries.iterator(); + while (index_entries_iter.next()) |entry| { + oids.appendAssumeCapacity(entry.key_ptr.*); + } + mem.sortUnstable(Oid, oids.items, {}, struct { + fn lessThan(_: void, o1: Oid, o2: Oid) bool { + return mem.lessThan(u8, o1.slice(), o2.slice()); + } + }.lessThan); + + var fan_out_table: [256]u32 = undefined; + var count: u32 = 0; + var fan_out_index: u8 = 0; + for (oids.items) |oid| { + const key = oid.slice()[0]; + if (key > fan_out_index) { + @memset(fan_out_table[fan_out_index..key], count); + fan_out_index = key; + } + count += 1; + } + @memset(fan_out_table[fan_out_index..], count); + + var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); + const writer = index_hashed_writer.writer(); + try writer.writeAll(IndexHeader.signature); + try writer.writeInt(u32, IndexHeader.supported_version, .big); + for (fan_out_table) |fan_out_entry| { + try writer.writeInt(u32, fan_out_entry, .big); + } + + for (oids.items) |oid| { + try writer.writeAll(oid.slice()); + } + + for (oids.items) |oid| { + try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); + } + + var big_offsets: std.ArrayListUnmanaged(u64) = .empty; + defer big_offsets.deinit(allocator); + for (oids.items) |oid| { + const offset = index_entries.get(oid).?.offset; + if (offset <= std.math.maxInt(u31)) { + try writer.writeInt(u32, @intCast(offset), .big); + } else { + const index = big_offsets.items.len; + try big_offsets.append(allocator, offset); + try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); + } + } + for (big_offsets.items) |offset| { + try writer.writeInt(u64, offset, .big); + } + + try writer.writeAll(pack_checksum.slice()); + const index_checksum = index_hashed_writer.hasher.finalResult(); + try index_writer.writeAll(index_checksum.slice()); +} + +/// Performs the first pass over the packfile data for index construction. +/// This will index all non-delta objects, queue delta objects for further +/// processing, and return the pack checksum (which is part of the index +/// format). +fn indexPackFirstPass( + allocator: Allocator, + format: Oid.Format, + pack: std.fs.File, + index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), + pending_deltas: *std.ArrayListUnmanaged(IndexEntry), +) !Oid { + var pack_buffered_reader = std.io.bufferedReader(pack.reader()); + var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); + var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); + const pack_reader = pack_hashed_reader.reader(); + + const pack_header = try PackHeader.read(pack_reader); + + var current_entry: u32 = 0; + while (current_entry < pack_header.total_objects) : (current_entry += 1) { + const entry_offset = pack_counting_reader.bytes_read; + var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); + const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); + switch (entry_header) { + .commit, .tree, .blob, .tag => |object| { + var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); + var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); + var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); + const entry_writer = entry_hashed_writer.writer(); + // The object header is not included in the pack data but is + // part of the object's ID + try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(entry_counting_reader.reader(), entry_writer); + if (entry_counting_reader.bytes_read != object.uncompressed_length) { + return error.InvalidObject; + } + const oid = entry_hashed_writer.hasher.finalResult(); + try index_entries.put(allocator, oid, .{ + .offset = entry_offset, + .crc32 = entry_crc32_reader.hasher.final(), + }); + }, + inline .ofs_delta, .ref_delta => |delta| { + var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); + var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); + if (entry_counting_reader.bytes_read != delta.uncompressed_length) { + return error.InvalidObject; + } + try pending_deltas.append(allocator, .{ + .offset = entry_offset, + .crc32 = entry_crc32_reader.hasher.final(), + }); + }, + } + } + + const pack_checksum = pack_hashed_reader.hasher.finalResult(); + const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); + if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { + return error.CorruptedPack; + } + _ = pack_reader.readByte() catch |e| switch (e) { + error.EndOfStream => return pack_checksum, + else => |other| return other, + }; + return error.InvalidFormat; +} + +/// Attempts to determine the final object ID of the given deltified object. +/// May return null if this is not yet possible (if the delta is a ref-based +/// delta and we do not yet know the offset of the base object). +fn indexPackHashDelta( + allocator: Allocator, + format: Oid.Format, + pack: std.fs.File, + delta: IndexEntry, + index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), + cache: *ObjectCache, +) !?Oid { + // Figure out the chain of deltas to resolve + var base_offset = delta.offset; + var base_header: EntryHeader = undefined; + var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; + defer delta_offsets.deinit(allocator); + const base_object = while (true) { + if (cache.get(base_offset)) |base_object| break base_object; + + try pack.seekTo(base_offset); + base_header = try EntryHeader.read(format, pack.reader()); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; + }, + else => { + const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); + errdefer allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try cache.put(allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); + + var entry_hasher: Oid.Hasher = .init(format); + var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); + try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); + entry_hasher.update(base_data); + return entry_hasher.finalResult(); +} + +/// Resolves a chain of deltas, returning the final base object data. `pack` is +/// assumed to be looking at the start of the object data for the base object of +/// the chain, and will then apply the deltas in `delta_offsets` in reverse order +/// to obtain the final object. +fn resolveDeltaChain( + allocator: Allocator, + format: Oid.Format, + pack: std.fs.File, + base_object: Object, + delta_offsets: []const u64, + cache: *ObjectCache, +) ![]const u8 { + var base_data = base_object.data; + var i: usize = delta_offsets.len; + while (i > 0) { + i -= 1; + + const delta_offset = delta_offsets[i]; + try pack.seekTo(delta_offset); + const delta_header = try EntryHeader.read(format, pack.reader()); + const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); + defer allocator.free(delta_data); + var delta_stream = std.io.fixedBufferStream(delta_data); + const delta_reader = delta_stream.reader(); + _ = try readSizeVarInt(delta_reader); // base object size + const expanded_size = try readSizeVarInt(delta_reader); + + const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; + const expanded_data = try allocator.alloc(u8, expanded_alloc_size); + errdefer allocator.free(expanded_data); + var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); + var base_stream = std.io.fixedBufferStream(base_data); + try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); + if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; + + try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); + base_data = expanded_data; + } + return base_data; +} + +/// Reads the complete contents of an object from `reader`. This function may +/// read more bytes than required from `reader`, so the reader position after +/// returning is not reliable. +fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { + const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; + var buffered_reader = std.io.bufferedReader(reader); + var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); + const data = try allocator.alloc(u8, alloc_size); + errdefer allocator.free(data); + try decompress_stream.reader().readNoEof(data); + _ = decompress_stream.reader().readByte() catch |e| switch (e) { + error.EndOfStream => return data, + else => |other| return other, + }; + return error.InvalidFormat; +} + +/// Expands delta data from `delta_reader` to `writer`. `base_object` must +/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). +/// +/// The format of the delta data is documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { + while (true) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { + error.EndOfStream => return, + else => |other| return other, + }); + if (inst.copy) { + const available: packed struct { + offset1: bool, + offset2: bool, + offset3: bool, + offset4: bool, + size1: bool, + size2: bool, + size3: bool, + } = @bitCast(inst.value); + const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ + .offset1 = if (available.offset1) try delta_reader.readByte() else 0, + .offset2 = if (available.offset2) try delta_reader.readByte() else 0, + .offset3 = if (available.offset3) try delta_reader.readByte() else 0, + .offset4 = if (available.offset4) try delta_reader.readByte() else 0, + }; + const offset: u32 = @bitCast(offset_parts); + const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ + .size1 = if (available.size1) try delta_reader.readByte() else 0, + .size2 = if (available.size2) try delta_reader.readByte() else 0, + .size3 = if (available.size3) try delta_reader.readByte() else 0, + }; + var size: u24 = @bitCast(size_parts); + if (size == 0) size = 0x10000; + try base_object.seekTo(offset); + var copy_reader = std.io.limitedReader(base_object.reader(), size); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(copy_reader.reader(), writer); + } else if (inst.value != 0) { + var data_reader = std.io.limitedReader(delta_reader, inst.value); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); + try fifo.pump(data_reader.reader(), writer); + } else { + return error.InvalidDeltaInstruction; + } + } +} + +/// Runs the packfile indexing and checkout test. +/// +/// The two testrepo repositories under testdata contain identical commit +/// histories and contents. +/// +/// To verify the contents of the packfiles using Git alone, run the +/// following commands in an empty directory: +/// +/// 1. `git init --object-format=(sha1|sha256)` +/// 2. `git unpack-objects assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .id = p.id, + .version = p.version, + .version_node = p.version_node, + .dependencies = try p.dependencies.clone(p.arena), + .dependencies_node = p.dependencies_node, + .paths = try p.paths.clone(p.arena), + .minimum_zig_version = p.minimum_zig_version, + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +pub fn copyErrorsIntoBundle( + man: Manifest, + ast: Ast, + /// ErrorBundle null-terminated string index + src_path: u32, + eb: *std.zig.ErrorBundle.Wip, +) Allocator.Error!void { + for (man.errors) |msg| { + const start_loc = ast.tokenLocation(0, msg.tok); + + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = ast.tokenStart(msg.tok), + .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), + .span_main = ast.tokenStart(msg.tok) + msg.off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }), + }); + } +} + +const Parse = struct { + gpa: Allocator, + ast: Ast, + arena: Allocator, + buf: std.ArrayListUnmanaged(u8), + errors: std.ArrayListUnmanaged(ErrorMessage), + + name: []const u8, + id: u32, + version: std.SemanticVersion, + version_node: Ast.Node.Index, + dependencies: std.StringArrayHashMapUnmanaged(Dependency), + dependencies_node: Ast.Node.OptionalIndex, + paths: std.StringArrayHashMapUnmanaged(void), + allow_missing_paths_field: bool, + allow_name_string: bool, + allow_missing_fingerprint: bool, + minimum_zig_version: ?std.SemanticVersion, + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + var have_included_paths = false; + var fingerprint: ?Package.Fingerprint = null; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + p.dependencies_node = field_init.toOptional(); + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "paths")) { + have_included_paths = true; + try parseIncludedPaths(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseName(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "fingerprint")) { + fingerprint = try parseFingerprint(p, field_init); + } else if (mem.eql(u8, field_name, "version")) { + p.version_node = field_init; + const version_text = try parseString(p, field_init); + if (version_text.len > max_version_len) { + try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); + } + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else if (mem.eql(u8, field_name, "minimum_zig_version")) { + const version_text = try parseString(p, field_init); + p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v null; + }; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } else { + if (fingerprint) |n| { + if (!n.validate(p.name)) { + return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ + n.int(), Package.Fingerprint.generate(p.name).int(), + }); + } + p.id = n.id; + } else if (!p.allow_missing_fingerprint) { + try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ + Package.Fingerprint.generate(p.name).int(), + }); + } else { + p.id = 0; + } + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + + if (!have_included_paths) { + if (p.allow_missing_paths_field) { + try p.paths.put(p.gpa, "", {}); + } else { + try appendError(p, main_token, "missing top-level 'paths' field", .{}); + } + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .location = undefined, + .location_tok = undefined, + .location_node = undefined, + .hash = null, + .hash_tok = .none, + .hash_node = .none, + .node = node, + .name_tok = undefined, + .lazy = false, + }; + var has_location = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + dep.name_tok = name_token; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + if (has_location) { + return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = ast.nodeMainToken(field_init); + dep.location_node = field_init; + } else if (mem.eql(u8, field_name, "path")) { + if (has_location) { + return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .path = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = ast.nodeMainToken(field_init); + dep.location_node = field_init; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); + dep.hash_node = field_init.toOptional(); + } else if (mem.eql(u8, field_name, "lazy")) { + dep.lazy = parseBool(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!has_location) { + try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); + } + + return dep; + } + + fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected paths expression to be a list of strings", .{}); + }; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parseString(p, elem_node); + // This is normalized so that it can be used in string comparisons + // against file system paths. + const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); + try p.paths.put(p.gpa, normalized, {}); + } + } + + fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { + const ast = p.ast; + if (ast.nodeTag(node) != .identifier) { + return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); + } + const ident_token = ast.nodeMainToken(node); + const token_bytes = ast.tokenSlice(ident_token); + if (mem.eql(u8, token_bytes, "true")) { + return true; + } else if (mem.eql(u8, token_bytes, "false")) { + return false; + } else { + return fail(p, ident_token, "expected boolean", .{}); + } + } + + fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + if (ast.nodeTag(node) != .number_literal) { + return fail(p, main_token, "expected integer literal", .{}); + } + const token_bytes = ast.tokenSlice(main_token); + const parsed = std.zig.parseNumberLiteral(token_bytes); + switch (parsed) { + .int => |n| return @bitCast(n), + .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ + @tagName(parsed), + }), + .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), + } + } + + fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + + if (p.allow_name_string and ast.nodeTag(node) == .string_literal) { + const name = try parseString(p, node); + if (!std.zig.isValidId(name)) + return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{}); + + if (name.len > max_name_len) + return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ + std.zig.fmtId(name), max_name_len, + }); + + return name; + } + + if (ast.nodeTag(node) != .enum_literal) + return fail(p, main_token, "expected enum literal", .{}); + + const ident_name = ast.tokenSlice(main_token); + if (mem.startsWith(u8, ident_name, "@")) + return fail(p, main_token, "name must be a valid bare zig identifier", .{}); + + if (ident_name.len > max_name_len) + return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ + std.zig.fmtId(ident_name), max_name_len, + }); + + return ident_name; + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + if (ast.nodeTag(node) != .string_literal) { + return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); + } + const str_lit_token = ast.nodeMainToken(node); + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const tok = ast.nodeMainToken(node); + const h = try parseString(p, node); + + if (h.len > Package.Hash.max_len) { + return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len}); + } + + return h; + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + assert(ast.tokenTag(token) == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayListUnmanaged(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + var buf_managed = buf.toManaged(p.gpa); + const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); + buf.* = buf_managed.moveToUnmanaged(); + switch (try result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "unicode escape does not correspond to a valid unicode scalar value", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_char_literal => { + try p.appendErrorOff(token, offset, "empty character literal", .{}); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .paths = .{""}, + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + \\ }, + \\ }, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast, .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].location.url, + ); + try testing.expectEqualStrings( + "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); + + try testing.expect(manifest.minimum_zig_version == null); +} + +test "minimum_zig_version" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .paths = .{""}, + \\ .minimum_zig_version = "0.11.1", + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast, .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expect(manifest.dependencies.count() == 0); + + try testing.expect(manifest.minimum_zig_version != null); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 0, + .minor = 11, + .patch = 1, + }), manifest.minimum_zig_version.?); +} + +test "minimum_zig_version - invalid version" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .minimum_zig_version = "X.11.1", + \\ .paths = .{""}, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast, .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 1); + try testing.expect(manifest.dependencies.count() == 0); + + try testing.expect(manifest.minimum_zig_version == null); +} diff --git a/lib/std/zig/Package/Templates.zig b/lib/std/zig/Package/Templates.zig new file mode 100644 index 0000000000000000000000000000000000000000..4a4a790d3293760fa75d3a8e85a0e31d2f60f651 --- /dev/null +++ b/lib/std/zig/Package/Templates.zig @@ -0,0 +1,90 @@ +const std = @import("../../std.zig"); +const Directory = std.Build.Cache.Directory; +const fs = std.fs; +const Allocator = std.mem.Allocator; +const fatal = std.process.fatal; + +const Templates = @This(); + +zig_lib_directory: Directory, +dir: fs.Dir, +buffer: std.ArrayListUnmanaged(u8), + +fn find(gpa: Allocator, zig_lib_directory: Directory) Templates { + const s = fs.path.sep_str; + const template_sub_path = "init"; + const template_dir = zig_lib_directory.handle.openDir(template_sub_path, .{}) catch |err| { + const path = zig_lib_directory.path orelse "."; + fatal("unable to open zig project template directory '{s}{s}{s}': {s}", .{ + path, s, template_sub_path, @errorName(err), + }); + }; + + return .{ + .zig_lib_directory = zig_lib_directory, + .dir = template_dir, + .buffer = std.ArrayListUnmanaged(u8).init(gpa), + }; +} + +fn deinit(templates: *Templates, gpa: Allocator) void { + templates.zig_lib_directory.handle.close(); + templates.dir.close(); + templates.buffer.deinit(gpa); + templates.* = undefined; +} + +fn write( + templates: *Templates, + gpa: Allocator, + out_dir: fs.Dir, + root_name: []const u8, + template_path: []const u8, + fingerprint: std.zig.Package.Fingerprint, + zig_version_string: []const u8, +) !void { + if (fs.path.dirname(template_path)) |dirname| { + out_dir.makePath(dirname) catch |err| { + fatal("unable to make path '{s}': {s}", .{ dirname, @errorName(err) }); + }; + } + + const max_bytes = 10 * 1024 * 1024; + const contents = templates.dir.readFileAlloc(gpa, template_path, max_bytes) catch |err| { + fatal("unable to read template file '{s}': {s}", .{ template_path, @errorName(err) }); + }; + defer gpa.free(contents); + templates.buffer.clearRetainingCapacity(); + try templates.buffer.ensureUnusedCapacity(gpa, contents.len); + var i: usize = 0; + while (i < contents.len) { + if (contents[i] == '.') { + if (std.mem.startsWith(u8, contents[i..], ".LITNAME")) { + try templates.buffer.append(gpa, '.'); + try templates.buffer.appendSlice(gpa, root_name); + i += ".LITNAME".len; + continue; + } else if (std.mem.startsWith(u8, contents[i..], ".NAME")) { + try templates.buffer.appendSlice(gpa, root_name); + i += ".NAME".len; + continue; + } else if (std.mem.startsWith(u8, contents[i..], ".FINGERPRINT")) { + try templates.buffer.writer(gpa).print("0x{x}", .{fingerprint.int()}); + i += ".FINGERPRINT".len; + continue; + } else if (std.mem.startsWith(u8, contents[i..], ".ZIGVER")) { + try templates.buffer.appendSlice(gpa, zig_version_string); + i += ".ZIGVER".len; + continue; + } + } + try templates.buffer.append(gpa, contents[i]); + i += 1; + } + + return out_dir.writeFile(.{ + .sub_path = template_path, + .data = templates.buffer.items, + .flags = .{ .exclusive = true }, + }); +} diff --git a/src/Module.zig b/src/Module.zig new file mode 100644 index 0000000000000000000000000000000000000000..0dec7bde76e5f55ec0be6bb34ae465eeac0e94d4 --- /dev/null +++ b/src/Module.zig @@ -0,0 +1,563 @@ +//! Corresponds to something that Zig source code can `@import`. + +/// Only files inside this directory can be imported. +root: Cache.Path, +/// Relative to `root`. May contain path separators. +root_src_path: []const u8, +/// Name used in compile errors. Looks like "root.foo.bar". +fully_qualified_name: []const u8, +/// The dependency table of this module. Shared dependencies such as 'std', +/// 'builtin', and 'root' are not specified in every dependency table, but +/// instead only in the table of `main_mod`. `Module.importFile` is +/// responsible for detecting these names and using the correct package. +deps: Deps = .{}, + +resolved_target: ResolvedTarget, +optimize_mode: std.builtin.OptimizeMode, +code_model: std.builtin.CodeModel, +single_threaded: bool, +error_tracing: bool, +valgrind: bool, +pic: bool, +strip: bool, +omit_frame_pointer: bool, +stack_check: bool, +stack_protector: u32, +red_zone: bool, +sanitize_c: bool, +sanitize_thread: bool, +fuzz: bool, +unwind_tables: std.builtin.UnwindTables, +cc_argv: []const []const u8, +/// (SPIR-V) whether to generate a structured control flow graph or not +structured_cfg: bool, +no_builtin: bool, + +/// If the module is an `@import("builtin")` module, this is the `File` that +/// is preallocated for it. Otherwise this field is null. +builtin_file: ?*File, + +pub const Deps = std.StringArrayHashMapUnmanaged(*Module); + +pub fn isBuiltin(m: Module) bool { + return m.builtin_file != null; +} + +pub const Tree = struct { + /// Each `Package` exposes a `Module` with build.zig as its root source file. + build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), +}; + +pub const CreateOptions = struct { + /// Where to store builtin.zig. The global cache directory is used because + /// it is a pure function based on CLI flags. + global_cache_directory: Cache.Directory, + paths: Paths, + fully_qualified_name: []const u8, + + cc_argv: []const []const u8, + inherited: Inherited, + global: Compilation.Config, + /// If this is null then `resolved_target` must be non-null. + parent: ?*Package.Module, + + builtin_mod: ?*Package.Module, + + /// Allocated into the given `arena`. Should be shared across all module creations in a Compilation. + /// Ignored if `builtin_mod` is passed or if `!have_zcu`. + /// Otherwise, may be `null` only if this Compilation consists of a single module. + builtin_modules: ?*std.StringHashMapUnmanaged(*Module), + + pub const Paths = struct { + root: Cache.Path, + /// Relative to `root`. May contain path separators. + root_src_path: []const u8, + }; + + pub const Inherited = struct { + /// If this is null then `parent` must be non-null. + resolved_target: ?ResolvedTarget = null, + optimize_mode: ?std.builtin.OptimizeMode = null, + code_model: ?std.builtin.CodeModel = null, + single_threaded: ?bool = null, + error_tracing: ?bool = null, + valgrind: ?bool = null, + pic: ?bool = null, + strip: ?bool = null, + omit_frame_pointer: ?bool = null, + stack_check: ?bool = null, + /// null means default. + /// 0 means no stack protector. + /// other number means stack protection with that buffer size. + stack_protector: ?u32 = null, + red_zone: ?bool = null, + unwind_tables: ?std.builtin.UnwindTables = null, + sanitize_c: ?bool = null, + sanitize_thread: ?bool = null, + fuzz: ?bool = null, + structured_cfg: ?bool = null, + no_builtin: ?bool = null, + }; +}; + +pub const ResolvedTarget = struct { + result: std.Target, + is_native_os: bool, + is_native_abi: bool, + llvm_cpu_features: ?[*:0]const u8 = null, +}; + +/// At least one of `parent` and `resolved_target` must be non-null. +pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { + if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); + if (options.inherited.fuzz == true) assert(options.global.any_fuzz); + if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); + if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); + if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); + + const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; + const target = resolved_target.result; + + const optimize_mode = options.inherited.optimize_mode orelse + if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; + + const strip = b: { + if (options.inherited.strip) |x| break :b x; + if (options.parent) |p| break :b p.strip; + break :b options.global.root_strip; + }; + + const valgrind = b: { + if (!target_util.hasValgrindSupport(target)) { + if (options.inherited.valgrind == true) + return error.ValgrindUnsupportedOnTarget; + break :b false; + } + if (options.inherited.valgrind) |x| break :b x; + if (options.parent) |p| break :b p.valgrind; + if (strip) break :b false; + break :b optimize_mode == .Debug; + }; + + const zig_backend = target_util.zigBackend(target, options.global.use_llvm); + + const single_threaded = b: { + if (target_util.alwaysSingleThreaded(target)) { + if (options.inherited.single_threaded == false) + return error.TargetRequiresSingleThreaded; + break :b true; + } + + if (options.global.have_zcu) { + if (!target_util.supportsThreads(target, zig_backend)) { + if (options.inherited.single_threaded == false) + return error.BackendRequiresSingleThreaded; + break :b true; + } + } + + if (options.inherited.single_threaded) |x| break :b x; + if (options.parent) |p| break :b p.single_threaded; + break :b target_util.defaultSingleThreaded(target); + }; + + const error_tracing = b: { + if (options.inherited.error_tracing) |x| break :b x; + if (options.parent) |p| break :b p.error_tracing; + break :b options.global.root_error_tracing; + }; + + const pic = b: { + if (target_util.requiresPIC(target, options.global.link_libc)) { + if (options.inherited.pic == false) + return error.TargetRequiresPic; + break :b true; + } + if (options.global.pie) { + if (options.inherited.pic == false) + return error.PieRequiresPic; + break :b true; + } + if (options.global.link_mode == .dynamic) { + if (options.inherited.pic == false) + return error.DynamicLinkingRequiresPic; + break :b true; + } + if (options.inherited.pic) |x| break :b x; + if (options.parent) |p| break :b p.pic; + break :b false; + }; + + const red_zone = b: { + if (!target_util.hasRedZone(target)) { + if (options.inherited.red_zone == true) + return error.TargetHasNoRedZone; + break :b false; + } + if (options.inherited.red_zone) |x| break :b x; + if (options.parent) |p| break :b p.red_zone; + break :b true; + }; + + const omit_frame_pointer = b: { + if (options.inherited.omit_frame_pointer) |x| break :b x; + if (options.parent) |p| break :b p.omit_frame_pointer; + if (optimize_mode == .ReleaseSmall) { + // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. + // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) + // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). + break :b !target.cpu.arch.isX86(); + } + break :b false; + }; + + const sanitize_thread = b: { + if (options.inherited.sanitize_thread) |x| break :b x; + if (options.parent) |p| break :b p.sanitize_thread; + break :b false; + }; + + const unwind_tables = b: { + if (options.inherited.unwind_tables) |x| break :b x; + if (options.parent) |p| break :b p.unwind_tables; + + break :b target_util.defaultUnwindTables( + target, + options.global.link_libunwind, + sanitize_thread or options.global.any_sanitize_thread, + ); + }; + + const fuzz = b: { + if (options.inherited.fuzz) |x| break :b x; + if (options.parent) |p| break :b p.fuzz; + break :b false; + }; + + const code_model = b: { + if (options.inherited.code_model) |x| break :b x; + if (options.parent) |p| break :b p.code_model; + break :b .default; + }; + + const is_safe_mode = switch (optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, + }; + + const sanitize_c = b: { + if (options.inherited.sanitize_c) |x| break :b x; + if (options.parent) |p| break :b p.sanitize_c; + break :b is_safe_mode; + }; + + const stack_check = b: { + if (!target_util.supportsStackProbing(target)) { + if (options.inherited.stack_check == true) + return error.StackCheckUnsupportedByTarget; + break :b false; + } + if (options.inherited.stack_check) |x| break :b x; + if (options.parent) |p| break :b p.stack_check; + break :b is_safe_mode; + }; + + const stack_protector: u32 = sp: { + const use_zig_backend = options.global.have_zcu or + (options.global.any_c_source_files and options.global.c_frontend == .aro); + if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnsupportedByTarget; + } + break :sp 0; + } + + if (options.global.any_c_source_files and options.global.c_frontend == .clang and + !target_util.clangSupportsStackProtector(target)) + { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnsupportedByTarget; + } + break :sp 0; + } + + // This logic is checking for linking libc because otherwise our start code + // which is trying to set up TLS (i.e. the fs/gs registers) but the stack + // protection code depends on fs/gs registers being already set up. + // If we were able to annotate start code, or perhaps the entire std lib, + // as being exempt from stack protection checks, we could change this logic + // to supporting stack protection even when not linking libc. + // TODO file issue about this + if (!options.global.link_libc) { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnavailableWithoutLibC; + } + break :sp 0; + } + + if (options.inherited.stack_protector) |x| break :sp x; + if (options.parent) |p| break :sp p.stack_protector; + if (!is_safe_mode) break :sp 0; + + break :sp target_util.default_stack_protector_buffer_size; + }; + + const structured_cfg = b: { + if (options.inherited.structured_cfg) |x| break :b x; + if (options.parent) |p| break :b p.structured_cfg; + // We always want a structured control flow in shaders. This option is + // only relevant for OpenCL kernels. + break :b switch (target.os.tag) { + .opencl => false, + else => true, + }; + }; + + const no_builtin = b: { + if (options.inherited.no_builtin) |x| break :b x; + if (options.parent) |p| break :b p.no_builtin; + + break :b target.cpu.arch.isBpf(); + }; + + const llvm_cpu_features: ?[*:0]const u8 = b: { + if (resolved_target.llvm_cpu_features) |x| break :b x; + if (!options.global.use_llvm) break :b null; + + var buf = std.ArrayList(u8).init(arena); + var disabled_features = std.ArrayList(u8).init(arena); + defer disabled_features.deinit(); + + // Append disabled features after enabled ones, so that their effects aren't overwritten. + for (target.cpu.arch.allFeaturesList()) |feature| { + if (feature.llvm_name) |llvm_name| { + const is_enabled = target.cpu.features.isEnabled(feature.index); + + if (is_enabled) { + try buf.ensureUnusedCapacity(2 + llvm_name.len); + buf.appendAssumeCapacity('+'); + buf.appendSliceAssumeCapacity(llvm_name); + buf.appendAssumeCapacity(','); + } else { + try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); + disabled_features.appendAssumeCapacity('-'); + disabled_features.appendSliceAssumeCapacity(llvm_name); + disabled_features.appendAssumeCapacity(','); + } + } + } + + try buf.appendSlice(disabled_features.items); + if (buf.items.len == 0) break :b ""; + assert(std.mem.endsWith(u8, buf.items, ",")); + buf.items[buf.items.len - 1] = 0; + buf.shrinkAndFree(buf.items.len); + break :b buf.items[0 .. buf.items.len - 1 :0].ptr; + }; + + const mod = try arena.create(Module); + mod.* = .{ + .root = options.paths.root, + .root_src_path = options.paths.root_src_path, + .fully_qualified_name = options.fully_qualified_name, + .resolved_target = .{ + .result = target, + .is_native_os = resolved_target.is_native_os, + .is_native_abi = resolved_target.is_native_abi, + .llvm_cpu_features = llvm_cpu_features, + }, + .optimize_mode = optimize_mode, + .single_threaded = single_threaded, + .error_tracing = error_tracing, + .valgrind = valgrind, + .pic = pic, + .strip = strip, + .omit_frame_pointer = omit_frame_pointer, + .stack_check = stack_check, + .stack_protector = stack_protector, + .code_model = code_model, + .red_zone = red_zone, + .sanitize_c = sanitize_c, + .sanitize_thread = sanitize_thread, + .fuzz = fuzz, + .unwind_tables = unwind_tables, + .cc_argv = options.cc_argv, + .structured_cfg = structured_cfg, + .no_builtin = no_builtin, + .builtin_file = null, + }; + + const opt_builtin_mod = options.builtin_mod orelse b: { + if (!options.global.have_zcu) break :b null; + + const generated_builtin_source = try Builtin.generate(.{ + .target = target, + .zig_backend = zig_backend, + .output_mode = options.global.output_mode, + .link_mode = options.global.link_mode, + .unwind_tables = unwind_tables, + .is_test = options.global.is_test, + .single_threaded = single_threaded, + .link_libc = options.global.link_libc, + .link_libcpp = options.global.link_libcpp, + .optimize_mode = optimize_mode, + .error_tracing = error_tracing, + .valgrind = valgrind, + .sanitize_thread = sanitize_thread, + .fuzz = fuzz, + .pic = pic, + .pie = options.global.pie, + .strip = strip, + .code_model = code_model, + .omit_frame_pointer = omit_frame_pointer, + .wasi_exec_model = options.global.wasi_exec_model, + }, arena); + + const new = if (options.builtin_modules) |builtins| new: { + const gop = try builtins.getOrPut(arena, generated_builtin_source); + if (gop.found_existing) break :b gop.value_ptr.*; + errdefer builtins.removeByPtr(gop.key_ptr); + const new = try arena.create(Module); + gop.value_ptr.* = new; + break :new new; + } else try arena.create(Module); + errdefer if (options.builtin_modules) |builtins| assert(builtins.remove(generated_builtin_source)); + + const new_file = try arena.create(File); + + const hex_digest = digest: { + var hasher: Cache.Hasher = Cache.hasher_init; + hasher.update(generated_builtin_source); + + var bin_digest: Cache.BinDigest = undefined; + hasher.final(&bin_digest); + + var hex_digest: Cache.HexDigest = undefined; + _ = std.fmt.bufPrint( + &hex_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; + + break :digest hex_digest; + }; + + const builtin_sub_path = try arena.dupe(u8, "b" ++ std.fs.path.sep_str ++ hex_digest); + + new.* = .{ + .root = .{ + .root_dir = options.global_cache_directory, + .sub_path = builtin_sub_path, + }, + .root_src_path = "builtin.zig", + .fully_qualified_name = if (options.parent == null) + "builtin" + else + try std.fmt.allocPrint(arena, "{s}.builtin", .{options.fully_qualified_name}), + .resolved_target = .{ + .result = target, + .is_native_os = resolved_target.is_native_os, + .is_native_abi = resolved_target.is_native_abi, + .llvm_cpu_features = llvm_cpu_features, + }, + .optimize_mode = optimize_mode, + .single_threaded = single_threaded, + .error_tracing = error_tracing, + .valgrind = valgrind, + .pic = pic, + .strip = strip, + .omit_frame_pointer = omit_frame_pointer, + .stack_check = stack_check, + .stack_protector = stack_protector, + .code_model = code_model, + .red_zone = red_zone, + .sanitize_c = sanitize_c, + .sanitize_thread = sanitize_thread, + .fuzz = fuzz, + .unwind_tables = unwind_tables, + .cc_argv = &.{}, + .structured_cfg = structured_cfg, + .no_builtin = no_builtin, + .builtin_file = new_file, + }; + new_file.* = .{ + .sub_file_path = "builtin.zig", + .stat = undefined, + .source = generated_builtin_source, + .tree = null, + .zir = null, + .zoir = null, + .status = .never_loaded, + .mod = new, + }; + break :b new; + }; + + if (opt_builtin_mod) |builtin_mod| { + try mod.deps.ensureUnusedCapacity(arena, 1); + mod.deps.putAssumeCapacityNoClobber("builtin", builtin_mod); + } + + return mod; +} + +/// All fields correspond to `CreateOptions`. +pub const LimitedOptions = struct { + root: Cache.Path, + root_src_path: []const u8, + fully_qualified_name: []const u8, +}; + +/// This one can only be used if the Module will only be used for AstGen and earlier in +/// the pipeline. Illegal behavior occurs if a limited module touches Sema. +pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { + const mod = try gpa.create(Module); + mod.* = .{ + .root = options.root, + .root_src_path = options.root_src_path, + .fully_qualified_name = options.fully_qualified_name, + + .resolved_target = undefined, + .optimize_mode = undefined, + .code_model = undefined, + .single_threaded = undefined, + .error_tracing = undefined, + .valgrind = undefined, + .pic = undefined, + .strip = undefined, + .omit_frame_pointer = undefined, + .stack_check = undefined, + .stack_protector = undefined, + .red_zone = undefined, + .sanitize_c = undefined, + .sanitize_thread = undefined, + .fuzz = undefined, + .unwind_tables = undefined, + .cc_argv = undefined, + .structured_cfg = undefined, + .no_builtin = undefined, + .builtin_file = null, + }; + return mod; +} + +/// Asserts that the module has a builtin module, which is not true for non-zig +/// modules such as ones only used for `@embedFile`, or the root module when +/// there is no Zig Compilation Unit. +pub fn getBuiltinDependency(m: Module) *Module { + const result = m.deps.values()[0]; + assert(result.isBuiltin()); + return result; +} + +const Module = @This(); +const Package = @import("../Package.zig"); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; +const target_util = @import("../target.zig"); +const Cache = std.Build.Cache; +const Builtin = @import("../Builtin.zig"); +const assert = std.debug.assert; +const Compilation = @import("../Compilation.zig"); +const File = @import("../Zcu.zig").File; diff --git a/src/Package.zig b/src/Package.zig deleted file mode 100644 index 7f231f5ad7e3ee84705108e1dd8970766249b936..0000000000000000000000000000000000000000 --- a/src/Package.zig +++ /dev/null @@ -1,200 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; - -pub const Module = @import("Package/Module.zig"); -pub const Fetch = @import("Package/Fetch.zig"); -pub const build_zig_basename = "build.zig"; -pub const Manifest = @import("Package/Manifest.zig"); - -pub const multihash_len = 1 + 1 + Hash.Algo.digest_length; -pub const multihash_hex_digest_len = 2 * multihash_len; -pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; - -pub const Fingerprint = packed struct(u64) { - id: u32, - checksum: u32, - - pub fn generate(name: []const u8) Fingerprint { - return .{ - .id = std.crypto.random.intRangeLessThan(u32, 1, 0xffffffff), - .checksum = std.hash.Crc32.hash(name), - }; - } - - pub fn validate(n: Fingerprint, name: []const u8) bool { - switch (n.id) { - 0x00000000, 0xffffffff => return false, - else => return std.hash.Crc32.hash(name) == n.checksum, - } - } - - pub fn int(n: Fingerprint) u64 { - return @bitCast(n); - } -}; - -/// A user-readable, file system safe hash that identifies an exact package -/// snapshot, including file contents. -/// -/// The hash is not only to prevent collisions but must resist attacks where -/// the adversary fully controls the contents being hashed. Thus, it contains -/// a full SHA-256 digest. -/// -/// This data structure can be used to store the legacy hash format too. Legacy -/// hash format is scheduled to be removed after 0.14.0 is tagged. -/// -/// There's also a third way this structure is used. When using path rather than -/// hash, a unique hash is still needed, so one is computed based on the path. -pub const Hash = struct { - /// Maximum size of a package hash. Unused bytes at the end are - /// filled with zeroes. - bytes: [max_len]u8, - - pub const Algo = std.crypto.hash.sha2.Sha256; - pub const Digest = [Algo.digest_length]u8; - - /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" - pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; - - pub fn fromSlice(s: []const u8) Hash { - assert(s.len <= max_len); - var result: Hash = undefined; - @memcpy(result.bytes[0..s.len], s); - @memset(result.bytes[s.len..], 0); - return result; - } - - pub fn toSlice(ph: *const Hash) []const u8 { - var end: usize = ph.bytes.len; - while (true) { - end -= 1; - if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; - } - } - - pub fn eql(a: *const Hash, b: *const Hash) bool { - return std.mem.eql(u8, &a.bytes, &b.bytes); - } - - /// Distinguishes whether the legacy multihash format is being stored here. - pub fn isOld(h: *const Hash) bool { - if (h.bytes.len < 2) return false; - const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false; - if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false; - if (h.toSlice().len != multihash_hex_digest_len) return false; - return std.mem.indexOfScalar(u8, &h.bytes, '-') == null; - } - - test isOld { - const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7"); - try std.testing.expect(h.isOld()); - } - - /// Produces "$name-$semver-$hashplus". - /// * name is the name field from build.zig.zon, asserted to be at most 32 - /// bytes and assumed be a valid zig identifier - /// * semver is the version field from build.zig.zon, asserted to be at - /// most 32 bytes - /// * hashplus is the following 33-byte array, base64 encoded using -_ to make - /// it filesystem safe: - /// - (4 bytes) LE u32 Package ID - /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated - /// - (25 bytes) truncated SHA-256 digest of hashed files of the package - pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { - assert(name.len <= 32); - assert(ver.len <= 32); - var result: Hash = undefined; - var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes); - buf.appendSliceAssumeCapacity(name); - buf.appendAssumeCapacity('-'); - buf.appendSliceAssumeCapacity(ver); - buf.appendAssumeCapacity('-'); - var hashplus: [33]u8 = undefined; - std.mem.writeInt(u32, hashplus[0..4], id, .little); - std.mem.writeInt(u32, hashplus[4..8], size, .little); - hashplus[8..].* = digest[0..25].*; - _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); - @memset(buf.unusedCapacitySlice(), 0); - return result; - } - - /// Produces a unique hash based on the path provided. The result should - /// not be user-visible. - pub fn initPath(sub_path: []const u8, is_global: bool) Hash { - var result: Hash = .{ .bytes = @splat(0) }; - var i: usize = 0; - if (is_global) { - result.bytes[0] = '/'; - i += 1; - } - if (i + sub_path.len <= result.bytes.len) { - @memcpy(result.bytes[i..][0..sub_path.len], sub_path); - return result; - } - var bin_digest: [Algo.digest_length]u8 = undefined; - Algo.hash(sub_path, &bin_digest, .{}); - _ = std.fmt.bufPrint(result.bytes[i..], "{}", .{std.fmt.fmtSliceHexLower(&bin_digest)}) catch unreachable; - return result; - } -}; - -pub const MultihashFunction = enum(u16) { - identity = 0x00, - sha1 = 0x11, - @"sha2-256" = 0x12, - @"sha2-512" = 0x13, - @"sha3-512" = 0x14, - @"sha3-384" = 0x15, - @"sha3-256" = 0x16, - @"sha3-224" = 0x17, - @"sha2-384" = 0x20, - @"sha2-256-trunc254-padded" = 0x1012, - @"sha2-224" = 0x1013, - @"sha2-512-224" = 0x1014, - @"sha2-512-256" = 0x1015, - @"blake2b-256" = 0xb220, - _, -}; - -pub const multihash_function: MultihashFunction = switch (Hash.Algo) { - std.crypto.hash.sha2.Sha256 => .@"sha2-256", - else => unreachable, -}; - -pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest { - const hex_charset = std.fmt.hex_charset; - - var result: MultiHashHexDigest = undefined; - - result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; - result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; - - result[2] = hex_charset[Hash.Algo.digest_length >> 4]; - result[3] = hex_charset[Hash.Algo.digest_length & 15]; - - for (digest, 0..) |byte, i| { - result[4 + i * 2] = hex_charset[byte >> 4]; - result[5 + i * 2] = hex_charset[byte & 15]; - } - return result; -} - -comptime { - // We avoid unnecessary uleb128 code in hexDigest by asserting here the - // values are small enough to be contained in the one-byte encoding. - assert(@intFromEnum(multihash_function) < 127); - assert(Hash.Algo.digest_length < 127); -} - -test Hash { - const example_digest: Hash.Digest = .{ - 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, - 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, - }; - const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); - try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); -} - -test { - _ = Fetch; -} diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig deleted file mode 100644 index bb705189105cd68ea3e4885a4a02558fbaf30994..0000000000000000000000000000000000000000 --- a/src/Package/Fetch.zig +++ /dev/null @@ -1,2421 +0,0 @@ -//! Represents one independent job whose responsibility is to: -//! -//! 1. Check the global zig package cache to see if the hash already exists. -//! If so, load, parse, and validate the build.zig.zon file therein, and -//! goto step 8. Likewise if the location is a relative path, treat this -//! the same as a cache hit. Otherwise, proceed. -//! 2. Fetch and unpack a URL into a temporary directory. -//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed -//! for the file to be missing, in which case this fetched package is considered -//! to be a "naked" package. -//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by -//! deleting excluded files. If any files had errors for files that were -//! ultimately excluded, those errors should be ignored, such as failure to -//! create symlinks that weren't supposed to be included anyway. -//! 5. Compute the package hash based on the remaining files in the temporary -//! directory. -//! 6. Rename the temporary directory into the global zig package cache -//! directory. If the hash already exists, delete the temporary directory and -//! leave the zig package cache directory untouched as it may be in use by the -//! system. This is done even if the hash is invalid, in case the package with -//! the different hash is used in the future. -//! 7. Validate the computed hash against the expected hash. If invalid, -//! this job is done. -//! 8. Spawn a new fetch job for each dependency in the manifest file. Use -//! a mutex and a hash map so that redundant jobs do not get queued up. -//! -//! All of this must be done with only referring to the state inside this struct -//! because this work will be done in a dedicated thread. - -arena: std.heap.ArenaAllocator, -location: Location, -location_tok: std.zig.Ast.TokenIndex, -hash_tok: std.zig.Ast.OptionalTokenIndex, -name_tok: std.zig.Ast.TokenIndex, -lazy_status: LazyStatus, -parent_package_root: Cache.Path, -parent_manifest_ast: ?*const std.zig.Ast, -prog_node: std.Progress.Node, -job_queue: *JobQueue, -/// If true, don't add an error for a missing hash. This flag is not passed -/// down to recursive dependencies. It's intended to be used only be the CLI. -omit_missing_hash_error: bool, -/// If true, don't fail when a manifest file is missing the `paths` field, -/// which specifies inclusion rules. This is intended to be true for the first -/// fetch task and false for the recursive dependencies. -allow_missing_paths_field: bool, -allow_missing_fingerprint: bool, -allow_name_string: bool, -/// If true and URL points to a Git repository, will use the latest commit. -use_latest_commit: bool, - -// Above this are fields provided as inputs to `run`. -// Below this are fields populated by `run`. - -/// This will either be relative to `global_cache`, or to the build root of -/// the root package. -package_root: Cache.Path, -error_bundle: ErrorBundle.Wip, -manifest: ?Manifest, -manifest_ast: std.zig.Ast, -computed_hash: ComputedHash, -/// Fetch logic notices whether a package has a build.zig file and sets this flag. -has_build_zig: bool, -/// Indicates whether the task aborted due to an out-of-memory condition. -oom_flag: bool, -/// If `use_latest_commit` was true, this will be set to the commit that was used. -/// If the resource pointed to by the location is not a Git-repository, this -/// will be left unchanged. -latest_commit: ?git.Oid, - -// This field is used by the CLI only, untouched by this file. - -/// The module for this `Fetch` tasks's package, which exposes `build.zig` as -/// the root source file. -module: ?*Package.Module, - -pub const LazyStatus = enum { - /// Not lazy. - eager, - /// Lazy, found. - available, - /// Lazy, not found. - unavailable, -}; - -/// Contains shared state among all `Fetch` tasks. -pub const JobQueue = struct { - mutex: std.Thread.Mutex = .{}, - /// It's an array hash map so that it can be sorted before rendering the - /// dependencies.zig source file. - /// Protected by `mutex`. - table: Table = .{}, - /// `table` may be missing some tasks such as ones that failed, so this - /// field contains references to all of them. - /// Protected by `mutex`. - all_fetches: std.ArrayListUnmanaged(*Fetch) = .empty, - - http_client: *std.http.Client, - thread_pool: *ThreadPool, - wait_group: WaitGroup = .{}, - global_cache: Cache.Directory, - /// If true then, no fetching occurs, and: - /// * The `global_cache` directory is assumed to be the direct parent - /// directory of on-disk packages rather than having the "p/" directory - /// prefix inside of it. - /// * An error occurs if any non-lazy packages are not already present in - /// the package cache directory. - /// * Missing hash field causes an error, and no fetching occurs so it does - /// not print the correct hash like usual. - read_only: bool, - recursive: bool, - /// Dumps hash information to stdout which can be used to troubleshoot why - /// two hashes of the same package do not match. - /// If this is true, `recursive` must be false. - debug_hash: bool, - work_around_btrfs_bug: bool, - /// Set of hashes that will be additionally fetched even if they are marked - /// as lazy. - unlazy_set: UnlazySet = .{}, - - pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch); - pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void); - - pub fn deinit(jq: *JobQueue) void { - if (jq.all_fetches.items.len == 0) return; - const gpa = jq.all_fetches.items[0].arena.child_allocator; - jq.table.deinit(gpa); - // These must be deinitialized in reverse order because subsequent - // `Fetch` instances are allocated in prior ones' arenas. - // Sorry, I know it's a bit weird, but it slightly simplifies the - // critical section. - while (jq.all_fetches.pop()) |f| f.deinit(); - jq.all_fetches.deinit(gpa); - jq.* = undefined; - } - - /// Dumps all subsequent error bundles into the first one. - pub fn consolidateErrors(jq: *JobQueue) !void { - const root = &jq.all_fetches.items[0].error_bundle; - const gpa = root.gpa; - for (jq.all_fetches.items[1..]) |fetch| { - if (fetch.error_bundle.root_list.items.len > 0) { - var bundle = try fetch.error_bundle.toOwnedBundle(""); - defer bundle.deinit(gpa); - try root.addBundleAsRoots(bundle); - } - } - } - - /// Creates the dependencies.zig source code for the build runner to obtain - /// via `@import("@dependencies")`. - pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { - const keys = jq.table.keys(); - - assert(keys.len != 0); // caller should have added the first one - if (keys.len == 1) { - // This is the first one. It must have no dependencies. - return createEmptyDependenciesSource(buf); - } - - try buf.appendSlice("pub const packages = struct {\n"); - - // Ensure the generated .zig file is deterministic. - jq.table.sortUnstable(@as(struct { - keys: []const Package.Hash, - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); - } - }, .{ .keys = keys })); - - for (keys, jq.table.values()) |*hash, fetch| { - if (fetch == jq.all_fetches.items[0]) { - // The first one is a dummy package for the current project. - continue; - } - - const hash_slice = hash.toSlice(); - - try buf.writer().print( - \\ pub const {} = struct {{ - \\ - , .{std.zig.fmtId(hash_slice)}); - - lazy: { - switch (fetch.lazy_status) { - .eager => break :lazy, - .available => { - try buf.appendSlice( - \\ pub const available = true; - \\ - ); - break :lazy; - }, - .unavailable => { - try buf.appendSlice( - \\ pub const available = false; - \\ }; - \\ - ); - continue; - }, - } - } - - try buf.writer().print( - \\ pub const build_root = "{q}"; - \\ - , .{fetch.package_root}); - - if (fetch.has_build_zig) { - try buf.writer().print( - \\ pub const build_zig = @import("{}"); - \\ - , .{std.zig.fmtEscapes(hash_slice)}); - } - - if (fetch.manifest) |*manifest| { - try buf.appendSlice( - \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.writer().print( - " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, - ); - } - - try buf.appendSlice( - \\ }; - \\ }; - \\ - ); - } else { - try buf.appendSlice( - \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; - \\ }; - \\ - ); - } - } - - try buf.appendSlice( - \\}; - \\ - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - - const root_fetch = jq.all_fetches.items[0]; - const root_manifest = &root_fetch.manifest.?; - - for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { - const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.writer().print( - " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, - ); - } - try buf.appendSlice("};\n"); - } - - pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void { - try buf.appendSlice( - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - } -}; - -pub const Location = union(enum) { - remote: Remote, - /// A directory found inside the parent package. - relative_path: Cache.Path, - /// Recursive Fetch tasks will never use this Location, but it may be - /// passed in by the CLI. Indicates the file contents here should be copied - /// into the global package cache. It may be a file relative to the cwd or - /// absolute, in which case it should be treated exactly like a `file://` - /// URL, or a directory, in which case it should be treated as an - /// already-unpacked directory (but still needs to be copied into the - /// global package cache and have inclusion rules applied). - path_or_url: []const u8, - - pub const Remote = struct { - url: []const u8, - /// If this is null it means the user omitted the hash field from a dependency. - /// It will be an error but the logic should still fetch and print the discovered hash. - hash: ?Package.Hash, - }; -}; - -pub const RunError = error{ - OutOfMemory, - /// This error code is intended to be handled by inspecting the - /// `error_bundle` field. - FetchFailed, -}; - -pub fn run(f: *Fetch) RunError!void { - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const cache_root = f.job_queue.global_cache; - - try eb.init(gpa); - - // Check the global zig package cache to see if the hash already exists. If - // so, load, parse, and validate the build.zig.zon file therein, and skip - // ahead to queuing up jobs for dependencies. Likewise if the location is a - // relative path, treat this the same as a cache hit. Otherwise, proceed. - - const remote = switch (f.location) { - .relative_path => |pkg_root| { - if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( - f.location_tok, - try eb.addString("expected path relative to build root; found absolute path"), - ); - if (f.hash_tok.unwrap()) |hash_tok| return f.fail( - hash_tok, - try eb.addString("path-based dependencies are not hashed"), - ); - // Packages fetched by URL may not use relative paths to escape outside the - // fetched package directory from within the package cache. - if (pkg_root.root_dir.eql(cache_root)) { - // `parent_package_root.sub_path` contains a path like this: - // "p/$hash", or - // "p/$hash/foo", with possibly more directories after "foo". - // We want to fail unless the resolved relative path has a - // prefix of "p/$hash/". - const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; - const parent_sub_path = f.parent_package_root.sub_path; - const end = find_end: { - if (parent_sub_path.len > prefix_len) { - // Use `isSep` instead of `indexOfScalarPos` to account for - // Windows accepting both `\` and `/` as path separators. - for (parent_sub_path[prefix_len..], prefix_len..) |c, i| { - if (std.fs.path.isSep(c)) break :find_end i; - } - } - break :find_end parent_sub_path.len; - }; - const expected_prefix = parent_sub_path[0..end]; - if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) { - return f.fail( - f.location_tok, - try eb.printString("dependency path outside project: '{}'", .{pkg_root}), - ); - } - } - f.package_root = pkg_root; - try loadManifest(f, pkg_root); - if (!f.has_build_zig) try checkBuildFileExistence(f); - if (!f.job_queue.recursive) return; - return queueJobsForDeps(f); - }, - .remote => |remote| remote, - .path_or_url => |path_or_url| { - if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| { - var resource: Resource = .{ .dir = dir }; - return f.runResource(path_or_url, &resource, null); - } else |dir_err| { - const file_err = if (dir_err == error.NotDir) e: { - if (fs.cwd().openFile(path_or_url, .{})) |file| { - var resource: Resource = .{ .file = file }; - return f.runResource(path_or_url, &resource, null); - } else |err| break :e err; - } else dir_err; - - const uri = std.Uri.parse(path_or_url) catch |uri_err| { - return f.fail(0, try eb.printString( - "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", - .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, - )); - }; - var server_header_buffer: [header_buffer_size]u8 = undefined; - var resource = try f.initResource(uri, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); - } - }, - }; - - if (remote.hash) |expected_hash| { - var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined; - prefixed_pkg_sub_path_buffer[0] = 'p'; - prefixed_pkg_sub_path_buffer[1] = fs.path.sep; - const hash_slice = expected_hash.toSlice(); - @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); - const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; - const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; - const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; - if (cache_root.handle.access(pkg_sub_path, .{})) |_| { - assert(f.lazy_status != .unavailable); - f.package_root = .{ - .root_dir = cache_root, - .sub_path = try arena.dupe(u8, pkg_sub_path), - }; - try loadManifest(f, f.package_root); - try checkBuildFileExistence(f); - if (!f.job_queue.recursive) return; - return queueJobsForDeps(f); - } else |err| switch (err) { - error.FileNotFound => { - switch (f.lazy_status) { - .eager => {}, - .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { - f.lazy_status = .unavailable; - return; - }, - .unavailable => unreachable, - } - if (f.job_queue.read_only) return f.fail( - f.name_tok, - try eb.printString("package not found at '{}{s}'", .{ - cache_root, pkg_sub_path, - }), - ); - }, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ - cache_root, pkg_sub_path, @errorName(e), - }), - }); - return error.FetchFailed; - }, - } - } else if (f.job_queue.read_only) { - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("dependency is missing hash field"), - .src_loc = try f.srcLoc(f.location_tok), - }); - return error.FetchFailed; - } - - // Fetch and unpack the remote into a temporary directory. - - const uri = std.Uri.parse(remote.url) catch |err| return f.fail( - f.location_tok, - try eb.printString("invalid URI: {s}", .{@errorName(err)}), - ); - var server_header_buffer: [header_buffer_size]u8 = undefined; - var resource = try f.initResource(uri, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); -} - -pub fn deinit(f: *Fetch) void { - f.error_bundle.deinit(); - f.arena.deinit(); -} - -/// Consumes `resource`, even if an error is returned. -fn runResource( - f: *Fetch, - uri_path: []const u8, - resource: *Resource, - remote_hash: ?Package.Hash, -) RunError!void { - defer resource.deinit(); - const arena = f.arena.allocator(); - const eb = &f.error_bundle; - const s = fs.path.sep_str; - const cache_root = f.job_queue.global_cache; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); - - const package_sub_path = blk: { - const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); - var tmp_directory: Cache.Directory = .{ - .path = tmp_directory_path, - .handle = handle: { - const dir = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ - .iterate = true, - }) catch |err| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ - tmp_directory_path, @errorName(err), - }), - }); - return error.FetchFailed; - }; - break :handle dir; - }, - }; - defer tmp_directory.handle.close(); - - // Fetch and unpack a resource into a temporary directory. - var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); - - var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; - - // Apply btrfs workaround if needed. Reopen tmp_directory. - if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { - // https://github.com/ziglang/zig/issues/17095 - pkg_path.root_dir.handle.close(); - pkg_path.root_dir.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ - .iterate = true, - }) catch @panic("btrfs workaround failed"); - } - - // Load, parse, and validate the unpacked build.zig.zon file. It is allowed - // for the file to be missing, in which case this fetched package is - // considered to be a "naked" package. - try loadManifest(f, pkg_path); - - const filter: Filter = .{ - .include_paths = if (f.manifest) |m| m.paths else .{}, - }; - - // Ignore errors that were excluded by manifest, such as failure to - // create symlinks that weren't supposed to be included anyway. - try unpack_result.validate(f, filter); - - // Apply the manifest's inclusion rules to the temporary directory by - // deleting excluded files. - // Empty directories have already been omitted by `unpackResource`. - // Compute the package hash based on the remaining files in the temporary - // directory. - f.computed_hash = try computeHash(f, pkg_path, filter); - - break :blk if (unpack_result.root_dir.len > 0) - try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) - else - tmp_dir_sub_path; - }; - - const computed_package_hash = computedPackageHash(f); - - // Rename the temporary directory into the global zig package cache - // directory. If the hash already exists, delete the temporary directory - // and leave the zig package cache directory untouched as it may be in use - // by the system. This is done even if the hash is invalid, in case the - // package with the different hash is used in the future. - - f.package_root = .{ - .root_dir = cache_root, - .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), - }; - renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { - const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); - const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", - .{ src, dest, @errorName(err) }, - ) }); - return error.FetchFailed; - }; - // Remove temporary directory root if not already renamed to global cache. - if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { - cache_root.handle.deleteDir(tmp_dir_sub_path) catch {}; - } - - // Validate the computed hash against the expected hash. If invalid, this - // job is done. - - if (remote_hash) |declared_hash| { - const hash_tok = f.hash_tok.unwrap().?; - if (declared_hash.isOld()) { - const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest); - if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) { - return f.fail(hash_tok, try eb.printString( - "hash mismatch: manifest declares {s} but the fetched package has {s}", - .{ declared_hash.toSlice(), actual_hex }, - )); - } - } else { - if (!computed_package_hash.eql(&declared_hash)) { - return f.fail(hash_tok, try eb.printString( - "hash mismatch: manifest declares {s} but the fetched package has {s}", - .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, - )); - } - } - } else if (!f.omit_missing_hash_error) { - const notes_len = 1; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("dependency is missing hash field"), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = notes_len, - }); - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), - })); - return error.FetchFailed; - } - - // Spawn a new fetch job for each dependency in the manifest file. Use - // a mutex and a hash map so that redundant jobs do not get queued up. - if (!f.job_queue.recursive) return; - return queueJobsForDeps(f); -} - -pub fn computedPackageHash(f: *const Fetch) Package.Hash { - const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); - if (f.manifest) |man| { - var version_buffer: [32]u8 = undefined; - const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer; - return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); - } - // In the future build.zig.zon fields will be added to allow overriding these values - // for naked tarballs. - return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); -} - -/// `computeHash` gets a free check for the existence of `build.zig`, but when -/// not computing a hash, we need to do a syscall to check for it. -fn checkBuildFileExistence(f: *Fetch) RunError!void { - const eb = &f.error_bundle; - if (f.package_root.access(Package.build_zig_basename, .{})) |_| { - f.has_build_zig = true; - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to access '{}{s}': {s}", .{ - f.package_root, Package.build_zig_basename, @errorName(e), - }), - }); - return error.FetchFailed; - }, - } -} - -/// This function populates `f.manifest` or leaves it `null`. -fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( - arena, - try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), - Manifest.max_bytes, - null, - 1, - 0, - ) catch |err| switch (err) { - error.FileNotFound => return, - else => |e| { - const file_path = try pkg_root.join(arena, Manifest.basename); - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ - file_path, @errorName(e), - }), - }); - return error.FetchFailed; - }, - }; - - const ast = &f.manifest_ast; - ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); - - if (ast.errors.len > 0) { - const file_path = try std.fmt.allocPrint(arena, "{}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root}); - try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); - return error.FetchFailed; - } - - f.manifest = try Manifest.parse(arena, ast.*, .{ - .allow_missing_paths_field = f.allow_missing_paths_field, - .allow_missing_fingerprint = f.allow_missing_fingerprint, - .allow_name_string = f.allow_name_string, - }); - const manifest = &f.manifest.?; - - if (manifest.errors.len > 0) { - const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename }); - try manifest.copyErrorsIntoBundle(ast.*, src_path, eb); - return error.FetchFailed; - } -} - -fn queueJobsForDeps(f: *Fetch) RunError!void { - assert(f.job_queue.recursive); - - // If the package does not have a build.zig.zon file then there are no dependencies. - const manifest = f.manifest orelse return; - - const new_fetches, const prog_names = nf: { - const parent_arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const cache_root = f.job_queue.global_cache; - const dep_names = manifest.dependencies.keys(); - const deps = manifest.dependencies.values(); - // Grab the new tasks into a temporary buffer so we can unlock that mutex - // as fast as possible. - // This overallocates any fetches that get skipped by the `continue` in the - // loop below. - const new_fetches = try parent_arena.alloc(Fetch, deps.len); - const prog_names = try parent_arena.alloc([]const u8, deps.len); - var new_fetch_index: usize = 0; - - f.job_queue.mutex.lock(); - defer f.job_queue.mutex.unlock(); - - try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); - try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); - - // There are four cases here: - // * Correct hash is provided by manifest. - // - Hash map already has the entry, no need to add it again. - // * Incorrect hash is provided by manifest. - // - Hash mismatch error emitted; `queueJobsForDeps` is not called. - // * Hash is not provided by manifest. - // - Hash missing error emitted; `queueJobsForDeps` is not called. - // * path-based location is used without a hash. - // - Hash is added to the table based on the path alone before - // calling run(); no need to add it again. - // - // If we add a dep as lazy and then later try to add the same dep as eager, - // eagerness takes precedence and the existing entry is updated. - - for (dep_names, deps) |dep_name, dep| { - const new_fetch = &new_fetches[new_fetch_index]; - const location: Location = switch (dep.location) { - .url => |url| .{ .remote = .{ - .url = url, - .hash = h: { - const h = dep.hash orelse break :h null; - const pkg_hash: Package.Hash = .fromSlice(h); - const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); - if (gop.found_existing) { - if (!dep.lazy) { - gop.value_ptr.*.lazy_status = .eager; - } - continue; - } - gop.value_ptr.* = new_fetch; - break :h pkg_hash; - }, - } }, - .path => |rel_path| l: { - // This might produce an invalid path, which is checked for - // at the beginning of run(). - const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); - const pkg_hash = relativePathDigest(new_root, cache_root); - const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); - if (gop.found_existing) { - if (!dep.lazy) { - gop.value_ptr.*.lazy_status = .eager; - } - continue; - } - gop.value_ptr.* = new_fetch; - break :l .{ .relative_path = new_root }; - }, - }; - prog_names[new_fetch_index] = dep_name; - new_fetch_index += 1; - f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); - new_fetch.* = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = location, - .location_tok = dep.location_tok, - .hash_tok = dep.hash_tok, - .name_tok = dep.name_tok, - .lazy_status = if (dep.lazy) .available else .eager, - .parent_package_root = f.package_root, - .parent_manifest_ast = &f.manifest_ast, - .prog_node = f.prog_node, - .job_queue = f.job_queue, - .omit_missing_hash_error = false, - .allow_missing_paths_field = true, - .allow_missing_fingerprint = true, - .allow_name_string = true, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = null, - .manifest_ast = undefined, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - } - - f.prog_node.increaseEstimatedTotalItems(new_fetch_index); - - break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; - }; - - // Now it's time to give tasks to the thread pool. - const thread_pool = f.job_queue.thread_pool; - - for (new_fetches, prog_names) |*new_fetch, prog_name| { - thread_pool.spawnWg(&f.job_queue.wait_group, workerRun, .{ new_fetch, prog_name }); - } -} - -pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { - return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); -} - -pub fn workerRun(f: *Fetch, prog_name: []const u8) void { - const prog_node = f.prog_node.start(prog_name, 0); - defer prog_node.end(); - - run(f) catch |err| switch (err) { - error.OutOfMemory => f.oom_flag = true, - error.FetchFailed => { - // Nothing to do because the errors are already reported in `error_bundle`, - // and a reference is kept to the `Fetch` task inside `all_fetches`. - }, - }; -} - -fn srcLoc( - f: *Fetch, - tok: std.zig.Ast.TokenIndex, -) Allocator.Error!ErrorBundle.SourceLocationIndex { - const ast = f.parent_manifest_ast orelse return .none; - const eb = &f.error_bundle; - const start_loc = ast.tokenLocation(0, tok); - const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); - const msg_off = 0; - return eb.addSourceLocation(.{ - .src_path = src_path, - .span_start = ast.tokenStart(tok), - .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), - .span_main = ast.tokenStart(tok) + msg_off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }); -} - -fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { - const eb = &f.error_bundle; - try eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = try f.srcLoc(msg_tok), - }); - return error.FetchFailed; -} - -const Resource = union(enum) { - file: fs.File, - http_request: std.http.Client.Request, - git: Git, - dir: fs.Dir, - - const Git = struct { - session: git.Session, - fetch_stream: git.Session.FetchStream, - want_oid: git.Oid, - }; - - fn deinit(resource: *Resource) void { - switch (resource.*) { - .file => |*file| file.close(), - .http_request => |*req| req.deinit(), - .git => |*git_resource| { - git_resource.fetch_stream.deinit(); - git_resource.session.deinit(); - }, - .dir => |*dir| dir.close(), - } - resource.* = undefined; - } - - fn reader(resource: *Resource) std.io.AnyReader { - return .{ - .context = resource, - .readFn = read, - }; - } - - fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { - const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); - switch (resource.*) { - .file => |*f| return f.read(buffer), - .http_request => |*r| return r.read(buffer), - .git => |*g| return g.fetch_stream.read(buffer), - .dir => unreachable, - } - } -}; - -const FileType = enum { - tar, - @"tar.gz", - @"tar.xz", - @"tar.zst", - git_pack, - zip, - - fn fromPath(file_path: []const u8) ?FileType { - if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; - if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; - if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; - if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; - return null; - } - - /// Parameter is a content-disposition header value. - fn fromContentDisposition(cd_header: []const u8) ?FileType { - const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse - return null; - - var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse - return null; - value_start += "filename".len; - if (cd_header[value_start] == '*') { - value_start += 1; - } - if (cd_header[value_start] != '=') return null; - value_start += 1; - - var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; - if (cd_header[value_end - 1] == '\"') { - value_end -= 1; - } - return fromPath(cd_header[value_start..value_end]); - } - - test fromContentDisposition { - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); - try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); - - try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; size=42") == null); - try std.testing.expect(fromContentDisposition("inline; size=42") == null); - try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); - try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); - } -}; - -const header_buffer_size = 16 * 1024; - -fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource { - const gpa = f.arena.child_allocator; - const arena = f.arena.allocator(); - const eb = &f.error_bundle; - - if (ascii.eqlIgnoreCase(uri.scheme, "file")) { - const path = try uri.path.toRawMaybeAlloc(arena); - return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ - f.parent_package_root, path, @errorName(err), - })); - } }; - } - - const http_client = f.job_queue.http_client; - - if (ascii.eqlIgnoreCase(uri.scheme, "http") or - ascii.eqlIgnoreCase(uri.scheme, "https")) - { - var req = http_client.open(.GET, uri, .{ - .server_header_buffer = server_header_buffer, - }) catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to connect to server: {s}", - .{@errorName(err)}, - )); - }; - errdefer req.deinit(); // releases more than memory - - req.send() catch |err| { - return f.fail(f.location_tok, try eb.printString( - "HTTP request failed: {s}", - .{@errorName(err)}, - )); - }; - req.wait() catch |err| { - return f.fail(f.location_tok, try eb.printString( - "invalid HTTP response: {s}", - .{@errorName(err)}, - )); - }; - - if (req.response.status != .ok) { - return f.fail(f.location_tok, try eb.printString( - "bad HTTP response code: '{d} {s}'", - .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, - )); - } - - return .{ .http_request = req }; - } - - if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or - ascii.eqlIgnoreCase(uri.scheme, "git+https")) - { - var transport_uri = uri; - transport_uri.scheme = uri.scheme["git+".len..]; - var session = git.Session.init(gpa, http_client, transport_uri, server_header_buffer) catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to discover remote git server capabilities: {s}", - .{@errorName(err)}, - )); - }; - errdefer session.deinit(); - - const want_oid = want_oid: { - const want_ref = - if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; - if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} - - const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); - const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); - - var ref_iterator = session.listRefs(.{ - .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, - .include_peeled = true, - .server_header_buffer = server_header_buffer, - }) catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to list refs: {s}", - .{@errorName(err)}, - )); - }; - defer ref_iterator.deinit(); - while (ref_iterator.next() catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to iterate refs: {s}", - .{@errorName(err)}, - )); - }) |ref| { - if (std.mem.eql(u8, ref.name, want_ref) or - std.mem.eql(u8, ref.name, want_ref_head) or - std.mem.eql(u8, ref.name, want_ref_tag)) - { - break :want_oid ref.peeled orelse ref.oid; - } - } - return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); - }; - if (f.use_latest_commit) { - f.latest_commit = want_oid; - } else if (uri.fragment == null) { - const notes_len = 1; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("url field is missing an explicit ref"), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = notes_len, - }); - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("try .url = \"{;+/}#{}\",", .{ uri, want_oid }), - })); - return error.FetchFailed; - } - - var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; - _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{want_oid}) catch unreachable; - var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to create fetch stream: {s}", - .{@errorName(err)}, - )); - }; - errdefer fetch_stream.deinit(); - - return .{ .git = .{ - .session = session, - .fetch_stream = fetch_stream, - .want_oid = want_oid, - } }; - } - - return f.fail(f.location_tok, try eb.printString( - "unsupported URL scheme: {s}", - .{uri.scheme}, - )); -} - -fn unpackResource( - f: *Fetch, - resource: *Resource, - uri_path: []const u8, - tmp_directory: Cache.Directory, -) RunError!UnpackResult { - const eb = &f.error_bundle; - const file_type = switch (resource.*) { - .file => FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), - - .http_request => |req| ft: { - // Content-Type takes first precedence. - const content_type = req.response.content_type orelse - return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); - - // Extract the MIME type, ignoring charset and boundary directives - const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; - const mime_type = content_type[0..mime_type_end]; - - if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) - break :ft .tar; - - if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or - ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or - ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or - ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or - ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) - { - break :ft .@"tar.gz"; - } - - if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) - break :ft .@"tar.xz"; - - if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) - break :ft .@"tar.zst"; - - if (ascii.eqlIgnoreCase(mime_type, "application/zip")) - break :ft .zip; - - if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and - !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) - { - return f.fail(f.location_tok, try eb.printString( - "unrecognized 'Content-Type' header: '{s}'", - .{content_type}, - )); - } - - // Next, the filename from 'content-disposition: attachment' takes precedence. - if (req.response.content_disposition) |cd_header| { - break :ft FileType.fromContentDisposition(cd_header) orelse { - return f.fail(f.location_tok, try eb.printString( - "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", - .{cd_header}, - )); - }; - } - - // Finally, the path from the URI is used. - break :ft FileType.fromPath(uri_path) orelse { - return f.fail(f.location_tok, try eb.printString( - "unknown file type: '{s}'", - .{uri_path}, - )); - }; - }, - - .git => .git_pack, - - .dir => |dir| { - f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to copy directory '{s}': {s}", - .{ uri_path, @errorName(err) }, - )); - }; - return .{}; - }, - }; - - switch (file_type) { - .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()), - .@"tar.gz" => { - const reader = resource.reader(); - var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - var dcp = std.compress.gzip.decompressor(br.reader()); - return try unpackTarball(f, tmp_directory.handle, dcp.reader()); - }, - .@"tar.xz" => { - const gpa = f.arena.child_allocator; - const reader = resource.reader(); - var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to decompress tarball: {s}", - .{@errorName(err)}, - )); - }; - defer dcp.deinit(); - return try unpackTarball(f, tmp_directory.handle, dcp.reader()); - }, - .@"tar.zst" => { - const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; - const window_buffer = try f.arena.allocator().create([window_size]u8); - const reader = resource.reader(); - var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - var dcp = std.compress.zstd.decompressor(br.reader(), .{ - .window_buffer = window_buffer, - }); - return try unpackTarball(f, tmp_directory.handle, dcp.reader()); - }, - .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { - error.FetchFailed => return error.FetchFailed, - error.OutOfMemory => return error.OutOfMemory, - else => |e| return f.fail(f.location_tok, try eb.printString( - "unable to unpack git files: {s}", - .{@errorName(e)}, - )), - }, - .zip => return try unzip(f, tmp_directory.handle, resource.reader()), - } -} - -fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - - var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; - - std.tar.pipeToFileSystem(out_dir, reader, .{ - .diagnostics = &diagnostics, - .strip_components = 0, - .mode_mode = .ignore, - .exclude_empty_directories = true, - }) catch |err| return f.fail(f.location_tok, try eb.printString( - "unable to unpack tarball to temporary directory: {s}", - .{@errorName(err)}, - )); - - var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; - if (diagnostics.errors.items.len > 0) { - try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); - for (diagnostics.errors.items) |item| { - switch (item) { - .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), - .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), - .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), - .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 - } - } - } - return res; -} - -fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { - // We write the entire contents to a file first because zip files - // must be processed back to front and they could be too large to - // load into memory. - - const cache_root = f.job_queue.global_cache; - - // TODO: the downside of this solution is if we get a failure/crash/oom/power out - // during this process, we leave behind a zip file that would be - // difficult to know if/when it can be cleaned up. - // Might be worth it to use a mechanism that enables other processes - // to see if the owning process of a file is still alive (on linux this - // can be done with file locks). - // Coupled with this mechansism, we could also use slots (i.e. zig-cache/tmp/0, - // zig-cache/tmp/1, etc) which would mean that subsequent runs would - // automatically clean up old dead files. - // This could all be done with a simple TmpFile abstraction. - const prefix = "tmp/"; - const suffix = ".zip"; - - const random_bytes_count = 20; - const random_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count); - var zip_path: [prefix.len + random_path_len + suffix.len]u8 = undefined; - @memcpy(zip_path[0..prefix.len], prefix); - @memcpy(zip_path[prefix.len + random_path_len ..], suffix); - { - var random_bytes: [random_bytes_count]u8 = undefined; - std.crypto.random.bytes(&random_bytes); - _ = std.fs.base64_encoder.encode( - zip_path[prefix.len..][0..random_path_len], - &random_bytes, - ); - } - - defer cache_root.handle.deleteFile(&zip_path) catch {}; - - const eb = &f.error_bundle; - - { - var zip_file = cache_root.handle.createFile( - &zip_path, - .{}, - ) catch |err| return f.fail(f.location_tok, try eb.printString( - "failed to create tmp zip file: {s}", - .{@errorName(err)}, - )); - defer zip_file.close(); - var buf: [4096]u8 = undefined; - while (true) { - const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( - "read zip stream failed: {s}", - .{@errorName(err)}, - )); - if (len == 0) break; - zip_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString( - "write temporary zip file failed: {s}", - .{@errorName(err)}, - )); - } - } - - var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; - // no need to deinit since we are using an arena allocator - - { - var zip_file = cache_root.handle.openFile( - &zip_path, - .{}, - ) catch |err| return f.fail(f.location_tok, try eb.printString( - "failed to open temporary zip file: {s}", - .{@errorName(err)}, - )); - defer zip_file.close(); - - std.zip.extract(out_dir, zip_file.seekableStream(), .{ - .allow_backslashes = true, - .diagnostics = &diagnostics, - }) catch |err| return f.fail(f.location_tok, try eb.printString( - "zip extract failed: {s}", - .{@errorName(err)}, - )); - } - - cache_root.handle.deleteFile(&zip_path) catch |err| return f.fail(f.location_tok, try eb.printString( - "delete temporary zip failed: {s}", - .{@errorName(err)}, - )); - - const res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; - return res; -} - -fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult { - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const object_format: git.Oid.Format = resource.want_oid; - - var res: UnpackResult = .{}; - // The .git directory is used to store the packfile and associated index, but - // we do not attempt to replicate the exact structure of a real .git - // directory, since that isn't relevant for fetching a package. - { - var pack_dir = try out_dir.makeOpenPath(".git", .{}); - defer pack_dir.close(); - var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); - defer pack_file.close(); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(resource.fetch_stream.reader(), pack_file.writer()); - try pack_file.sync(); - - var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); - defer index_file.close(); - { - const index_prog_node = f.prog_node.start("Index pack", 0); - defer index_prog_node.end(); - var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); - try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer()); - try index_buffered_writer.flush(); - try index_file.sync(); - } - - { - const checkout_prog_node = f.prog_node.start("Checkout", 0); - defer checkout_prog_node.end(); - var repository = try git.Repository.init(gpa, object_format, pack_file, index_file); - defer repository.deinit(); - var diagnostics: git.Diagnostics = .{ .allocator = arena }; - try repository.checkout(out_dir, resource.want_oid, &diagnostics); - - if (diagnostics.errors.items.len > 0) { - try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); - for (diagnostics.errors.items) |item| { - switch (item) { - .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), - .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), - } - } - } - } - } - - try out_dir.deleteTree(".git"); - return res; -} - -fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { - const gpa = f.arena.child_allocator; - // Recursive directory copy. - var it = try dir.walk(gpa); - defer it.deinit(); - while (try it.next()) |entry| { - switch (entry.kind) { - .directory => {}, // omit empty directories - .file => { - dir.copyFile( - entry.path, - tmp_dir, - entry.path, - .{}, - ) catch |err| switch (err) { - error.FileNotFound => { - if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); - try dir.copyFile(entry.path, tmp_dir, entry.path, .{}); - }, - else => |e| return e, - }; - }, - .sym_link => { - var buf: [fs.max_path_bytes]u8 = undefined; - const link_name = try dir.readLink(entry.path, &buf); - // TODO: if this would create a symlink to outside - // the destination directory, fail with an error instead. - tmp_dir.symLink(link_name, entry.path, .{}) catch |err| switch (err) { - error.FileNotFound => { - if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); - try tmp_dir.symLink(link_name, entry.path, .{}); - }, - else => |e| return e, - }; - }, - else => return error.IllegalFileTypeInPackage, - } - } -} - -pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { - assert(dest_dir_sub_path[1] == fs.path.sep); - var handled_missing_dir = false; - while (true) { - cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { - error.FileNotFound => { - if (handled_missing_dir) return err; - cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { - error.PathAlreadyExists => handled_missing_dir = true, - else => |e| return e, - }; - continue; - }, - error.PathAlreadyExists, error.AccessDenied => { - // Package has been already downloaded and may already be in use on the system. - cache_dir.deleteTree(tmp_dir_sub_path) catch { - // Garbage files leftover in zig-cache/tmp/ is, as they say - // on Star Trek, "operating within normal parameters". - }; - }, - else => |e| return e, - }; - break; - } -} - -const ComputedHash = struct { - digest: Package.Hash.Digest, - total_size: u64, -}; - -/// Assumes that files not included in the package have already been filtered -/// prior to calling this function. This ensures that files not protected by -/// the hash are not present on the file system. Empty directories are *not -/// hashed* and must not be present on the file system when calling this -/// function. -fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { - // All the path name strings need to be in memory for sorting. - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const eb = &f.error_bundle; - const thread_pool = f.job_queue.thread_pool; - const root_dir = pkg_path.root_dir.handle; - - // Collect all files, recursively, then sort. - var all_files = std.ArrayList(*HashedFile).init(gpa); - defer all_files.deinit(); - - var deleted_files = std.ArrayList(*DeletedFile).init(gpa); - defer deleted_files.deinit(); - - // Track directories which had any files deleted from them so that empty directories - // can be deleted. - var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; - defer sus_dirs.deinit(gpa); - - var walker = try root_dir.walk(gpa); - defer walker.deinit(); - - // Total number of bytes of file contents included in the package. - var total_size: u64 = 0; - - { - // The final hash will be a hash of each file hashed independently. This - // allows hashing in parallel. - var wait_group: WaitGroup = .{}; - // `computeHash` is called from a worker thread so there must not be - // any waiting without working or a deadlock could occur. - defer thread_pool.waitAndWork(&wait_group); - - while (walker.next() catch |err| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to walk temporary directory '{}': {s}", - .{ pkg_path, @errorName(err) }, - ) }); - return error.FetchFailed; - }) |entry| { - if (entry.kind == .directory) continue; - - const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); - if (!filter.includePath(entry_pkg_path)) { - // Delete instead of including in hash calculation. - const fs_path = try arena.dupe(u8, entry.path); - - // Also track the parent directory in case it becomes empty. - if (fs.path.dirname(fs_path)) |parent| - try sus_dirs.put(gpa, parent, {}); - - const deleted_file = try arena.create(DeletedFile); - deleted_file.* = .{ - .fs_path = fs_path, - .failure = undefined, // to be populated by the worker - }; - thread_pool.spawnWg(&wait_group, workerDeleteFile, .{ root_dir, deleted_file }); - try deleted_files.append(deleted_file); - continue; - } - - const kind: HashedFile.Kind = switch (entry.kind) { - .directory => unreachable, - .file => .file, - .sym_link => .link, - else => return f.fail(f.location_tok, try eb.printString( - "package contains '{s}' which has illegal file type '{s}'", - .{ entry.path, @tagName(entry.kind) }, - )), - }; - - if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) - f.has_build_zig = true; - - const fs_path = try arena.dupe(u8, entry.path); - const hashed_file = try arena.create(HashedFile); - hashed_file.* = .{ - .fs_path = fs_path, - .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), - .kind = kind, - .hash = undefined, // to be populated by the worker - .failure = undefined, // to be populated by the worker - .size = undefined, // to be populated by the worker - }; - thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file }); - try all_files.append(hashed_file); - } - } - - { - // Sort by length, descending, so that child directories get removed first. - sus_dirs.sortUnstable(@as(struct { - keys: []const []const u8, - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return ctx.keys[b_index].len < ctx.keys[a_index].len; - } - }, .{ .keys = sus_dirs.keys() })); - - // During this loop, more entries will be added, so we must loop by index. - var i: usize = 0; - while (i < sus_dirs.count()) : (i += 1) { - const sus_dir = sus_dirs.keys()[i]; - root_dir.deleteDir(sus_dir) catch |err| switch (err) { - error.DirNotEmpty => continue, - error.FileNotFound => continue, - else => |e| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to delete empty directory '{s}': {s}", - .{ sus_dir, @errorName(e) }, - ) }); - return error.FetchFailed; - }, - }; - if (fs.path.dirname(sus_dir)) |parent| { - try sus_dirs.put(gpa, parent, {}); - } - } - } - - std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); - - var hasher = Package.Hash.Algo.init(.{}); - var any_failures = false; - for (all_files.items) |hashed_file| { - hashed_file.failure catch |err| { - any_failures = true; - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to hash '{s}': {s}", .{ - hashed_file.fs_path, @errorName(err), - }), - }); - }; - hasher.update(&hashed_file.hash); - total_size += hashed_file.size; - } - for (deleted_files.items) |deleted_file| { - deleted_file.failure catch |err| { - any_failures = true; - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ - deleted_file.fs_path, @errorName(err), - }), - }); - }; - } - - if (any_failures) return error.FetchFailed; - - if (f.job_queue.debug_hash) { - assert(!f.job_queue.recursive); - // Print something to stdout that can be text diffed to figure out why - // the package hash is different. - dumpHashInfo(all_files.items) catch |err| { - std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)}); - std.process.exit(1); - }; - } - - return .{ - .digest = hasher.finalResult(), - .total_size = total_size, - }; -} - -fn dumpHashInfo(all_files: []const *const HashedFile) !void { - const stdout = std.io.getStdOut(); - var bw = std.io.bufferedWriter(stdout.writer()); - const w = bw.writer(); - - for (all_files) |hashed_file| { - try w.print("{s}: {s}: {s}\n", .{ - @tagName(hashed_file.kind), - std.fmt.fmtSliceHexLower(&hashed_file.hash), - hashed_file.normalized_path, - }); - } - - try bw.flush(); -} - -fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile) void { - hashed_file.failure = hashFileFallible(dir, hashed_file); -} - -fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void { - deleted_file.failure = deleteFileFallible(dir, deleted_file); -} - -fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { - var buf: [8000]u8 = undefined; - var hasher = Package.Hash.Algo.init(.{}); - hasher.update(hashed_file.normalized_path); - var file_size: u64 = 0; - - switch (hashed_file.kind) { - .file => { - var file = try dir.openFile(hashed_file.fs_path, .{}); - defer file.close(); - // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 - hasher.update(&.{ 0, 0 }); - var file_header: FileHeader = .{}; - while (true) { - const bytes_read = try file.read(&buf); - if (bytes_read == 0) break; - file_size += bytes_read; - hasher.update(buf[0..bytes_read]); - file_header.update(buf[0..bytes_read]); - } - if (file_header.isExecutable()) { - try setExecutable(file); - } - }, - .link => { - const link_name = try dir.readLink(hashed_file.fs_path, &buf); - if (fs.path.sep != canonical_sep) { - // Package hashes are intended to be consistent across - // platforms which means we must normalize path separators - // inside symlinks. - normalizePath(link_name); - } - hasher.update(link_name); - }, - } - hasher.final(&hashed_file.hash); - hashed_file.size = file_size; -} - -fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { - try dir.deleteFile(deleted_file.fs_path); -} - -fn setExecutable(file: fs.File) !void { - if (!std.fs.has_executable_bit) return; - - const S = std.posix.S; - const mode = fs.File.default_mode | S.IXUSR | S.IXGRP | S.IXOTH; - try file.chmod(mode); -} - -const DeletedFile = struct { - fs_path: []const u8, - failure: Error!void, - - const Error = - fs.Dir.DeleteFileError || - fs.Dir.DeleteDirError; -}; - -const HashedFile = struct { - fs_path: []const u8, - normalized_path: []const u8, - hash: Package.Hash.Digest, - failure: Error!void, - kind: Kind, - size: u64, - - const Error = - fs.File.OpenError || - fs.File.ReadError || - fs.File.StatError || - fs.File.ChmodError || - fs.Dir.ReadLinkError; - - const Kind = enum { file, link }; - - fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { - _ = context; - return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); - } -}; - -/// Strips root directory name from file system path. -fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { - if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; - - if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { - return fs_path[root_dir.len + 1 ..]; - } - - return fs_path; -} - -/// Make a file system path identical independently of operating system path inconsistencies. -/// This converts backslashes into forward slashes. -fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { - const normalized = try arena.dupe(u8, pkg_path); - if (fs.path.sep == canonical_sep) return normalized; - normalizePath(normalized); - return normalized; -} - -const canonical_sep = fs.path.sep_posix; - -fn normalizePath(bytes: []u8) void { - assert(fs.path.sep != canonical_sep); - std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); -} - -const Filter = struct { - include_paths: std.StringArrayHashMapUnmanaged(void) = .empty, - - /// sub_path is relative to the package root. - pub fn includePath(self: Filter, sub_path: []const u8) bool { - if (self.include_paths.count() == 0) return true; - if (self.include_paths.contains("")) return true; - if (self.include_paths.contains(".")) return true; - if (self.include_paths.contains(sub_path)) return true; - - // Check if any included paths are parent directories of sub_path. - var dirname = sub_path; - while (std.fs.path.dirname(dirname)) |next_dirname| { - if (self.include_paths.contains(next_dirname)) return true; - dirname = next_dirname; - } - - return false; - } - - test includePath { - const gpa = std.testing.allocator; - var filter: Filter = .{}; - defer filter.include_paths.deinit(gpa); - - try filter.include_paths.put(gpa, "src", {}); - try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); - try std.testing.expect(!filter.includePath(".gitignore")); - } -}; - -pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { - if (dep.hash) |h| return .fromSlice(h); - - switch (dep.location) { - .url => return null, - .path => |rel_path| { - var buf: [fs.max_path_bytes]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch - return null; - return relativePathDigest(new_root, cache_root); - }, - } -} - -const builtin = @import("builtin"); -const std = @import("std"); -const fs = std.fs; -const assert = std.debug.assert; -const ascii = std.ascii; -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const ThreadPool = std.Thread.Pool; -const WaitGroup = std.Thread.WaitGroup; -const Fetch = @This(); -const git = @import("Fetch/git.zig"); -const Package = @import("../Package.zig"); -const Manifest = Package.Manifest; -const ErrorBundle = std.zig.ErrorBundle; -const native_os = builtin.os.tag; - -test { - _ = Filter; - _ = FileType; - _ = UnpackResult; -} - -// Detects executable header: ELF or Macho-O magic header or shebang line. -const FileHeader = struct { - header: [4]u8 = undefined, - bytes_read: usize = 0, - - pub fn update(self: *FileHeader, buf: []const u8) void { - if (self.bytes_read >= self.header.len) return; - const n = @min(self.header.len - self.bytes_read, buf.len); - @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); - self.bytes_read += n; - } - - fn isScript(self: *FileHeader) bool { - const shebang = "#!"; - return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); - } - - fn isElf(self: *FileHeader) bool { - const elf_magic = std.elf.MAGIC; - return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); - } - - fn isMachO(self: *FileHeader) bool { - if (self.bytes_read < 4) return false; - const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); - return magic_number == std.macho.MH_MAGIC or - magic_number == std.macho.MH_MAGIC_64 or - magic_number == std.macho.FAT_MAGIC or - magic_number == std.macho.FAT_MAGIC_64 or - magic_number == std.macho.MH_CIGAM or - magic_number == std.macho.MH_CIGAM_64 or - magic_number == std.macho.FAT_CIGAM or - magic_number == std.macho.FAT_CIGAM_64; - } - - pub fn isExecutable(self: *FileHeader) bool { - return self.isScript() or self.isElf() or self.isMachO(); - } -}; - -test FileHeader { - var h: FileHeader = .{}; - try std.testing.expect(!h.isExecutable()); - - const elf_magic = std.elf.MAGIC; - h.update(elf_magic[0..2]); - try std.testing.expect(!h.isExecutable()); - h.update(elf_magic[2..4]); - try std.testing.expect(h.isExecutable()); - - h.update(elf_magic[2..4]); - try std.testing.expect(h.isExecutable()); - - const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; - h.bytes_read = 0; - h.update(&macho64_magic_bytes); - try std.testing.expect(h.isExecutable()); - - const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; - h.bytes_read = 0; - h.update(&macho64_cigam_bytes); - try std.testing.expect(h.isExecutable()); -} - -// Result of the `unpackResource` operation. Enables collecting errors from -// tar/git diagnostic, filtering that errors by manifest inclusion rules and -// emitting remaining errors to an `ErrorBundle`. -const UnpackResult = struct { - errors: []Error = undefined, - errors_count: usize = 0, - root_error_message: []const u8 = "", - - // A non empty value means that the package contents are inside a - // sub-directory indicated by the named path. - root_dir: []const u8 = "", - - const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - unable_to_create_file: struct { - code: anyerror, - file_name: []const u8, - }, - unsupported_file_type: struct { - file_name: []const u8, - file_type: u8, - }, - - fn excluded(self: Error, filter: Filter) bool { - const file_name = switch (self) { - .unable_to_create_file => |info| info.file_name, - .unable_to_create_sym_link => |info| info.file_name, - .unsupported_file_type => |info| info.file_name, - }; - return !filter.includePath(file_name); - } - }; - - fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { - self.root_error_message = try arena.dupe(u8, root_error_message); - self.errors = try arena.alloc(UnpackResult.Error, n); - } - - fn hasErrors(self: *UnpackResult) bool { - return self.errors_count > 0; - } - - fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { - self.errors[self.errors_count] = .{ .unable_to_create_file = .{ - .code = err, - .file_name = file_name, - } }; - self.errors_count += 1; - } - - fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { - self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ - .code = err, - .file_name = file_name, - .link_name = link_name, - } }; - self.errors_count += 1; - } - - fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { - self.errors[self.errors_count] = .{ .unsupported_file_type = .{ - .file_name = file_name, - .file_type = file_type, - } }; - self.errors_count += 1; - } - - fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { - if (self.errors_count == 0) return; - - var unfiltered_errors: u32 = 0; - for (self.errors) |item| { - if (item.excluded(filter)) continue; - unfiltered_errors += 1; - } - if (unfiltered_errors == 0) return; - - // Emmit errors to an `ErrorBundle`. - const eb = &f.error_bundle; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString(self.root_error_message), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = unfiltered_errors, - }); - var note_i: u32 = try eb.reserveNotes(unfiltered_errors); - for (self.errors) |item| { - if (item.excluded(filter)) continue; - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - .unable_to_create_file => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create file '{s}': {s}", .{ - info.file_name, @errorName(info.code), - }), - })); - }, - .unsupported_file_type => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ - info.file_name, info.file_type, - }), - })); - }, - } - note_i += 1; - } - - return error.FetchFailed; - } - - test validate { - const gpa = std.testing.allocator; - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - // fill UnpackResult with errors - var res: UnpackResult = .{}; - try res.allocErrors(arena, 4, "unable to unpack"); - try std.testing.expectEqual(0, res.errors_count); - res.unableToCreateFile("dir1/file1", error.File1); - res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); - res.unableToCreateFile("dir1/file3", error.File3); - res.unsupportedFileType("dir2/file4", 'x'); - try std.testing.expectEqual(4, res.errors_count); - - // create filter, includes dir2, excludes dir1 - var filter: Filter = .{}; - try filter.include_paths.put(arena, "dir2", {}); - - // init Fetch - var fetch: Fetch = undefined; - fetch.parent_manifest_ast = null; - fetch.location_tok = 0; - try fetch.error_bundle.init(gpa); - defer fetch.error_bundle.deinit(); - - // validate errors with filter - try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); - - // output errors to string - var errors = try fetch.error_bundle.toOwnedBundle(""); - defer errors.deinit(gpa); - var out = std.ArrayList(u8).init(gpa); - defer out.deinit(); - try errors.renderToWriter(.{ .ttyconf = .no_color }, out.writer()); - try std.testing.expectEqualStrings( - \\error: unable to unpack - \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError - \\ note: file 'dir2/file4' has unsupported type 'x' - \\ - , out.items); - } -}; - -test "zip" { - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const test_files = [_]std.zip.testutil.File{ - .{ .name = "foo", .content = "this is just foo\n", .compression = .store }, - .{ .name = "bar", .content = "another file\n", .compression = .deflate }, - }; - { - var zip_file = try tmp.dir.createFile("test.zip", .{}); - defer zip_file.close(); - var bw = std.io.bufferedWriter(zip_file.writer()); - var store: [test_files.len]std.zip.testutil.FileStore = undefined; - try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); - try bw.flush(); - } - - const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); - defer gpa.free(zip_path); - - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, zip_path); - defer fb.deinit(); - - try fetch.run(); - - var out = try fb.packageDir(); - defer out.close(); - - try std.zip.testutil.expectFiles(&test_files, out, .{}); -} - -test "zip with one root folder" { - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const test_files = [_]std.zip.testutil.File{ - .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store }, - .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store }, - }; - { - var zip_file = try tmp.dir.createFile("test.zip", .{}); - defer zip_file.close(); - var bw = std.io.bufferedWriter(zip_file.writer()); - var store: [test_files.len]std.zip.testutil.FileStore = undefined; - try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); - try bw.flush(); - } - - const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); - defer gpa.free(zip_path); - - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, zip_path); - defer fb.deinit(); - - try fetch.run(); - - var out = try fb.packageDir(); - defer out.close(); - - try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" }); -} - -test "tarball with duplicate paths" { - // This tarball has duplicate path 'dir1/file1' to simulate case sensitve - // file system on any file sytstem. - // - // duplicate_paths/ - // duplicate_paths/dir1/ - // duplicate_paths/dir1/file1 - // duplicate_paths/dir1/file1 - // duplicate_paths/build.zig.zon - // duplicate_paths/src/ - // duplicate_paths/src/main.zig - // duplicate_paths/src/root.zig - // duplicate_paths/build.zig - // - - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "duplicate_paths.tar.gz"; - try saveEmbedFile(tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // Run tarball fetch, expect to fail - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, tarball_path); - defer fb.deinit(); - try std.testing.expectError(error.FetchFailed, fetch.run()); - - try fb.expectFetchErrors(1, - \\error: unable to unpack tarball - \\ note: unable to create file 'dir1/file1': PathAlreadyExists - \\ - ); -} - -test "tarball with excluded duplicate paths" { - // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. - // - // .paths = .{ - // "build.zig", - // "build.zig.zon", - // "src", - // } - // - - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "duplicate_paths_excluded.tar.gz"; - try saveEmbedFile(tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // Run tarball fetch, should succeed - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, tarball_path); - defer fb.deinit(); - try fetch.run(); - - const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); - try std.testing.expectEqualStrings( - "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", - &hex_digest, - ); - - const expected_files: []const []const u8 = &.{ - "build.zig", - "build.zig.zon", - "src/main.zig", - "src/root.zig", - }; - try fb.expectPackageFiles(expected_files); -} - -test "tarball without root folder" { - // Tarball with root folder. Manifest excludes dir1 and dir2. - // - // build.zig - // build.zig.zon - // dir1/ - // dir1/file2 - // dir1/file1 - // dir2/ - // dir2/file2 - // src/ - // src/main.zig - // - - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "no_root.tar.gz"; - try saveEmbedFile(tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // Run tarball fetch, should succeed - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, tarball_path); - defer fb.deinit(); - try fetch.run(); - - const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); - try std.testing.expectEqualStrings( - "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", - &hex_digest, - ); - - const expected_files: []const []const u8 = &.{ - "build.zig", - "build.zig.zon", - "src/main.zig", - }; - try fb.expectPackageFiles(expected_files); -} - -test "set executable bit based on file content" { - if (!std.fs.has_executable_bit) return error.SkipZigTest; - const gpa = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "executables.tar.gz"; - try saveEmbedFile(tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // $ tar -tvf executables.tar.gz - // drwxrwxr-x 0 executables/ - // -rwxrwxr-x 170 executables/hello - // lrwxrwxrwx 0 executables/hello_ln -> hello - // -rw-rw-r-- 0 executables/file1 - // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit - // -rwxrwxr-x 7 executables/script_without_shebang - // -rwxrwxr-x 17 executables/script - - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, tmp.dir, tarball_path); - defer fb.deinit(); - - try fetch.run(); - try std.testing.expectEqualStrings( - "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", - &Package.multiHashHexDigest(fetch.computed_hash.digest), - ); - - var out = try fb.packageDir(); - defer out.close(); - const S = std.posix.S; - // expect executable bit not set - try std.testing.expect((try out.statFile("file1")).mode & S.IXUSR == 0); - try std.testing.expect((try out.statFile("script_without_shebang")).mode & S.IXUSR == 0); - // expect executable bit set - try std.testing.expect((try out.statFile("hello")).mode & S.IXUSR != 0); - try std.testing.expect((try out.statFile("script")).mode & S.IXUSR != 0); - try std.testing.expect((try out.statFile("script_with_shebang_without_exec_bit")).mode & S.IXUSR != 0); - try std.testing.expect((try out.statFile("hello_ln")).mode & S.IXUSR != 0); - - // - // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3 - // -rw-rw-r-- 1 0 Apr file1 - // -rwxrwxr-x 1 170 Apr hello - // lrwxrwxrwx 1 5 Apr hello_ln -> hello - // -rwxrwxr-x 1 17 Apr script - // -rw-rw-r-- 1 7 Apr script_without_shebang - // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit -} - -fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void { - //const tarball_name = "duplicate_paths_excluded.tar.gz"; - const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name); - var tmp_file = try dir.createFile(tarball_name, .{}); - defer tmp_file.close(); - try tmp_file.writeAll(tarball_content); -} - -// Builds Fetch with required dependencies, clears dependencies on deinit(). -const TestFetchBuilder = struct { - thread_pool: ThreadPool, - http_client: std.http.Client, - global_cache_directory: Cache.Directory, - job_queue: Fetch.JobQueue, - fetch: Fetch, - - fn build( - self: *TestFetchBuilder, - allocator: std.mem.Allocator, - cache_parent_dir: std.fs.Dir, - path_or_url: []const u8, - ) !*Fetch { - const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{}); - - try self.thread_pool.init(.{ .allocator = allocator }); - self.http_client = .{ .allocator = allocator }; - self.global_cache_directory = .{ .handle = cache_dir, .path = null }; - - self.job_queue = .{ - .http_client = &self.http_client, - .thread_pool = &self.thread_pool, - .global_cache = self.global_cache_directory, - .recursive = false, - .read_only = false, - .debug_hash = false, - .work_around_btrfs_bug = false, - }; - - self.fetch = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .location = .{ .path_or_url = path_or_url }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, - .parent_manifest_ast = null, - .prog_node = std.Progress.Node.none, - .job_queue = &self.job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz - .allow_name_string = true, // so we can keep using the old testdata .tar.gz - .use_latest_commit = true, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = null, - .manifest_ast = undefined, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - return &self.fetch; - } - - fn deinit(self: *TestFetchBuilder) void { - self.fetch.deinit(); - self.job_queue.deinit(); - self.fetch.prog_node.end(); - self.global_cache_directory.handle.close(); - self.http_client.deinit(); - self.thread_pool.deinit(); - } - - fn packageDir(self: *TestFetchBuilder) !fs.Dir { - const root = self.fetch.package_root; - return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); - } - - // Test helper, asserts thet package dir constains expected_files. - // expected_files must be sorted. - fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { - var package_dir = try self.packageDir(); - defer package_dir.close(); - - var actual_files: std.ArrayListUnmanaged([]u8) = .empty; - defer actual_files.deinit(std.testing.allocator); - defer for (actual_files.items) |file| std.testing.allocator.free(file); - var walker = try package_dir.walk(std.testing.allocator); - defer walker.deinit(); - while (try walker.next()) |entry| { - if (entry.kind != .file) continue; - const path = try std.testing.allocator.dupe(u8, entry.path); - errdefer std.testing.allocator.free(path); - std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); - try actual_files.append(std.testing.allocator, path); - } - std.mem.sortUnstable([]u8, actual_files.items, {}, struct { - fn lessThan(_: void, a: []u8, b: []u8) bool { - return std.mem.lessThan(u8, a, b); - } - }.lessThan); - - try std.testing.expectEqual(expected_files.len, actual_files.items.len); - for (expected_files, 0..) |file_name, i| { - try std.testing.expectEqualStrings(file_name, actual_files.items[i]); - } - try std.testing.expectEqualDeep(expected_files, actual_files.items); - } - - // Test helper, asserts that fetch has failed with `msg` error message. - fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { - var errors = try self.fetch.error_bundle.toOwnedBundle(""); - defer errors.deinit(std.testing.allocator); - - const em = errors.getErrorMessage(errors.getMessages()[0]); - try std.testing.expectEqual(1, em.count); - if (notes_len > 0) { - try std.testing.expectEqual(notes_len, em.notes_len); - } - var al = std.ArrayList(u8).init(std.testing.allocator); - defer al.deinit(); - try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); - try std.testing.expectEqualStrings(msg, al.items); - } -}; diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig deleted file mode 100644 index f6e3dc16152a0024cc6d546f655c51db282a4e6d..0000000000000000000000000000000000000000 --- a/src/Package/Fetch/git.zig +++ /dev/null @@ -1,1689 +0,0 @@ -//! Git support for package fetching. -//! -//! This is not intended to support all features of Git: it is limited to the -//! basic functionality needed to clone a repository for the purpose of fetching -//! a package. - -const std = @import("std"); -const mem = std.mem; -const testing = std.testing; -const Allocator = mem.Allocator; -const Sha1 = std.crypto.hash.Sha1; -const Sha256 = std.crypto.hash.sha2.Sha256; -const assert = std.debug.assert; - -/// The ID of a Git object. -pub const Oid = union(Format) { - sha1: [Sha1.digest_length]u8, - sha256: [Sha256.digest_length]u8, - - pub const max_formatted_length = len: { - var max: usize = 0; - for (std.enums.values(Format)) |f| { - max = @max(max, f.formattedLength()); - } - break :len max; - }; - - pub const Format = enum { - sha1, - sha256, - - pub fn byteLength(f: Format) usize { - return switch (f) { - .sha1 => Sha1.digest_length, - .sha256 => Sha256.digest_length, - }; - } - - pub fn formattedLength(f: Format) usize { - return 2 * f.byteLength(); - } - }; - - const Hasher = union(Format) { - sha1: Sha1, - sha256: Sha256, - - fn init(oid_format: Format) Hasher { - return switch (oid_format) { - .sha1 => .{ .sha1 = Sha1.init(.{}) }, - .sha256 => .{ .sha256 = Sha256.init(.{}) }, - }; - } - - // Must be public for use from HashedReader and HashedWriter. - pub fn update(hasher: *Hasher, b: []const u8) void { - switch (hasher.*) { - inline else => |*inner| inner.update(b), - } - } - - fn finalResult(hasher: *Hasher) Oid { - return switch (hasher.*) { - inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), - }; - } - }; - - pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { - assert(bytes.len == oid_format.byteLength()); - return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), - }; - } - - pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { - return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), - }; - } - - pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { - switch (oid_format) { - inline else => |tag| { - if (s.len != tag.formattedLength()) return error.InvalidOid; - var bytes: [tag.byteLength()]u8 = undefined; - for (&bytes, 0..) |*b, i| { - b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; - } - return @unionInit(Oid, @tagName(tag), bytes); - }, - } - } - - test parse { - try testing.expectEqualSlices( - u8, - &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, - &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, - ); - try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); - try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); - try testing.expectEqualSlices( - u8, - &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, - &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, - ); - try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); - try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); - try testing.expectError(error.InvalidOid, parse(.sha1, "master")); - try testing.expectError(error.InvalidOid, parse(.sha256, "master")); - try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); - try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); - } - - pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { - return for (std.enums.values(Format)) |f| { - if (s.len == f.formattedLength()) break parse(f, s); - } else error.InvalidOid; - } - - pub fn format( - oid: Oid, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) @TypeOf(writer).Error!void { - _ = fmt; - _ = options; - try writer.print("{}", .{std.fmt.fmtSliceHexLower(oid.slice())}); - } - - pub fn slice(oid: *const Oid) []const u8 { - return switch (oid.*) { - inline else => |*bytes| bytes, - }; - } -}; - -pub const Diagnostics = struct { - allocator: Allocator, - errors: std.ArrayListUnmanaged(Error) = .empty, - - pub const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - unable_to_create_file: struct { - code: anyerror, - file_name: []const u8, - }, - }; - - pub fn deinit(d: *Diagnostics) void { - for (d.errors.items) |item| { - switch (item) { - .unable_to_create_sym_link => |info| { - d.allocator.free(info.file_name); - d.allocator.free(info.link_name); - }, - .unable_to_create_file => |info| { - d.allocator.free(info.file_name); - }, - } - } - d.errors.deinit(d.allocator); - d.* = undefined; - } -}; - -pub const Repository = struct { - odb: Odb, - - pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { - return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; - } - - pub fn deinit(repository: *Repository) void { - repository.odb.deinit(); - repository.* = undefined; - } - - /// Checks out the repository at `commit_oid` to `worktree`. - pub fn checkout( - repository: *Repository, - worktree: std.fs.Dir, - commit_oid: Oid, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(commit_oid); - const tree_oid = tree_oid: { - const commit_object = try repository.odb.readObject(); - if (commit_object.type != .commit) return error.NotACommit; - break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); - }; - try repository.checkoutTree(worktree, tree_oid, "", diagnostics); - } - - /// Checks out the tree at `tree_oid` to `worktree`. - fn checkoutTree( - repository: *Repository, - dir: std.fs.Dir, - tree_oid: Oid, - current_path: []const u8, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(tree_oid); - const tree_object = try repository.odb.readObject(); - if (tree_object.type != .tree) return error.NotATree; - // The tree object may be evicted from the object cache while we're - // iterating over it, so we can make a defensive copy here to make sure - // it remains valid until we're done with it - const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); - defer repository.odb.allocator.free(tree_data); - - var tree_iter: TreeIterator = .{ - .format = repository.odb.format, - .data = tree_data, - .pos = 0, - }; - while (try tree_iter.next()) |entry| { - switch (entry.type) { - .directory => { - try dir.makeDir(entry.name); - var subdir = try dir.openDir(entry.name, .{}); - defer subdir.close(); - const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); - defer repository.odb.allocator.free(sub_path); - try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics); - }, - .file => { - try repository.odb.seekOid(entry.oid); - const file_object = try repository.odb.readObject(); - if (file_object.type != .blob) return error.InvalidFile; - var file = dir.createFile(entry.name, .{ .exclusive = true }) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ - .code = e, - .file_name = file_name, - } }); - continue; - }; - defer file.close(); - try file.writeAll(file_object.data); - try file.sync(); - }, - .symlink => { - try repository.odb.seekOid(entry.oid); - const symlink_object = try repository.odb.readObject(); - if (symlink_object.type != .blob) return error.InvalidFile; - const link_name = symlink_object.data; - dir.symLink(link_name, entry.name, .{}) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); - errdefer diagnostics.allocator.free(link_name_dup); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ - .code = e, - .file_name = file_name, - .link_name = link_name_dup, - } }); - }; - }, - .gitlink => { - // Consistent with git archive behavior, create the directory but - // do nothing else - try dir.makeDir(entry.name); - }, - } - } - } - - /// Returns the ID of the tree associated with the given commit (provided as - /// raw object data). - fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { - if (!mem.startsWith(u8, commit_data, "tree ") or - commit_data.len < "tree ".len + format.formattedLength() + "\n".len or - commit_data["tree ".len + format.formattedLength()] != '\n') - { - return error.InvalidCommit; - } - return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); - } - - const TreeIterator = struct { - format: Oid.Format, - data: []const u8, - pos: usize, - - const Entry = struct { - type: Type, - executable: bool, - name: [:0]const u8, - oid: Oid, - - const Type = enum(u4) { - directory = 0o4, - file = 0o10, - symlink = 0o12, - gitlink = 0o16, - }; - }; - - fn next(iterator: *TreeIterator) !?Entry { - if (iterator.pos == iterator.data.len) return null; - - const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; - const mode: packed struct { - permission: u9, - unused: u3, - type: u4, - } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); - const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; - const executable = switch (mode.permission) { - 0 => if (@"type" == .file) return error.InvalidTree else false, - 0o644 => if (@"type" != .file) return error.InvalidTree else false, - 0o755 => if (@"type" != .file) return error.InvalidTree else true, - else => return error.InvalidTree, - }; - iterator.pos = mode_end + 1; - - const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; - const name = iterator.data[iterator.pos..name_end :0]; - iterator.pos = name_end + 1; - - const oid_length = iterator.format.byteLength(); - if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; - const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); - iterator.pos += oid_length; - - return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; - } - }; -}; - -/// A Git object database backed by a packfile. A packfile index is also used -/// for efficient access to objects in the packfile. -/// -/// The format of the packfile and its associated index are documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -const Odb = struct { - format: Oid.Format, - pack_file: std.fs.File, - index_header: IndexHeader, - index_file: std.fs.File, - cache: ObjectCache = .{}, - allocator: Allocator, - - /// Initializes the database from open pack and index files. - fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { - try pack_file.seekTo(0); - try index_file.seekTo(0); - const index_header = try IndexHeader.read(index_file.reader()); - return .{ - .format = format, - .pack_file = pack_file, - .index_header = index_header, - .index_file = index_file, - .allocator = allocator, - }; - } - - fn deinit(odb: *Odb) void { - odb.cache.deinit(odb.allocator); - odb.* = undefined; - } - - /// Reads the object at the current position in the database. - fn readObject(odb: *Odb) !Object { - var base_offset = try odb.pack_file.getPos(); - var base_header: EntryHeader = undefined; - var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; - defer delta_offsets.deinit(odb.allocator); - const base_object = while (true) { - if (odb.cache.get(base_offset)) |base_object| break base_object; - - base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(odb.allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; - try odb.pack_file.seekTo(base_offset); - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(odb.allocator, base_offset); - try odb.seekOid(ref_delta.base_object); - base_offset = try odb.pack_file.getPos(); - }, - else => { - const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); - errdefer odb.allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try odb.cache.put(odb.allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain( - odb.allocator, - odb.format, - odb.pack_file, - base_object, - delta_offsets.items, - &odb.cache, - ); - - return .{ .type = base_object.type, .data = base_data }; - } - - /// Seeks to the beginning of the object with the given ID. - fn seekOid(odb: *Odb, oid: Oid) !void { - const oid_length = odb.format.byteLength(); - const key = oid.slice()[0]; - var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; - var end_index = odb.index_header.fan_out_table[key]; - const found_index = while (start_index < end_index) { - const mid_index = start_index + (end_index - start_index) / 2; - try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); - switch (mem.order(u8, mid_oid.slice(), oid.slice())) { - .lt => start_index = mid_index + 1, - .gt => end_index = mid_index, - .eq => break mid_index, - } - } else return error.ObjectNotFound; - - const n_objects = odb.index_header.fan_out_table[255]; - const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); - try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big)); - const pack_offset = pack_offset: { - if (l1_offset.big) { - const l2_offset_values_start = offset_values_start + n_objects * 4; - try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.reader().readInt(u64, .big); - } else { - break :pack_offset l1_offset.value; - } - }; - - try odb.pack_file.seekTo(pack_offset); - } -}; - -const Object = struct { - type: Type, - data: []const u8, - - const Type = enum { - commit, - tree, - blob, - tag, - }; -}; - -/// A cache for object data. -/// -/// The purpose of this cache is to speed up resolution of deltas by caching the -/// results of resolving delta objects, while maintaining a maximum cache size -/// to avoid excessive memory usage. If the total size of the objects in the -/// cache exceeds the maximum, the cache will begin evicting the least recently -/// used objects: when resolving delta chains, the most recently used objects -/// will likely be more helpful as they will be further along in the chain -/// (skipping earlier reconstruction steps). -/// -/// Object data stored in the cache is managed by the cache. It should not be -/// freed by the caller at any point after inserting it into the cache. Any -/// objects remaining in the cache will be freed when the cache itself is freed. -const ObjectCache = struct { - objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, - lru_nodes: LruList = .{}, - byte_size: usize = 0, - - const max_byte_size = 128 * 1024 * 1024; // 128MiB - /// A list of offsets stored in the cache, with the most recently used - /// entries at the end. - const LruList = std.DoublyLinkedList(u64); - const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; - - fn deinit(cache: *ObjectCache, allocator: Allocator) void { - var object_iterator = cache.objects.iterator(); - while (object_iterator.next()) |object| { - allocator.free(object.value_ptr.object.data); - allocator.destroy(object.value_ptr.lru_node); - } - cache.objects.deinit(allocator); - cache.* = undefined; - } - - /// Gets an object from the cache, moving it to the most recently used - /// position if it is present. - fn get(cache: *ObjectCache, offset: u64) ?Object { - if (cache.objects.get(offset)) |entry| { - cache.lru_nodes.remove(entry.lru_node); - cache.lru_nodes.append(entry.lru_node); - return entry.object; - } else { - return null; - } - } - - /// Puts an object in the cache, possibly evicting older entries if the - /// cache exceeds its maximum size. Note that, although old objects may - /// be evicted, the object just added to the cache with this function - /// will not be evicted before the next call to `put` or `deinit` even if - /// it exceeds the maximum cache size. - fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { - const lru_node = try allocator.create(LruList.Node); - errdefer allocator.destroy(lru_node); - lru_node.data = offset; - - const gop = try cache.objects.getOrPut(allocator, offset); - if (gop.found_existing) { - cache.byte_size -= gop.value_ptr.object.data.len; - cache.lru_nodes.remove(gop.value_ptr.lru_node); - allocator.destroy(gop.value_ptr.lru_node); - allocator.free(gop.value_ptr.object.data); - } - gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; - cache.byte_size += object.data.len; - cache.lru_nodes.append(lru_node); - - while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { - // The > 1 check is to make sure that we don't evict the most - // recently added node, even if it by itself happens to exceed the - // maximum size of the cache. - const evict_node = cache.lru_nodes.popFirst().?; - const evict_offset = evict_node.data; - allocator.destroy(evict_node); - const evict_object = cache.objects.get(evict_offset).?.object; - cache.byte_size -= evict_object.data.len; - allocator.free(evict_object.data); - _ = cache.objects.remove(evict_offset); - } - } -}; - -/// A single pkt-line in the Git protocol. -/// -/// The format of a pkt-line is documented in -/// [protocol-common](https://git-scm.com/docs/protocol-common). The special -/// meanings of the delimiter and response-end packets are documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -const Packet = union(enum) { - flush, - delimiter, - response_end, - data: []const u8, - - const max_data_length = 65516; - - /// Reads a packet in pkt-line format. - fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { - const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; - switch (length) { - 0 => return .flush, - 1 => return .delimiter, - 2 => return .response_end, - 3 => return error.InvalidPacket, - else => if (length - 4 > max_data_length) return error.InvalidPacket, - } - const data = buf[0 .. length - 4]; - try reader.readNoEof(data); - return .{ .data = data }; - } - - /// Writes a packet in pkt-line format. - fn write(packet: Packet, writer: anytype) !void { - switch (packet) { - .flush => try writer.writeAll("0000"), - .delimiter => try writer.writeAll("0001"), - .response_end => try writer.writeAll("0002"), - .data => |data| { - assert(data.len <= max_data_length); - try writer.print("{x:0>4}", .{data.len + 4}); - try writer.writeAll(data); - }, - } - } - - /// Returns the normalized form of textual packet data, stripping any - /// trailing '\n'. - /// - /// As documented in - /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), - /// non-binary (textual) pkt-line data should contain a trailing '\n', but - /// is not required to do so (implementations must support both forms). - fn normalizeText(data: []const u8) []const u8 { - return if (mem.endsWith(u8, data, "\n")) - data[0 .. data.len - 1] - else - data; - } -}; - -/// A client session for the Git protocol, currently limited to an HTTP(S) -/// transport. Only protocol version 2 is supported, as documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -pub const Session = struct { - transport: *std.http.Client, - location: Location, - supports_agent: bool, - supports_shallow: bool, - object_format: Oid.Format, - allocator: Allocator, - - const agent = "zig/" ++ @import("builtin").zig_version_string; - const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); - - /// Initializes a client session and discovers the capabilities of the - /// server for optimal transport. - pub fn init( - allocator: Allocator, - transport: *std.http.Client, - uri: std.Uri, - http_headers_buffer: []u8, - ) !Session { - var session: Session = .{ - .transport = transport, - .location = try .init(allocator, uri), - .supports_agent = false, - .supports_shallow = false, - .object_format = .sha1, - .allocator = allocator, - }; - errdefer session.deinit(); - var capability_iterator = try session.getCapabilities(http_headers_buffer); - defer capability_iterator.deinit(); - while (try capability_iterator.next()) |capability| { - if (mem.eql(u8, capability.key, "agent")) { - session.supports_agent = true; - } else if (mem.eql(u8, capability.key, "fetch")) { - var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); - while (feature_iterator.next()) |feature| { - if (mem.eql(u8, feature, "shallow")) { - session.supports_shallow = true; - } - } - } else if (mem.eql(u8, capability.key, "object-format")) { - if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { - session.object_format = format; - } - } - } - return session; - } - - pub fn deinit(session: *Session) void { - session.location.deinit(session.allocator); - session.* = undefined; - } - - /// An owned `std.Uri` representing the location of the server (base URI). - const Location = struct { - uri: std.Uri, - - fn init(allocator: Allocator, uri: std.Uri) !Location { - const scheme = try allocator.dupe(u8, uri.scheme); - errdefer allocator.free(scheme); - const user = if (uri.user) |user| try std.fmt.allocPrint(allocator, "{user}", .{user}) else null; - errdefer if (user) |s| allocator.free(s); - const password = if (uri.password) |password| try std.fmt.allocPrint(allocator, "{password}", .{password}) else null; - errdefer if (password) |s| allocator.free(s); - const host = if (uri.host) |host| try std.fmt.allocPrint(allocator, "{host}", .{host}) else null; - errdefer if (host) |s| allocator.free(s); - const path = try std.fmt.allocPrint(allocator, "{path}", .{uri.path}); - errdefer allocator.free(path); - // The query and fragment are not used as part of the base server URI. - return .{ - .uri = .{ - .scheme = scheme, - .user = if (user) |s| .{ .percent_encoded = s } else null, - .password = if (password) |s| .{ .percent_encoded = s } else null, - .host = if (host) |s| .{ .percent_encoded = s } else null, - .port = uri.port, - .path = .{ .percent_encoded = path }, - }, - }; - } - - fn deinit(loc: *Location, allocator: Allocator) void { - allocator.free(loc.uri.scheme); - if (loc.uri.user) |user| allocator.free(user.percent_encoded); - if (loc.uri.password) |password| allocator.free(password.percent_encoded); - if (loc.uri.host) |host| allocator.free(host.percent_encoded); - allocator.free(loc.uri.path.percent_encoded); - } - }; - - /// Returns an iterator over capabilities supported by the server. - /// - /// The `session.location` is updated if the server returns a redirect, so - /// that subsequent session functions do not need to handle redirects. - fn getCapabilities(session: *Session, http_headers_buffer: []u8) !CapabilityIterator { - var info_refs_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); - defer session.allocator.free(session_uri_path); - info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "info/refs" }) }; - } - defer session.allocator.free(info_refs_uri.path.percent_encoded); - info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; - info_refs_uri.fragment = null; - - const max_redirects = 3; - var request = try session.transport.open(.GET, info_refs_uri, .{ - .redirect_behavior = @enumFromInt(max_redirects), - .server_header_buffer = http_headers_buffer, - .extra_headers = &.{ - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }); - errdefer request.deinit(); - try request.send(); - try request.finish(); - - try request.wait(); - if (request.response.status != .ok) return error.ProtocolError; - const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; - if (any_redirects_occurred) { - const request_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{request.uri.path}); - defer session.allocator.free(request_uri_path); - if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; - var new_uri = request.uri; - new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; - const new_location: Location = try .init(session.allocator, new_uri); - session.location.deinit(session.allocator); - session.location = new_location; - } - - const reader = request.reader(); - var buf: [Packet.max_data_length]u8 = undefined; - var state: enum { response_start, response_content } = .response_start; - while (true) { - // Some Git servers (at least GitHub) include an additional - // '# service=git-upload-pack' informative response before sending - // the expected 'version 2' packet and capability information. - // This is not universal: SourceHut, for example, does not do this. - // Thus, we need to skip any such useless additional responses - // before we get the one we're actually looking for. The responses - // will be delimited by flush packets. - const packet = Packet.read(reader, &buf) catch |e| switch (e) { - error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found - else => |other| return other, - }; - switch (packet) { - .flush => state = .response_start, - .data => |data| switch (state) { - .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { - return .{ .request = request }; - } else { - state = .response_content; - }, - else => {}, - }, - else => return error.UnexpectedPacket, - } - } - } - - const CapabilityIterator = struct { - request: std.http.Client.Request, - buf: [Packet.max_data_length]u8 = undefined, - - const Capability = struct { - key: []const u8, - value: ?[]const u8 = null, - - fn parse(data: []const u8) Capability { - return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| - .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } - else - .{ .key = data }; - } - }; - - fn deinit(iterator: *CapabilityIterator) void { - iterator.request.deinit(); - iterator.* = undefined; - } - - fn next(iterator: *CapabilityIterator) !?Capability { - switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { - .flush => return null, - .data => |data| return Capability.parse(Packet.normalizeText(data)), - else => return error.UnexpectedPacket, - } - } - }; - - const ListRefsOptions = struct { - /// The ref prefixes (if any) to use to filter the refs available on the - /// server. Note that the client must still check the returned refs - /// against its desired filters itself: the server is not required to - /// respect these prefix filters and may return other refs as well. - ref_prefixes: []const []const u8 = &.{}, - /// Whether to include symref targets for returned symbolic refs. - include_symrefs: bool = false, - /// Whether to include the peeled object ID for returned tag refs. - include_peeled: bool = false, - server_header_buffer: []u8, - }; - - /// Returns an iterator over refs known to the server. - pub fn listRefs(session: Session, options: ListRefsOptions) !RefIterator { - var upload_pack_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); - defer session.allocator.free(session_uri_path); - upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; - } - defer session.allocator.free(upload_pack_uri.path.percent_encoded); - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var body: std.ArrayListUnmanaged(u8) = .empty; - defer body.deinit(session.allocator); - const body_writer = body.writer(session.allocator); - try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, body_writer); - } - { - const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); - defer session.allocator.free(object_format_packet); - try Packet.write(.{ .data = object_format_packet }, body_writer); - } - try Packet.write(.delimiter, body_writer); - for (options.ref_prefixes) |ref_prefix| { - const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix}); - defer session.allocator.free(ref_prefix_packet); - try Packet.write(.{ .data = ref_prefix_packet }, body_writer); - } - if (options.include_symrefs) { - try Packet.write(.{ .data = "symrefs\n" }, body_writer); - } - if (options.include_peeled) { - try Packet.write(.{ .data = "peel\n" }, body_writer); - } - try Packet.write(.flush, body_writer); - - var request = try session.transport.open(.POST, upload_pack_uri, .{ - .redirect_behavior = .unhandled, - .server_header_buffer = options.server_header_buffer, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }); - errdefer request.deinit(); - request.transfer_encoding = .{ .content_length = body.items.len }; - try request.send(); - try request.writeAll(body.items); - try request.finish(); - - try request.wait(); - if (request.response.status != .ok) return error.ProtocolError; - - return .{ - .format = session.object_format, - .request = request, - }; - } - - pub const RefIterator = struct { - format: Oid.Format, - request: std.http.Client.Request, - buf: [Packet.max_data_length]u8 = undefined, - - pub const Ref = struct { - oid: Oid, - name: []const u8, - symref_target: ?[]const u8, - peeled: ?Oid, - }; - - pub fn deinit(iterator: *RefIterator) void { - iterator.request.deinit(); - iterator.* = undefined; - } - - pub fn next(iterator: *RefIterator) !?Ref { - switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { - .flush => return null, - .data => |data| { - const ref_data = Packet.normalizeText(data); - const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; - const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; - - const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; - const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; - - var symref_target: ?[]const u8 = null; - var peeled: ?Oid = null; - var last_sep_pos = name_sep_pos; - while (last_sep_pos < ref_data.len) { - const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; - const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; - if (mem.startsWith(u8, attribute, "symref-target:")) { - symref_target = attribute["symref-target:".len..]; - } else if (mem.startsWith(u8, attribute, "peeled:")) { - peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; - } - last_sep_pos = next_sep_pos; - } - - return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; - }, - else => return error.UnexpectedPacket, - } - } - }; - - /// Fetches the given refs from the server. A shallow fetch (depth 1) is - /// performed if the server supports it. - pub fn fetch( - session: Session, - wants: []const []const u8, - http_headers_buffer: []u8, - ) !FetchStream { - var upload_pack_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); - defer session.allocator.free(session_uri_path); - upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; - } - defer session.allocator.free(upload_pack_uri.path.percent_encoded); - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var body: std.ArrayListUnmanaged(u8) = .empty; - defer body.deinit(session.allocator); - const body_writer = body.writer(session.allocator); - try Packet.write(.{ .data = "command=fetch\n" }, body_writer); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, body_writer); - } - { - const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); - defer session.allocator.free(object_format_packet); - try Packet.write(.{ .data = object_format_packet }, body_writer); - } - try Packet.write(.delimiter, body_writer); - // Our packfile parser supports the OFS_DELTA object type - try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); - // We do not currently convey server progress information to the user - try Packet.write(.{ .data = "no-progress\n" }, body_writer); - if (session.supports_shallow) { - try Packet.write(.{ .data = "deepen 1\n" }, body_writer); - } - for (wants) |want| { - var buf: [Packet.max_data_length]u8 = undefined; - const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; - try Packet.write(.{ .data = arg }, body_writer); - } - try Packet.write(.{ .data = "done\n" }, body_writer); - try Packet.write(.flush, body_writer); - - var request = try session.transport.open(.POST, upload_pack_uri, .{ - .redirect_behavior = .not_allowed, - .server_header_buffer = http_headers_buffer, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }); - errdefer request.deinit(); - request.transfer_encoding = .{ .content_length = body.items.len }; - try request.send(); - try request.writeAll(body.items); - try request.finish(); - - try request.wait(); - if (request.response.status != .ok) return error.ProtocolError; - - const reader = request.reader(); - // We are not interested in any of the sections of the returned fetch - // data other than the packfile section, since we aren't doing anything - // complex like ref negotiation (this is a fresh clone). - var state: enum { section_start, section_content } = .section_start; - while (true) { - var buf: [Packet.max_data_length]u8 = undefined; - const packet = try Packet.read(reader, &buf); - switch (state) { - .section_start => switch (packet) { - .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { - return .{ .request = request }; - } else { - state = .section_content; - }, - else => return error.UnexpectedPacket, - }, - .section_content => switch (packet) { - .delimiter => state = .section_start, - .data => {}, - else => return error.UnexpectedPacket, - }, - } - } - } - - pub const FetchStream = struct { - request: std.http.Client.Request, - buf: [Packet.max_data_length]u8 = undefined, - pos: usize = 0, - len: usize = 0, - - pub fn deinit(stream: *FetchStream) void { - stream.request.deinit(); - } - - pub const ReadError = std.http.Client.Request.ReadError || error{ - InvalidPacket, - ProtocolError, - UnexpectedPacket, - }; - pub const Reader = std.io.Reader(*FetchStream, ReadError, read); - - const StreamCode = enum(u8) { - pack_data = 1, - progress = 2, - fatal_error = 3, - _, - }; - - pub fn reader(stream: *FetchStream) Reader { - return .{ .context = stream }; - } - - pub fn read(stream: *FetchStream, buf: []u8) !usize { - if (stream.pos == stream.len) { - while (true) { - switch (try Packet.read(stream.request.reader(), &stream.buf)) { - .flush => return 0, - .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { - .pack_data => { - stream.pos = 1; - stream.len = data.len; - break; - }, - .fatal_error => return error.ProtocolError, - else => {}, - }, - else => return error.UnexpectedPacket, - } - } - } - - const size = @min(buf.len, stream.len - stream.pos); - @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); - stream.pos += size; - return size; - } - }; -}; - -const PackHeader = struct { - total_objects: u32, - - const signature = "PACK"; - const supported_version = 2; - - fn read(reader: anytype) !PackHeader { - const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; - const version = reader.readInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - return .{ .total_objects = total_objects }; - } -}; - -const EntryHeader = union(Type) { - commit: Undeltified, - tree: Undeltified, - blob: Undeltified, - tag: Undeltified, - ofs_delta: OfsDelta, - ref_delta: RefDelta, - - const Type = enum(u3) { - commit = 1, - tree = 2, - blob = 3, - tag = 4, - ofs_delta = 6, - ref_delta = 7, - }; - - const Undeltified = struct { - uncompressed_length: u64, - }; - - const OfsDelta = struct { - offset: u64, - uncompressed_length: u64, - }; - - const RefDelta = struct { - base_object: Oid, - uncompressed_length: u64, - }; - - fn objectType(header: EntryHeader) Object.Type { - return switch (header) { - inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), - else => unreachable, - }; - } - - fn uncompressedLength(header: EntryHeader) u64 { - return switch (header) { - inline else => |entry| entry.uncompressed_length, - }; - } - - fn read(format: Oid.Format, reader: anytype) !EntryHeader { - const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }); - const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; - var uncompressed_length: u64 = initial.len; - uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; - const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; - return switch (@"type") { - inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ - .uncompressed_length = uncompressed_length, - }), - .ofs_delta => .{ .ofs_delta = .{ - .offset = try readOffsetVarInt(reader), - .uncompressed_length = uncompressed_length, - } }, - .ref_delta => .{ .ref_delta = .{ - .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }, - .uncompressed_length = uncompressed_length, - } }, - }; - } -}; - -fn readSizeVarInt(r: anytype) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); - var value: u64 = b.value; - var shift: u6 = 0; - while (b.has_next) { - b = @bitCast(try r.readByte()); - shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; - value |= @as(u64, b.value) << shift; - } - return value; -} - -fn readOffsetVarInt(r: anytype) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); - var value: u64 = b.value; - while (b.has_next) { - b = @bitCast(try r.readByte()); - value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; - value |= b.value; - } - return value; -} - -const IndexHeader = struct { - fan_out_table: [256]u32, - - const signature = "\xFFtOc"; - const supported_version = 2; - const size = 4 + 4 + @sizeOf([256]u32); - - fn read(reader: anytype) !IndexHeader { - var header_bytes = try reader.readBytesNoEof(size); - if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; - const version = mem.readInt(u32, header_bytes[4..8], .big); - if (version != supported_version) return error.UnsupportedVersion; - - var fan_out_table: [256]u32 = undefined; - var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); - const fan_out_table_reader = fan_out_table_stream.reader(); - for (&fan_out_table) |*entry| { - entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; - } - return .{ .fan_out_table = fan_out_table }; - } -}; - -const IndexEntry = struct { - offset: u64, - crc32: u32, -}; - -/// Writes out a version 2 index for the given packfile, as documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { - try pack.seekTo(0); - - var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; - defer index_entries.deinit(allocator); - var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty; - defer pending_deltas.deinit(allocator); - - const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); - - var cache: ObjectCache = .{}; - defer cache.deinit(allocator); - var remaining_deltas = pending_deltas.items.len; - while (remaining_deltas > 0) { - var i: usize = remaining_deltas; - while (i > 0) { - i -= 1; - const delta = pending_deltas.items[i]; - if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { - try index_entries.put(allocator, oid, delta); - _ = pending_deltas.swapRemove(i); - } - } - if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; - remaining_deltas = pending_deltas.items.len; - } - - var oids: std.ArrayListUnmanaged(Oid) = .empty; - defer oids.deinit(allocator); - try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); - var index_entries_iter = index_entries.iterator(); - while (index_entries_iter.next()) |entry| { - oids.appendAssumeCapacity(entry.key_ptr.*); - } - mem.sortUnstable(Oid, oids.items, {}, struct { - fn lessThan(_: void, o1: Oid, o2: Oid) bool { - return mem.lessThan(u8, o1.slice(), o2.slice()); - } - }.lessThan); - - var fan_out_table: [256]u32 = undefined; - var count: u32 = 0; - var fan_out_index: u8 = 0; - for (oids.items) |oid| { - const key = oid.slice()[0]; - if (key > fan_out_index) { - @memset(fan_out_table[fan_out_index..key], count); - fan_out_index = key; - } - count += 1; - } - @memset(fan_out_table[fan_out_index..], count); - - var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); - const writer = index_hashed_writer.writer(); - try writer.writeAll(IndexHeader.signature); - try writer.writeInt(u32, IndexHeader.supported_version, .big); - for (fan_out_table) |fan_out_entry| { - try writer.writeInt(u32, fan_out_entry, .big); - } - - for (oids.items) |oid| { - try writer.writeAll(oid.slice()); - } - - for (oids.items) |oid| { - try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); - } - - var big_offsets: std.ArrayListUnmanaged(u64) = .empty; - defer big_offsets.deinit(allocator); - for (oids.items) |oid| { - const offset = index_entries.get(oid).?.offset; - if (offset <= std.math.maxInt(u31)) { - try writer.writeInt(u32, @intCast(offset), .big); - } else { - const index = big_offsets.items.len; - try big_offsets.append(allocator, offset); - try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); - } - } - for (big_offsets.items) |offset| { - try writer.writeInt(u64, offset, .big); - } - - try writer.writeAll(pack_checksum.slice()); - const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.writeAll(index_checksum.slice()); -} - -/// Performs the first pass over the packfile data for index construction. -/// This will index all non-delta objects, queue delta objects for further -/// processing, and return the pack checksum (which is part of the index -/// format). -fn indexPackFirstPass( - allocator: Allocator, - format: Oid.Format, - pack: std.fs.File, - index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), - pending_deltas: *std.ArrayListUnmanaged(IndexEntry), -) !Oid { - var pack_buffered_reader = std.io.bufferedReader(pack.reader()); - var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); - var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); - const pack_reader = pack_hashed_reader.reader(); - - const pack_header = try PackHeader.read(pack_reader); - - var current_entry: u32 = 0; - while (current_entry < pack_header.total_objects) : (current_entry += 1) { - const entry_offset = pack_counting_reader.bytes_read; - var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); - const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); - switch (entry_header) { - .commit, .tree, .blob, .tag => |object| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); - const entry_writer = entry_hashed_writer.writer(); - // The object header is not included in the pack data but is - // part of the object's ID - try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), entry_writer); - if (entry_counting_reader.bytes_read != object.uncompressed_length) { - return error.InvalidObject; - } - const oid = entry_hashed_writer.hasher.finalResult(); - try index_entries.put(allocator, oid, .{ - .offset = entry_offset, - .crc32 = entry_crc32_reader.hasher.final(), - }); - }, - inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); - if (entry_counting_reader.bytes_read != delta.uncompressed_length) { - return error.InvalidObject; - } - try pending_deltas.append(allocator, .{ - .offset = entry_offset, - .crc32 = entry_crc32_reader.hasher.final(), - }); - }, - } - } - - const pack_checksum = pack_hashed_reader.hasher.finalResult(); - const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); - if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { - return error.CorruptedPack; - } - _ = pack_reader.readByte() catch |e| switch (e) { - error.EndOfStream => return pack_checksum, - else => |other| return other, - }; - return error.InvalidFormat; -} - -/// Attempts to determine the final object ID of the given deltified object. -/// May return null if this is not yet possible (if the delta is a ref-based -/// delta and we do not yet know the offset of the base object). -fn indexPackHashDelta( - allocator: Allocator, - format: Oid.Format, - pack: std.fs.File, - delta: IndexEntry, - index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), - cache: *ObjectCache, -) !?Oid { - // Figure out the chain of deltas to resolve - var base_offset = delta.offset; - var base_header: EntryHeader = undefined; - var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; - defer delta_offsets.deinit(allocator); - const base_object = while (true) { - if (cache.get(base_offset)) |base_object| break base_object; - - try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, pack.reader()); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; - }, - else => { - const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); - errdefer allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try cache.put(allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); - - var entry_hasher: Oid.Hasher = .init(format); - var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); - try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); - entry_hasher.update(base_data); - return entry_hasher.finalResult(); -} - -/// Resolves a chain of deltas, returning the final base object data. `pack` is -/// assumed to be looking at the start of the object data for the base object of -/// the chain, and will then apply the deltas in `delta_offsets` in reverse order -/// to obtain the final object. -fn resolveDeltaChain( - allocator: Allocator, - format: Oid.Format, - pack: std.fs.File, - base_object: Object, - delta_offsets: []const u64, - cache: *ObjectCache, -) ![]const u8 { - var base_data = base_object.data; - var i: usize = delta_offsets.len; - while (i > 0) { - i -= 1; - - const delta_offset = delta_offsets[i]; - try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, pack.reader()); - const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); - defer allocator.free(delta_data); - var delta_stream = std.io.fixedBufferStream(delta_data); - const delta_reader = delta_stream.reader(); - _ = try readSizeVarInt(delta_reader); // base object size - const expanded_size = try readSizeVarInt(delta_reader); - - const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; - const expanded_data = try allocator.alloc(u8, expanded_alloc_size); - errdefer allocator.free(expanded_data); - var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); - var base_stream = std.io.fixedBufferStream(base_data); - try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); - if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; - - try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); - base_data = expanded_data; - } - return base_data; -} - -/// Reads the complete contents of an object from `reader`. This function may -/// read more bytes than required from `reader`, so the reader position after -/// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { - const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); - const data = try allocator.alloc(u8, alloc_size); - errdefer allocator.free(data); - try decompress_stream.reader().readNoEof(data); - _ = decompress_stream.reader().readByte() catch |e| switch (e) { - error.EndOfStream => return data, - else => |other| return other, - }; - return error.InvalidFormat; -} - -/// Expands delta data from `delta_reader` to `writer`. `base_object` must -/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). -/// -/// The format of the delta data is documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { - while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { - error.EndOfStream => return, - else => |other| return other, - }); - if (inst.copy) { - const available: packed struct { - offset1: bool, - offset2: bool, - offset3: bool, - offset4: bool, - size1: bool, - size2: bool, - size3: bool, - } = @bitCast(inst.value); - const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.readByte() else 0, - .offset2 = if (available.offset2) try delta_reader.readByte() else 0, - .offset3 = if (available.offset3) try delta_reader.readByte() else 0, - .offset4 = if (available.offset4) try delta_reader.readByte() else 0, - }; - const offset: u32 = @bitCast(offset_parts); - const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.readByte() else 0, - .size2 = if (available.size2) try delta_reader.readByte() else 0, - .size3 = if (available.size3) try delta_reader.readByte() else 0, - }; - var size: u24 = @bitCast(size_parts); - if (size == 0) size = 0x10000; - try base_object.seekTo(offset); - var copy_reader = std.io.limitedReader(base_object.reader(), size); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(copy_reader.reader(), writer); - } else if (inst.value != 0) { - var data_reader = std.io.limitedReader(delta_reader, inst.value); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(data_reader.reader(), writer); - } else { - return error.InvalidDeltaInstruction; - } - } -} - -/// Runs the packfile indexing and checkout test. -/// -/// The two testrepo repositories under testdata contain identical commit -/// histories and contents. -/// -/// To verify the contents of the packfiles using Git alone, run the -/// following commands in an empty directory: -/// -/// 1. `git init --object-format=(sha1|sha256)` -/// 2. `git unpack-objects assert(p.errors.items.len > 0), - else => |e| return e, - }; - - return .{ - .name = p.name, - .id = p.id, - .version = p.version, - .version_node = p.version_node, - .dependencies = try p.dependencies.clone(p.arena), - .dependencies_node = p.dependencies_node, - .paths = try p.paths.clone(p.arena), - .minimum_zig_version = p.minimum_zig_version, - .errors = try p.arena.dupe(ErrorMessage, p.errors.items), - .arena_state = arena_instance.state, - }; -} - -pub fn deinit(man: *Manifest, gpa: Allocator) void { - man.arena_state.promote(gpa).deinit(); - man.* = undefined; -} - -pub fn copyErrorsIntoBundle( - man: Manifest, - ast: Ast, - /// ErrorBundle null-terminated string index - src_path: u32, - eb: *std.zig.ErrorBundle.Wip, -) Allocator.Error!void { - for (man.errors) |msg| { - const start_loc = ast.tokenLocation(0, msg.tok); - - try eb.addRootErrorMessage(.{ - .msg = try eb.addString(msg.msg), - .src_loc = try eb.addSourceLocation(.{ - .src_path = src_path, - .span_start = ast.tokenStart(msg.tok), - .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), - .span_main = ast.tokenStart(msg.tok) + msg.off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - }); - } -} - -const Parse = struct { - gpa: Allocator, - ast: Ast, - arena: Allocator, - buf: std.ArrayListUnmanaged(u8), - errors: std.ArrayListUnmanaged(ErrorMessage), - - name: []const u8, - id: u32, - version: std.SemanticVersion, - version_node: Ast.Node.Index, - dependencies: std.StringArrayHashMapUnmanaged(Dependency), - dependencies_node: Ast.Node.OptionalIndex, - paths: std.StringArrayHashMapUnmanaged(void), - allow_missing_paths_field: bool, - allow_name_string: bool, - allow_missing_fingerprint: bool, - minimum_zig_version: ?std.SemanticVersion, - - const InnerError = error{ ParseFailure, OutOfMemory }; - - fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - return fail(p, main_token, "expected top level expression to be a struct", .{}); - }; - - var have_name = false; - var have_version = false; - var have_included_paths = false; - var fingerprint: ?Package.Fingerprint = null; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "dependencies")) { - p.dependencies_node = field_init.toOptional(); - try parseDependencies(p, field_init); - } else if (mem.eql(u8, field_name, "paths")) { - have_included_paths = true; - try parseIncludedPaths(p, field_init); - } else if (mem.eql(u8, field_name, "name")) { - p.name = try parseName(p, field_init); - have_name = true; - } else if (mem.eql(u8, field_name, "fingerprint")) { - fingerprint = try parseFingerprint(p, field_init); - } else if (mem.eql(u8, field_name, "version")) { - p.version_node = field_init; - const version_text = try parseString(p, field_init); - if (version_text.len > max_version_len) { - try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); - } - p.version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v undefined; - }; - have_version = true; - } else if (mem.eql(u8, field_name, "minimum_zig_version")) { - const version_text = try parseString(p, field_init); - p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v null; - }; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!have_name) { - try appendError(p, main_token, "missing top-level 'name' field", .{}); - } else { - if (fingerprint) |n| { - if (!n.validate(p.name)) { - return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ - n.int(), Package.Fingerprint.generate(p.name).int(), - }); - } - p.id = n.id; - } else if (!p.allow_missing_fingerprint) { - try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ - Package.Fingerprint.generate(p.name).int(), - }); - } else { - p.id = 0; - } - } - - if (!have_version) { - try appendError(p, main_token, "missing top-level 'version' field", .{}); - } - - if (!have_included_paths) { - if (p.allow_missing_paths_field) { - try p.paths.put(p.gpa, "", {}); - } else { - try appendError(p, main_token, "missing top-level 'paths' field", .{}); - } - } - } - - fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected dependencies expression to be a struct", .{}); - }; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const dep_name = try identifierTokenString(p, name_token); - const dep = try parseDependency(p, field_init); - try p.dependencies.put(p.gpa, dep_name, dep); - } - } - - fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected dependency expression to be a struct", .{}); - }; - - var dep: Dependency = .{ - .location = undefined, - .location_tok = undefined, - .location_node = undefined, - .hash = null, - .hash_tok = .none, - .hash_node = .none, - .node = node, - .name_tok = undefined, - .lazy = false, - }; - var has_location = false; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - dep.name_tok = name_token; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "url")) { - if (has_location) { - return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .url = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = ast.nodeMainToken(field_init); - dep.location_node = field_init; - } else if (mem.eql(u8, field_name, "path")) { - if (has_location) { - return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .path = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = ast.nodeMainToken(field_init); - dep.location_node = field_init; - } else if (mem.eql(u8, field_name, "hash")) { - dep.hash = parseHash(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); - dep.hash_node = field_init.toOptional(); - } else if (mem.eql(u8, field_name, "lazy")) { - dep.lazy = parseBool(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!has_location) { - try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); - } - - return dep; - } - - fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const array_init = ast.fullArrayInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected paths expression to be a list of strings", .{}); - }; - - for (array_init.ast.elements) |elem_node| { - const path_string = try parseString(p, elem_node); - // This is normalized so that it can be used in string comparisons - // against file system paths. - const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); - try p.paths.put(p.gpa, normalized, {}); - } - } - - fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { - const ast = p.ast; - if (ast.nodeTag(node) != .identifier) { - return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); - } - const ident_token = ast.nodeMainToken(node); - const token_bytes = ast.tokenSlice(ident_token); - if (mem.eql(u8, token_bytes, "true")) { - return true; - } else if (mem.eql(u8, token_bytes, "false")) { - return false; - } else { - return fail(p, ident_token, "expected boolean", .{}); - } - } - - fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - if (ast.nodeTag(node) != .number_literal) { - return fail(p, main_token, "expected integer literal", .{}); - } - const token_bytes = ast.tokenSlice(main_token); - const parsed = std.zig.parseNumberLiteral(token_bytes); - switch (parsed) { - .int => |n| return @bitCast(n), - .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ - @tagName(parsed), - }), - .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), - } - } - - fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - - if (p.allow_name_string and ast.nodeTag(node) == .string_literal) { - const name = try parseString(p, node); - if (!std.zig.isValidId(name)) - return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{}); - - if (name.len > max_name_len) - return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ - std.zig.fmtId(name), max_name_len, - }); - - return name; - } - - if (ast.nodeTag(node) != .enum_literal) - return fail(p, main_token, "expected enum literal", .{}); - - const ident_name = ast.tokenSlice(main_token); - if (mem.startsWith(u8, ident_name, "@")) - return fail(p, main_token, "name must be a valid bare zig identifier", .{}); - - if (ident_name.len > max_name_len) - return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ - std.zig.fmtId(ident_name), max_name_len, - }); - - return ident_name; - } - - fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - if (ast.nodeTag(node) != .string_literal) { - return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); - } - const str_lit_token = ast.nodeMainToken(node); - const token_bytes = ast.tokenSlice(str_lit_token); - p.buf.clearRetainingCapacity(); - try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const tok = ast.nodeMainToken(node); - const h = try parseString(p, node); - - if (h.len > Package.Hash.max_len) { - return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len}); - } - - return h; - } - - /// TODO: try to DRY this with AstGen.identifierTokenString - fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { - const ast = p.ast; - assert(ast.tokenTag(token) == .identifier); - const ident_name = ast.tokenSlice(token); - if (!mem.startsWith(u8, ident_name, "@")) { - return ident_name; - } - p.buf.clearRetainingCapacity(); - try parseStrLit(p, token, &p.buf, ident_name, 1); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - /// TODO: try to DRY this with AstGen.parseStrLit - fn parseStrLit( - p: *Parse, - token: Ast.TokenIndex, - buf: *std.ArrayListUnmanaged(u8), - bytes: []const u8, - offset: u32, - ) InnerError!void { - const raw_string = bytes[offset..]; - var buf_managed = buf.toManaged(p.gpa); - const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); - buf.* = buf_managed.moveToUnmanaged(); - switch (try result) { - .success => {}, - .failure => |err| try p.appendStrLitError(err, token, bytes, offset), - } - } - - /// TODO: try to DRY this with AstGen.failWithStrLitError - fn appendStrLitError( - p: *Parse, - err: std.zig.string_literal.Error, - token: Ast.TokenIndex, - bytes: []const u8, - offset: u32, - ) Allocator.Error!void { - const raw_string = bytes[offset..]; - switch (err) { - .invalid_escape_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid escape character: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .expected_hex_digit => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit, found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_unicode_escape_sequence => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "empty unicode escape sequence", - .{}, - ); - }, - .expected_hex_digit_or_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit or '}}', found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .invalid_unicode_codepoint => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "unicode escape does not correspond to a valid unicode scalar value", - .{}, - ); - }, - .expected_lbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '{{', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '}}', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_single_quote => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected single quote ('), found '{c}", - .{raw_string[bad_index]}, - ); - }, - .invalid_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid byte in string or character literal: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_char_literal => { - try p.appendErrorOff(token, offset, "empty character literal", .{}); - }, - } - } - - fn fail( - p: *Parse, - tok: Ast.TokenIndex, - comptime fmt: []const u8, - args: anytype, - ) InnerError { - try appendError(p, tok, fmt, args); - return error.ParseFailure; - } - - fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { - return appendErrorOff(p, tok, 0, fmt, args); - } - - fn appendErrorOff( - p: *Parse, - tok: Ast.TokenIndex, - byte_offset: u32, - comptime fmt: []const u8, - args: anytype, - ) Allocator.Error!void { - try p.errors.append(p.gpa, .{ - .msg = try std.fmt.allocPrint(p.arena, fmt, args), - .tok = tok, - .off = byte_offset, - }); - } -}; - -test "basic" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = "foo", - \\ .version = "3.2.1", - \\ .paths = .{""}, - \\ .dependencies = .{ - \\ .bar = .{ - \\ .url = "https://example.com/baz.tar.gz", - \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", - \\ }, - \\ }, - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var manifest = try Manifest.parse(gpa, ast, .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 0); - try testing.expectEqualStrings("foo", manifest.name); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 3, - .minor = 2, - .patch = 1, - }), manifest.version); - - try testing.expect(manifest.dependencies.count() == 1); - try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); - try testing.expectEqualStrings( - "https://example.com/baz.tar.gz", - manifest.dependencies.values()[0].location.url, - ); - try testing.expectEqualStrings( - "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", - manifest.dependencies.values()[0].hash orelse return error.TestFailed, - ); - - try testing.expect(manifest.minimum_zig_version == null); -} - -test "minimum_zig_version" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = "foo", - \\ .version = "3.2.1", - \\ .paths = .{""}, - \\ .minimum_zig_version = "0.11.1", - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var manifest = try Manifest.parse(gpa, ast, .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 0); - try testing.expect(manifest.dependencies.count() == 0); - - try testing.expect(manifest.minimum_zig_version != null); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 0, - .minor = 11, - .patch = 1, - }), manifest.minimum_zig_version.?); -} - -test "minimum_zig_version - invalid version" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = "foo", - \\ .version = "3.2.1", - \\ .minimum_zig_version = "X.11.1", - \\ .paths = .{""}, - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var manifest = try Manifest.parse(gpa, ast, .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 1); - try testing.expect(manifest.dependencies.count() == 0); - - try testing.expect(manifest.minimum_zig_version == null); -} diff --git a/src/Package/Module.zig b/src/Package/Module.zig deleted file mode 100644 index 0dec7bde76e5f55ec0be6bb34ae465eeac0e94d4..0000000000000000000000000000000000000000 --- a/src/Package/Module.zig +++ /dev/null @@ -1,563 +0,0 @@ -//! Corresponds to something that Zig source code can `@import`. - -/// Only files inside this directory can be imported. -root: Cache.Path, -/// Relative to `root`. May contain path separators. -root_src_path: []const u8, -/// Name used in compile errors. Looks like "root.foo.bar". -fully_qualified_name: []const u8, -/// The dependency table of this module. Shared dependencies such as 'std', -/// 'builtin', and 'root' are not specified in every dependency table, but -/// instead only in the table of `main_mod`. `Module.importFile` is -/// responsible for detecting these names and using the correct package. -deps: Deps = .{}, - -resolved_target: ResolvedTarget, -optimize_mode: std.builtin.OptimizeMode, -code_model: std.builtin.CodeModel, -single_threaded: bool, -error_tracing: bool, -valgrind: bool, -pic: bool, -strip: bool, -omit_frame_pointer: bool, -stack_check: bool, -stack_protector: u32, -red_zone: bool, -sanitize_c: bool, -sanitize_thread: bool, -fuzz: bool, -unwind_tables: std.builtin.UnwindTables, -cc_argv: []const []const u8, -/// (SPIR-V) whether to generate a structured control flow graph or not -structured_cfg: bool, -no_builtin: bool, - -/// If the module is an `@import("builtin")` module, this is the `File` that -/// is preallocated for it. Otherwise this field is null. -builtin_file: ?*File, - -pub const Deps = std.StringArrayHashMapUnmanaged(*Module); - -pub fn isBuiltin(m: Module) bool { - return m.builtin_file != null; -} - -pub const Tree = struct { - /// Each `Package` exposes a `Module` with build.zig as its root source file. - build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), -}; - -pub const CreateOptions = struct { - /// Where to store builtin.zig. The global cache directory is used because - /// it is a pure function based on CLI flags. - global_cache_directory: Cache.Directory, - paths: Paths, - fully_qualified_name: []const u8, - - cc_argv: []const []const u8, - inherited: Inherited, - global: Compilation.Config, - /// If this is null then `resolved_target` must be non-null. - parent: ?*Package.Module, - - builtin_mod: ?*Package.Module, - - /// Allocated into the given `arena`. Should be shared across all module creations in a Compilation. - /// Ignored if `builtin_mod` is passed or if `!have_zcu`. - /// Otherwise, may be `null` only if this Compilation consists of a single module. - builtin_modules: ?*std.StringHashMapUnmanaged(*Module), - - pub const Paths = struct { - root: Cache.Path, - /// Relative to `root`. May contain path separators. - root_src_path: []const u8, - }; - - pub const Inherited = struct { - /// If this is null then `parent` must be non-null. - resolved_target: ?ResolvedTarget = null, - optimize_mode: ?std.builtin.OptimizeMode = null, - code_model: ?std.builtin.CodeModel = null, - single_threaded: ?bool = null, - error_tracing: ?bool = null, - valgrind: ?bool = null, - pic: ?bool = null, - strip: ?bool = null, - omit_frame_pointer: ?bool = null, - stack_check: ?bool = null, - /// null means default. - /// 0 means no stack protector. - /// other number means stack protection with that buffer size. - stack_protector: ?u32 = null, - red_zone: ?bool = null, - unwind_tables: ?std.builtin.UnwindTables = null, - sanitize_c: ?bool = null, - sanitize_thread: ?bool = null, - fuzz: ?bool = null, - structured_cfg: ?bool = null, - no_builtin: ?bool = null, - }; -}; - -pub const ResolvedTarget = struct { - result: std.Target, - is_native_os: bool, - is_native_abi: bool, - llvm_cpu_features: ?[*:0]const u8 = null, -}; - -/// At least one of `parent` and `resolved_target` must be non-null. -pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { - if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); - if (options.inherited.fuzz == true) assert(options.global.any_fuzz); - if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); - if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); - if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); - - const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; - const target = resolved_target.result; - - const optimize_mode = options.inherited.optimize_mode orelse - if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; - - const strip = b: { - if (options.inherited.strip) |x| break :b x; - if (options.parent) |p| break :b p.strip; - break :b options.global.root_strip; - }; - - const valgrind = b: { - if (!target_util.hasValgrindSupport(target)) { - if (options.inherited.valgrind == true) - return error.ValgrindUnsupportedOnTarget; - break :b false; - } - if (options.inherited.valgrind) |x| break :b x; - if (options.parent) |p| break :b p.valgrind; - if (strip) break :b false; - break :b optimize_mode == .Debug; - }; - - const zig_backend = target_util.zigBackend(target, options.global.use_llvm); - - const single_threaded = b: { - if (target_util.alwaysSingleThreaded(target)) { - if (options.inherited.single_threaded == false) - return error.TargetRequiresSingleThreaded; - break :b true; - } - - if (options.global.have_zcu) { - if (!target_util.supportsThreads(target, zig_backend)) { - if (options.inherited.single_threaded == false) - return error.BackendRequiresSingleThreaded; - break :b true; - } - } - - if (options.inherited.single_threaded) |x| break :b x; - if (options.parent) |p| break :b p.single_threaded; - break :b target_util.defaultSingleThreaded(target); - }; - - const error_tracing = b: { - if (options.inherited.error_tracing) |x| break :b x; - if (options.parent) |p| break :b p.error_tracing; - break :b options.global.root_error_tracing; - }; - - const pic = b: { - if (target_util.requiresPIC(target, options.global.link_libc)) { - if (options.inherited.pic == false) - return error.TargetRequiresPic; - break :b true; - } - if (options.global.pie) { - if (options.inherited.pic == false) - return error.PieRequiresPic; - break :b true; - } - if (options.global.link_mode == .dynamic) { - if (options.inherited.pic == false) - return error.DynamicLinkingRequiresPic; - break :b true; - } - if (options.inherited.pic) |x| break :b x; - if (options.parent) |p| break :b p.pic; - break :b false; - }; - - const red_zone = b: { - if (!target_util.hasRedZone(target)) { - if (options.inherited.red_zone == true) - return error.TargetHasNoRedZone; - break :b false; - } - if (options.inherited.red_zone) |x| break :b x; - if (options.parent) |p| break :b p.red_zone; - break :b true; - }; - - const omit_frame_pointer = b: { - if (options.inherited.omit_frame_pointer) |x| break :b x; - if (options.parent) |p| break :b p.omit_frame_pointer; - if (optimize_mode == .ReleaseSmall) { - // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. - // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) - // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). - break :b !target.cpu.arch.isX86(); - } - break :b false; - }; - - const sanitize_thread = b: { - if (options.inherited.sanitize_thread) |x| break :b x; - if (options.parent) |p| break :b p.sanitize_thread; - break :b false; - }; - - const unwind_tables = b: { - if (options.inherited.unwind_tables) |x| break :b x; - if (options.parent) |p| break :b p.unwind_tables; - - break :b target_util.defaultUnwindTables( - target, - options.global.link_libunwind, - sanitize_thread or options.global.any_sanitize_thread, - ); - }; - - const fuzz = b: { - if (options.inherited.fuzz) |x| break :b x; - if (options.parent) |p| break :b p.fuzz; - break :b false; - }; - - const code_model = b: { - if (options.inherited.code_model) |x| break :b x; - if (options.parent) |p| break :b p.code_model; - break :b .default; - }; - - const is_safe_mode = switch (optimize_mode) { - .Debug, .ReleaseSafe => true, - .ReleaseFast, .ReleaseSmall => false, - }; - - const sanitize_c = b: { - if (options.inherited.sanitize_c) |x| break :b x; - if (options.parent) |p| break :b p.sanitize_c; - break :b is_safe_mode; - }; - - const stack_check = b: { - if (!target_util.supportsStackProbing(target)) { - if (options.inherited.stack_check == true) - return error.StackCheckUnsupportedByTarget; - break :b false; - } - if (options.inherited.stack_check) |x| break :b x; - if (options.parent) |p| break :b p.stack_check; - break :b is_safe_mode; - }; - - const stack_protector: u32 = sp: { - const use_zig_backend = options.global.have_zcu or - (options.global.any_c_source_files and options.global.c_frontend == .aro); - if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnsupportedByTarget; - } - break :sp 0; - } - - if (options.global.any_c_source_files and options.global.c_frontend == .clang and - !target_util.clangSupportsStackProtector(target)) - { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnsupportedByTarget; - } - break :sp 0; - } - - // This logic is checking for linking libc because otherwise our start code - // which is trying to set up TLS (i.e. the fs/gs registers) but the stack - // protection code depends on fs/gs registers being already set up. - // If we were able to annotate start code, or perhaps the entire std lib, - // as being exempt from stack protection checks, we could change this logic - // to supporting stack protection even when not linking libc. - // TODO file issue about this - if (!options.global.link_libc) { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnavailableWithoutLibC; - } - break :sp 0; - } - - if (options.inherited.stack_protector) |x| break :sp x; - if (options.parent) |p| break :sp p.stack_protector; - if (!is_safe_mode) break :sp 0; - - break :sp target_util.default_stack_protector_buffer_size; - }; - - const structured_cfg = b: { - if (options.inherited.structured_cfg) |x| break :b x; - if (options.parent) |p| break :b p.structured_cfg; - // We always want a structured control flow in shaders. This option is - // only relevant for OpenCL kernels. - break :b switch (target.os.tag) { - .opencl => false, - else => true, - }; - }; - - const no_builtin = b: { - if (options.inherited.no_builtin) |x| break :b x; - if (options.parent) |p| break :b p.no_builtin; - - break :b target.cpu.arch.isBpf(); - }; - - const llvm_cpu_features: ?[*:0]const u8 = b: { - if (resolved_target.llvm_cpu_features) |x| break :b x; - if (!options.global.use_llvm) break :b null; - - var buf = std.ArrayList(u8).init(arena); - var disabled_features = std.ArrayList(u8).init(arena); - defer disabled_features.deinit(); - - // Append disabled features after enabled ones, so that their effects aren't overwritten. - for (target.cpu.arch.allFeaturesList()) |feature| { - if (feature.llvm_name) |llvm_name| { - const is_enabled = target.cpu.features.isEnabled(feature.index); - - if (is_enabled) { - try buf.ensureUnusedCapacity(2 + llvm_name.len); - buf.appendAssumeCapacity('+'); - buf.appendSliceAssumeCapacity(llvm_name); - buf.appendAssumeCapacity(','); - } else { - try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); - disabled_features.appendAssumeCapacity('-'); - disabled_features.appendSliceAssumeCapacity(llvm_name); - disabled_features.appendAssumeCapacity(','); - } - } - } - - try buf.appendSlice(disabled_features.items); - if (buf.items.len == 0) break :b ""; - assert(std.mem.endsWith(u8, buf.items, ",")); - buf.items[buf.items.len - 1] = 0; - buf.shrinkAndFree(buf.items.len); - break :b buf.items[0 .. buf.items.len - 1 :0].ptr; - }; - - const mod = try arena.create(Module); - mod.* = .{ - .root = options.paths.root, - .root_src_path = options.paths.root_src_path, - .fully_qualified_name = options.fully_qualified_name, - .resolved_target = .{ - .result = target, - .is_native_os = resolved_target.is_native_os, - .is_native_abi = resolved_target.is_native_abi, - .llvm_cpu_features = llvm_cpu_features, - }, - .optimize_mode = optimize_mode, - .single_threaded = single_threaded, - .error_tracing = error_tracing, - .valgrind = valgrind, - .pic = pic, - .strip = strip, - .omit_frame_pointer = omit_frame_pointer, - .stack_check = stack_check, - .stack_protector = stack_protector, - .code_model = code_model, - .red_zone = red_zone, - .sanitize_c = sanitize_c, - .sanitize_thread = sanitize_thread, - .fuzz = fuzz, - .unwind_tables = unwind_tables, - .cc_argv = options.cc_argv, - .structured_cfg = structured_cfg, - .no_builtin = no_builtin, - .builtin_file = null, - }; - - const opt_builtin_mod = options.builtin_mod orelse b: { - if (!options.global.have_zcu) break :b null; - - const generated_builtin_source = try Builtin.generate(.{ - .target = target, - .zig_backend = zig_backend, - .output_mode = options.global.output_mode, - .link_mode = options.global.link_mode, - .unwind_tables = unwind_tables, - .is_test = options.global.is_test, - .single_threaded = single_threaded, - .link_libc = options.global.link_libc, - .link_libcpp = options.global.link_libcpp, - .optimize_mode = optimize_mode, - .error_tracing = error_tracing, - .valgrind = valgrind, - .sanitize_thread = sanitize_thread, - .fuzz = fuzz, - .pic = pic, - .pie = options.global.pie, - .strip = strip, - .code_model = code_model, - .omit_frame_pointer = omit_frame_pointer, - .wasi_exec_model = options.global.wasi_exec_model, - }, arena); - - const new = if (options.builtin_modules) |builtins| new: { - const gop = try builtins.getOrPut(arena, generated_builtin_source); - if (gop.found_existing) break :b gop.value_ptr.*; - errdefer builtins.removeByPtr(gop.key_ptr); - const new = try arena.create(Module); - gop.value_ptr.* = new; - break :new new; - } else try arena.create(Module); - errdefer if (options.builtin_modules) |builtins| assert(builtins.remove(generated_builtin_source)); - - const new_file = try arena.create(File); - - const hex_digest = digest: { - var hasher: Cache.Hasher = Cache.hasher_init; - hasher.update(generated_builtin_source); - - var bin_digest: Cache.BinDigest = undefined; - hasher.final(&bin_digest); - - var hex_digest: Cache.HexDigest = undefined; - _ = std.fmt.bufPrint( - &hex_digest, - "{s}", - .{std.fmt.fmtSliceHexLower(&bin_digest)}, - ) catch unreachable; - - break :digest hex_digest; - }; - - const builtin_sub_path = try arena.dupe(u8, "b" ++ std.fs.path.sep_str ++ hex_digest); - - new.* = .{ - .root = .{ - .root_dir = options.global_cache_directory, - .sub_path = builtin_sub_path, - }, - .root_src_path = "builtin.zig", - .fully_qualified_name = if (options.parent == null) - "builtin" - else - try std.fmt.allocPrint(arena, "{s}.builtin", .{options.fully_qualified_name}), - .resolved_target = .{ - .result = target, - .is_native_os = resolved_target.is_native_os, - .is_native_abi = resolved_target.is_native_abi, - .llvm_cpu_features = llvm_cpu_features, - }, - .optimize_mode = optimize_mode, - .single_threaded = single_threaded, - .error_tracing = error_tracing, - .valgrind = valgrind, - .pic = pic, - .strip = strip, - .omit_frame_pointer = omit_frame_pointer, - .stack_check = stack_check, - .stack_protector = stack_protector, - .code_model = code_model, - .red_zone = red_zone, - .sanitize_c = sanitize_c, - .sanitize_thread = sanitize_thread, - .fuzz = fuzz, - .unwind_tables = unwind_tables, - .cc_argv = &.{}, - .structured_cfg = structured_cfg, - .no_builtin = no_builtin, - .builtin_file = new_file, - }; - new_file.* = .{ - .sub_file_path = "builtin.zig", - .stat = undefined, - .source = generated_builtin_source, - .tree = null, - .zir = null, - .zoir = null, - .status = .never_loaded, - .mod = new, - }; - break :b new; - }; - - if (opt_builtin_mod) |builtin_mod| { - try mod.deps.ensureUnusedCapacity(arena, 1); - mod.deps.putAssumeCapacityNoClobber("builtin", builtin_mod); - } - - return mod; -} - -/// All fields correspond to `CreateOptions`. -pub const LimitedOptions = struct { - root: Cache.Path, - root_src_path: []const u8, - fully_qualified_name: []const u8, -}; - -/// This one can only be used if the Module will only be used for AstGen and earlier in -/// the pipeline. Illegal behavior occurs if a limited module touches Sema. -pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { - const mod = try gpa.create(Module); - mod.* = .{ - .root = options.root, - .root_src_path = options.root_src_path, - .fully_qualified_name = options.fully_qualified_name, - - .resolved_target = undefined, - .optimize_mode = undefined, - .code_model = undefined, - .single_threaded = undefined, - .error_tracing = undefined, - .valgrind = undefined, - .pic = undefined, - .strip = undefined, - .omit_frame_pointer = undefined, - .stack_check = undefined, - .stack_protector = undefined, - .red_zone = undefined, - .sanitize_c = undefined, - .sanitize_thread = undefined, - .fuzz = undefined, - .unwind_tables = undefined, - .cc_argv = undefined, - .structured_cfg = undefined, - .no_builtin = undefined, - .builtin_file = null, - }; - return mod; -} - -/// Asserts that the module has a builtin module, which is not true for non-zig -/// modules such as ones only used for `@embedFile`, or the root module when -/// there is no Zig Compilation Unit. -pub fn getBuiltinDependency(m: Module) *Module { - const result = m.deps.values()[0]; - assert(result.isBuiltin()); - return result; -} - -const Module = @This(); -const Package = @import("../Package.zig"); -const std = @import("std"); -const Allocator = std.mem.Allocator; -const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; -const target_util = @import("../target.zig"); -const Cache = std.Build.Cache; -const Builtin = @import("../Builtin.zig"); -const assert = std.debug.assert; -const Compilation = @import("../Compilation.zig"); -const File = @import("../Zcu.zig").File; diff --git a/src/dev.zig b/src/dev.zig index f4be5a36a905fe435dcf237077744dae33d3ed8b..5fcc40bfbcea911c85374c5ab2d9609697cd0070 100644 --- a/src/dev.zig +++ b/src/dev.zig @@ -54,7 +54,6 @@ pub const Env = enum { .test_command, .run_command, .ar_command, - .build_command, .clang_command, .stdio_listen, .build_import_lib, @@ -87,7 +86,6 @@ pub const Env = enum { .translate_c_command, .fmt_command, .jit_command, - .fetch_command, .init_command, .targets_command, .version_command, @@ -135,7 +133,6 @@ pub const Env = enum { else => Env.ast_gen.supports(feature), }, .@"x86_64-linux" => switch (feature) { - .build_command, .stdio_listen, .incremental, .x86_64_backend, @@ -178,13 +175,11 @@ pub const Feature = enum { test_command, run_command, ar_command, - build_command, clang_command, cc_command, translate_c_command, fmt_command, jit_command, - fetch_command, init_command, targets_command, version_command, diff --git a/src/main.zig b/src/main.zig index 356808804abc9f2da5537722ddb6fed2b96250df..9f1b508df7de3b2d6e23202d81a26a1fc2eb3f0d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,5 +1,7 @@ -const std = @import("std"); const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("std"); const assert = std.debug.assert; const io = std.io; const fs = std.fs; @@ -12,7 +14,6 @@ const Color = std.zig.Color; const warn = std.log.warn; const ThreadPool = std.Thread.Pool; const cleanExit = std.process.cleanExit; -const native_os = builtin.os.tag; const Cache = std.Build.Cache; const Path = std.Build.Cache.Path; const Directory = std.Build.Cache.Directory; @@ -34,6 +35,7 @@ const crash_report = @import("crash_report.zig"); const Zcu = @import("Zcu.zig"); const mingw = @import("mingw.zig"); const dev = @import("dev.zig"); +const Module = @import("Module.zig"); test { _ = Package; @@ -289,8 +291,14 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { dev.check(.ar_command); return process.exit(try llvmArMain(arena, args)); } else if (mem.eql(u8, cmd, "build")) { - dev.check(.build_command); - return cmdBuild(gpa, arena, cmd_args); + return jitCmd(gpa, arena, cmd_args, .{ + .cmd_name = "build", + .root_src_path = "build.zig", + .prepend_zig_lib_dir_path = true, + .prepend_global_cache_path = true, + .prepend_zig_exe_path = true, + .optimize_mode = .ReleaseSafe, // Sprinkle some safety on the networking code. + }); } else if (mem.eql(u8, cmd, "clang") or mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { @@ -329,7 +337,13 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .root_src_path = "objcopy.zig", }); } else if (mem.eql(u8, cmd, "fetch")) { - return cmdFetch(gpa, arena, cmd_args); + return jitCmd(gpa, arena, cmd_args, .{ + .cmd_name = "fetch", + .root_src_path = "fetch.zig", + .prepend_global_cache_path = true, + .prepend_zig_lib_dir_path = true, + .optimize_mode = .ReleaseSafe, // Sprinkle some safety on the networking code. + }); } else if (mem.eql(u8, cmd, "libc")) { return jitCmd(gpa, arena, cmd_args, .{ .cmd_name = "libc", @@ -790,14 +804,14 @@ const Framework = struct { }; const CliModule = struct { - paths: Package.Module.CreateOptions.Paths, + paths: Module.CreateOptions.Paths, cc_argv: []const []const u8, - inherited: Package.Module.CreateOptions.Inherited, + inherited: Module.CreateOptions.Inherited, target_arch_os_abi: ?[]const u8, target_mcpu: ?[]const u8, deps: []const Dep, - resolved: ?*Package.Module, + resolved: ?*Module, c_source_files_start: usize, c_source_files_end: usize, @@ -944,7 +958,7 @@ fn buildOutputType( // These get set by CLI flags and then snapshotted when a `-M` flag is // encountered. - var mod_opts: Package.Module.CreateOptions.Inherited = .{}; + var mod_opts: Module.CreateOptions.Inherited = .{}; // These get appended to by CLI flags and then slurped when a `-M` flag // is encountered. @@ -2991,7 +3005,7 @@ fn buildOutputType( create_module.opts.emit_bin = emit_bin != .no; create_module.opts.any_c_source_files = create_module.c_source_files.items.len != 0; - var builtin_modules: std.StringHashMapUnmanaged(*Package.Module) = .empty; + var builtin_modules: std.StringHashMapUnmanaged(*Module) = .empty; // `builtin_modules` allocated into `arena`, so no deinit const main_mod = try createModule(gpa, arena, &create_module, 0, null, zig_lib_directory, &builtin_modules, color); for (create_module.modules.keys(), create_module.modules.values()) |key, cli_mod| { @@ -3023,7 +3037,7 @@ fn buildOutputType( const root_mod = if (arg_mode == .zig_test) root_mod: { const test_mod = if (test_runner_path) |test_runner| test_mod: { - const test_mod = try Package.Module.create(arena, .{ + const test_mod = try Module.create(arena, .{ .global_cache_directory = global_cache_directory, .paths = .{ .root = .{ @@ -3042,7 +3056,7 @@ fn buildOutputType( }); test_mod.deps = try main_mod.deps.clone(arena); break :test_mod test_mod; - } else try Package.Module.create(arena, .{ + } else try Module.create(arena, .{ .global_cache_directory = global_cache_directory, .paths = .{ .root = .{ @@ -3824,11 +3838,11 @@ fn createModule( arena: Allocator, create_module: *CreateModule, index: usize, - parent: ?*Package.Module, + parent: ?*Module, zig_lib_directory: Cache.Directory, - builtin_modules: *std.StringHashMapUnmanaged(*Package.Module), + builtin_modules: *std.StringHashMapUnmanaged(*Module), color: std.zig.Color, -) Allocator.Error!*Package.Module { +) Allocator.Error!*Module { const cli_mod = &create_module.modules.values()[index]; if (cli_mod.resolved) |m| return m; @@ -4114,7 +4128,7 @@ fn createModule( }; } - const mod = Package.Module.create(arena, .{ + const mod = Module.create(arena, .{ .global_cache_directory = create_module.global_cache_directory, .paths = cli_mod.paths, .fully_qualified_name = name, @@ -4740,12 +4754,19 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } } - var templates = findTemplates(gpa, arena); - defer templates.deinit(); + const self_exe_path = introspect.findZigExePath(arena) catch |err| { + fatal("unable to find self exe path: {s}", .{@errorName(err)}); + }; + const zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { + fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) }); + }; + + var templates = std.zig.Package.Templates.find(gpa, zig_lib_directory); + defer templates.deinit(gpa); const cwd_path = try process.getCwdAlloc(arena); const cwd_basename = fs.path.basename(cwd_path); - const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename); + const sanitized_root_name = try Package.sanitizeExampleName(arena, cwd_basename); const s = fs.path.sep_str; const template_paths = [_][]const u8{ @@ -4757,9 +4778,10 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var ok_count: usize = 0; const fingerprint: Package.Fingerprint = .generate(sanitized_root_name); + const zig_ver = build_options.version; for (template_paths) |template_path| { - if (templates.write(arena, fs.cwd(), sanitized_root_name, template_path, fingerprint)) |_| { + if (templates.write(arena, fs.cwd(), sanitized_root_name, template_path, fingerprint, zig_ver)) |_| { std.log.info("created {s}", .{template_path}); ok_count += 1; } else |err| switch (err) { @@ -4776,700 +4798,6 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { return cleanExit(); } -fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 { - var result: std.ArrayListUnmanaged(u8) = .empty; - for (bytes, 0..) |byte, i| switch (byte) { - '0'...'9' => { - if (i == 0) try result.append(arena, '_'); - try result.append(arena, byte); - }, - '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte), - '-', '.', ' ' => try result.append(arena, '_'), - else => continue, - }; - if (!std.zig.isValidId(result.items)) return "foo"; - if (result.items.len > Package.Manifest.max_name_len) - result.shrinkRetainingCapacity(Package.Manifest.max_name_len); - - return result.toOwnedSlice(arena); -} - -test sanitizeExampleName { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!")); - try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a")); - try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!")); - try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test")); - try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests")); - try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); -} - -fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { - dev.check(.build_command); - - var build_file: ?[]const u8 = null; - var override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena); - var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); - var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena); - var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena); - var child_argv = std.ArrayList([]const u8).init(arena); - var reference_trace: ?u32 = null; - var debug_compile_errors = false; - var verbose_link = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_LINK.isSet(); - var verbose_cc = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_CC.isSet(); - var verbose_air = false; - var verbose_intern_pool = false; - var verbose_generic_instances = false; - var verbose_llvm_ir: ?[]const u8 = null; - var verbose_llvm_bc: ?[]const u8 = null; - var verbose_cimport = false; - var verbose_llvm_cpu_features = false; - var fetch_only = false; - var system_pkg_dir_path: ?[]const u8 = null; - var debug_target: ?[]const u8 = null; - - const argv_index_exe = child_argv.items.len; - _ = try child_argv.addOne(); - - const self_exe_path = try introspect.findZigExePath(arena); - try child_argv.append(self_exe_path); - - const argv_index_zig_lib_dir = child_argv.items.len; - _ = try child_argv.addOne(); - - const argv_index_build_file = child_argv.items.len; - _ = try child_argv.addOne(); - - const argv_index_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(); - - const argv_index_global_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(); - - try child_argv.appendSlice(&.{ - "--seed", - try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}), - }); - const argv_index_seed = child_argv.items.len - 1; - - // This parent process needs a way to obtain results from the configuration - // phase of the child process. In the future, the make phase will be - // executed in a separate process than the configure phase, and we can then - // use stdout from the configuration phase for this purpose. - // - // However, currently, both phases are in the same process, and Run Step - // provides API for making the runned subprocesses inherit stdout and stderr - // which means these streams are not available for passing metadata back - // to the parent. - // - // Until make and configure phases are separated into different processes, - // the strategy is to choose a temporary file name ahead of time, and then - // read this file in the parent to obtain the results, in the case the child - // exits with code 3. - const results_tmp_file_nonce = std.fmt.hex(std.crypto.random.int(u64)); - try child_argv.append("-Z" ++ results_tmp_file_nonce); - - var color: Color = .auto; - var n_jobs: ?u32 = null; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "--build-file")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - build_file = args[i]; - continue; - } else if (mem.eql(u8, arg, "--zig-lib-dir")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - override_lib_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--build-runner")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - override_build_runner = args[i]; - continue; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.eql(u8, arg, "--fetch")) { - fetch_only = true; - } else if (mem.eql(u8, arg, "--system")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - system_pkg_dir_path = args[i]; - try child_argv.append("--system"); - continue; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - fatal("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - try child_argv.appendSlice(args[i .. i + 2]); - i += 1; - if (!build_options.enable_logging) { - warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{}); - } else { - try log_scopes.append(arena, args[i]); - } - continue; - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - if (build_options.enable_debug_extensions) { - debug_compile_errors = true; - } else { - warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{}); - } - } else if (mem.eql(u8, arg, "--debug-target")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_target = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{}); - } - } else if (mem.eql(u8, arg, "--verbose-link")) { - verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-intern-pool")) { - verbose_intern_pool = true; - } else if (mem.eql(u8, arg, "--verbose-generic-instances")) { - verbose_generic_instances = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - verbose_llvm_ir = "-"; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { - verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; - } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { - verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; - } else if (mem.eql(u8, arg, "--verbose-cimport")) { - verbose_cimport = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - verbose_llvm_cpu_features = true; - } else if (mem.eql(u8, arg, "--color")) { - if (i + 1 >= args.len) fatal("expected [auto|on|off] after {s}", .{arg}); - i += 1; - color = std.meta.stringToEnum(Color, args[i]) orelse { - fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] }); - }; - try child_argv.appendSlice(&.{ arg, args[i] }); - continue; - } else if (mem.startsWith(u8, arg, "-j")) { - const str = arg["-j".len..]; - const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| { - fatal("unable to parse jobs count '{s}': {s}", .{ - str, @errorName(err), - }); - }; - if (num < 1) { - fatal("number of jobs must be at least 1\n", .{}); - } - n_jobs = num; - } else if (mem.eql(u8, arg, "--seed")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - child_argv.items[argv_index_seed] = args[i]; - continue; - } else if (mem.eql(u8, arg, "--")) { - // The rest of the args are supposed to get passed onto - // build runner's `build.args` - try child_argv.appendSlice(args[i..]); - break; - } - } - try child_argv.append(arg); - } - } - - const work_around_btrfs_bug = native_os == .linux and - EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); - const root_prog_node = std.Progress.start(.{ - .disable_printing = (color == .off), - .root_name = "Compile Build Script", - }); - defer root_prog_node.end(); - - // Normally the build runner is compiled for the host target but here is - // some code to help when debugging edits to the build runner so that you - // can make sure it compiles successfully on other targets. - const resolved_target: Package.Module.ResolvedTarget = t: { - if (build_options.enable_debug_extensions) { - if (debug_target) |triple| { - const target_query = try std.Target.Query.parse(.{ - .arch_os_abi = triple, - }); - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(target_query), - .is_native_os = false, - .is_native_abi = false, - }; - } - } - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(.{}), - .is_native_os = true, - .is_native_abi = true, - }; - }; - - const exe_basename = try std.zig.binNameAlloc(arena, .{ - .root_name = "build", - .target = resolved_target.result, - .output_mode = .Exe, - }); - const emit_bin: Compilation.EmitLoc = .{ - .directory = null, // Use the local zig-cache. - .basename = exe_basename, - }; - - process.raiseFileDescriptorLimit(); - - var zig_lib_directory: Directory = if (override_lib_dir) |lib_dir| .{ - .path = lib_dir, - .handle = fs.cwd().openDir(lib_dir, .{}) catch |err| { - fatal("unable to open zig lib directory from 'zig-lib-dir' argument: '{s}': {s}", .{ lib_dir, @errorName(err) }); - }, - } else introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { - fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) }); - }; - defer zig_lib_directory.handle.close(); - - const cwd_path = try process.getCwdAlloc(arena); - child_argv.items[argv_index_zig_lib_dir] = zig_lib_directory.path orelse cwd_path; - - const build_root = try findBuildRoot(arena, .{ - .cwd_path = cwd_path, - .build_file = build_file, - }); - child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; - - var global_cache_directory: Directory = l: { - const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); - const dir = fs.cwd().makeOpenPath(p, .{}) catch |err| { - const base_msg = "unable to open or create the global Zig cache at '{s}': {s}.{s}"; - const extra = "\nIf this location is not writable then consider specifying an " ++ - "alternative with the ZIG_GLOBAL_CACHE_DIR environment variable or the " ++ - "--global-cache-dir option."; - const show_extra = err == error.AccessDenied or err == error.ReadOnlyFileSystem; - fatal(base_msg, .{ p, @errorName(err), if (show_extra) extra else "" }); - }; - break :l .{ - .handle = dir, - .path = p, - }; - }; - defer global_cache_directory.handle.close(); - - child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path; - - var local_cache_directory: Directory = l: { - if (override_local_cache_dir) |local_cache_dir_path| { - break :l .{ - .handle = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}), - .path = local_cache_dir_path, - }; - } - const cache_dir_path = try build_root.directory.join(arena, &.{default_local_zig_cache_basename}); - break :l .{ - .handle = try build_root.directory.handle.makeOpenPath(default_local_zig_cache_basename, .{}), - .path = cache_dir_path, - }; - }; - defer local_cache_directory.handle.close(); - - child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path; - - var thread_pool: ThreadPool = undefined; - try thread_pool.init(.{ - .allocator = gpa, - .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)), - .track_ids = true, - .stack_size = thread_stack_size, - }); - defer thread_pool.deinit(); - - // Dummy http client that is not actually used when fetch_command is unsupported. - // Prevents bootstrap from depending on a bunch of unnecessary stuff. - var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { - allocator: Allocator, - fn deinit(_: @This()) void {} - } = .{ .allocator = gpa }; - defer http_client.deinit(); - - var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; - - // This loop is re-evaluated when the build script exits with an indication that it - // could not continue due to missing lazy dependencies. - while (true) { - // We want to release all the locks before executing the child process, so we make a nice - // big block here to ensure the cleanup gets run when we extract out our argv. - { - const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{ - .root = .{ - .root_dir = Cache.Directory.cwd(), - .sub_path = fs.path.dirname(runner) orelse "", - }, - .root_src_path = fs.path.basename(runner), - } else .{ - .root = .{ - .root_dir = zig_lib_directory, - .sub_path = "compiler", - }, - .root_src_path = "build_runner.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .resolved_target = resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .global_cache_directory = global_cache_directory, - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = resolved_target, - }, - .global = config, - .parent = null, - .builtin_mod = null, - .builtin_modules = null, // all modules will inherit this one's builtin - }); - - const builtin_mod = root_mod.getBuiltinDependency(); - - const build_mod = try Package.Module.create(arena, .{ - .global_cache_directory = global_cache_directory, - .paths = .{ - .root = .{ .root_dir = build_root.directory }, - .root_src_path = build_root.build_zig_basename, - }, - .fully_qualified_name = "root.@build", - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - .builtin_mod = builtin_mod, - .builtin_modules = null, // `builtin_mod` is specified - }); - - var cleanup_build_dir: ?fs.Dir = null; - defer if (cleanup_build_dir) |*dir| dir.close(); - - if (dev.env.supports(.fetch_command)) { - const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); - defer fetch_prog_node.end(); - - var job_queue: Package.Fetch.JobQueue = .{ - .http_client = &http_client, - .thread_pool = &thread_pool, - .global_cache = global_cache_directory, - .read_only = false, - .recursive = true, - .debug_hash = false, - .work_around_btrfs_bug = work_around_btrfs_bug, - .unlazy_set = unlazy_set, - }; - defer job_queue.deinit(); - - if (system_pkg_dir_path) |p| { - job_queue.global_cache = .{ - .path = p, - .handle = fs.cwd().openDir(p, .{}) catch |err| { - fatal("unable to open system package directory '{s}': {s}", .{ - p, @errorName(err), - }); - }, - }; - job_queue.read_only = true; - cleanup_build_dir = job_queue.global_cache.handle; - } else { - try http_client.initDefaultProxies(arena); - } - - try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); - try job_queue.table.ensureUnusedCapacity(gpa, 1); - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = build_mod.root }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .parent_package_root = build_mod.root, - .parent_manifest_ast = null, - .prog_node = fetch_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .allow_missing_fingerprint = false, - .allow_name_string = false, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = null, - .manifest_ast = undefined, - .computed_hash = undefined, - .has_build_zig = true, - .oom_flag = false, - .latest_commit = null, - - .module = build_mod, - }; - job_queue.all_fetches.appendAssumeCapacity(&fetch); - - job_queue.table.putAssumeCapacityNoClobber( - Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory), - &fetch, - ); - - job_queue.thread_pool.spawnWg(&job_queue.wait_group, Package.Fetch.workerRun, .{ - &fetch, "root", - }); - job_queue.wait_group.wait(); - - try job_queue.consolidateErrors(); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStdErr(color.renderOptions()); - process.exit(1); - } - - if (fetch_only) return cleanExit(); - - var source_buf = std.ArrayList(u8).init(gpa); - defer source_buf.deinit(); - try job_queue.createDependenciesSource(&source_buf); - const deps_mod = try createDependenciesModule( - arena, - source_buf.items, - root_mod, - global_cache_directory, - local_cache_directory, - builtin_mod, - config, - ); - - { - // We need a Module for each package's build.zig. - const hashes = job_queue.table.keys(); - const fetches = job_queue.table.values(); - try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); - for (hashes, fetches) |*hash, f| { - if (f == &fetch) { - // The first one is a dummy package for the current project. - continue; - } - if (!f.has_build_zig) - continue; - const hash_slice = hash.toSlice(); - const m = try Package.Module.create(arena, .{ - .global_cache_directory = global_cache_directory, - .paths = .{ - .root = try f.package_root.clone(arena), - .root_src_path = Package.build_zig_basename, - }, - .fully_qualified_name = try std.fmt.allocPrint( - arena, - "root.@dependencies.{s}", - .{hash_slice}, - ), - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - .builtin_mod = builtin_mod, - .builtin_modules = null, // `builtin_mod` is specified - }); - const hash_cloned = try arena.dupe(u8, hash_slice); - deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); - f.module = m; - } - - // Each build.zig module needs access to each of its - // dependencies' build.zig modules by name. - for (fetches) |f| { - const mod = f.module orelse continue; - const man = f.manifest orelse continue; - const dep_names = man.dependencies.keys(); - try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); - for (dep_names, man.dependencies.values()) |name, dep| { - const dep_digest = Package.Fetch.depDigest( - f.package_root, - global_cache_directory, - dep, - ) orelse continue; - const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; - const name_cloned = try arena.dupe(u8, name); - mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); - } - } - } - } else try createEmptyDependenciesModule( - arena, - root_mod, - global_cache_directory, - local_cache_directory, - builtin_mod, - config, - ); - - try root_mod.deps.put(arena, "@build", build_mod); - - const comp = Compilation.create(gpa, arena, .{ - .zig_lib_directory = zig_lib_directory, - .local_cache_directory = local_cache_directory, - .global_cache_directory = global_cache_directory, - .root_name = "build", - .config = config, - .root_mod = root_mod, - .main_mod = build_mod, - .emit_bin = emit_bin, - .emit_h = null, - .self_exe_path = self_exe_path, - .thread_pool = &thread_pool, - .verbose_cc = verbose_cc, - .verbose_link = verbose_link, - .verbose_air = verbose_air, - .verbose_intern_pool = verbose_intern_pool, - .verbose_generic_instances = verbose_generic_instances, - .verbose_llvm_ir = verbose_llvm_ir, - .verbose_llvm_bc = verbose_llvm_bc, - .verbose_cimport = verbose_cimport, - .verbose_llvm_cpu_features = verbose_llvm_cpu_features, - .cache_mode = .whole, - .reference_trace = reference_trace, - .debug_compile_errors = debug_compile_errors, - }) catch |err| { - fatal("unable to create compilation: {s}", .{@errorName(err)}); - }; - defer comp.destroy(); - - updateModule(comp, color, root_prog_node) catch |err| switch (err) { - error.SemanticAnalyzeFail => process.exit(2), - else => |e| return e, - }; - - // Since incremental compilation isn't done yet, we use cache_mode = whole - // above, and thus the output file is already closed. - //try comp.makeBinFileExecutable(); - child_argv.items[argv_index_exe] = - try local_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?}); - } - - if (process.can_spawn) { - var child = std.process.Child.init(child_argv.items, gpa); - child.stdin_behavior = .Inherit; - child.stdout_behavior = .Inherit; - child.stderr_behavior = .Inherit; - - const term = t: { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - break :t child.spawnAndWait() catch |err| { - fatal("failed to spawn build runner {s}: {s}", .{ child_argv.items[0], @errorName(err) }); - }; - }; - - switch (term) { - .Exited => |code| { - if (code == 0) return cleanExit(); - // Indicates that the build runner has reported compile errors - // and this parent process does not need to report any further - // diagnostics. - if (code == 2) process.exit(2); - - if (code == 3) { - if (!dev.env.supports(.fetch_command)) process.exit(3); - // Indicates the configure phase failed due to missing lazy - // dependencies and stdout contains the hashes of the ones - // that are missing. - const s = fs.path.sep_str; - const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce; - const stdout = local_cache_directory.handle.readFileAlloc(arena, tmp_sub_path, 50 * 1024 * 1024) catch |err| { - fatal("unable to read results of configure phase from '{}{s}': {s}", .{ - local_cache_directory, tmp_sub_path, @errorName(err), - }); - }; - local_cache_directory.handle.deleteFile(tmp_sub_path) catch {}; - - var it = mem.splitScalar(u8, stdout, '\n'); - var any_errors = false; - while (it.next()) |hash| { - if (hash.len == 0) continue; - if (hash.len > Package.Hash.max_len) { - std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ - hash.len, hash, - }); - any_errors = true; - continue; - } - try unlazy_set.put(arena, .fromSlice(hash), {}); - } - if (any_errors) process.exit(3); - if (system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. - for (unlazy_set.keys()) |*hash| { - std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ - p, hash.toSlice(), - }); - } - std.log.info("remote package fetching disabled due to --system mode", .{}); - std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(3); - } - continue; - } - - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); - }, - else => { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command crashed:\n{s}", .{cmd}); - }, - } - } else { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(native_os), cmd }); - } - } -} - const JitCmdOptions = struct { cmd_name: []const u8, root_src_path: []const u8, @@ -5481,6 +4809,7 @@ const JitCmdOptions = struct { /// Send error bundles via std.zig.Server over stdout server: bool = false, progress_node: ?std.Progress.Node = null, + optimize_mode: std.builtin.OptimizeMode = .ReleaseFast, }; fn jitCmd( @@ -5497,7 +4826,7 @@ fn jitCmd( }); const target_query: std.Target.Query = .{}; - const resolved_target: Package.Module.ResolvedTarget = .{ + const resolved_target: Module.ResolvedTarget = .{ .result = std.zig.resolveTargetQueryOrFatal(target_query), .is_native_os = true, .is_native_abi = true, @@ -5520,7 +4849,7 @@ fn jitCmd( const optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet()) .Debug else - .ReleaseFast; + options.optimize_mode; const strip = optimize_mode != .Debug; const override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena); const override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); @@ -5554,12 +4883,12 @@ fn jitCmd( defer thread_pool.deinit(); var child_argv: std.ArrayListUnmanaged([]const u8) = .empty; - try child_argv.ensureUnusedCapacity(arena, args.len + 4); + try child_argv.ensureUnusedCapacity(arena, args.len + 6); // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. { - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ + const main_mod_paths: Module.CreateOptions.Paths = .{ .root = .{ .root_dir = zig_lib_directory, .sub_path = "compiler", @@ -5577,7 +4906,7 @@ fn jitCmd( .is_test = false, }); - const root_mod = try Package.Module.create(arena, .{ + const root_mod = try Module.create(arena, .{ .global_cache_directory = global_cache_directory, .paths = main_mod_paths, .fully_qualified_name = "root", @@ -5594,7 +4923,7 @@ fn jitCmd( }); if (options.depend_on_aro) { - const aro_mod = try Package.Module.create(arena, .{ + const aro_mod = try Module.create(arena, .{ .global_cache_directory = global_cache_directory, .paths = .{ .root = .{ @@ -5669,7 +4998,24 @@ fn jitCmd( if (options.prepend_global_cache_path) child_argv.appendAssumeCapacity(global_cache_directory.path.?); - child_argv.appendSliceAssumeCapacity(args); + if (options.add_seed_argument) { + child_argv.appendSliceAssumeCapacity(&.{ + "--seed", try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}), + }); + const seed_arg_index = child_argv.items.len - 1; + var i: usize = 0; + while (i < args.len) { + if (mem.eql(u8, args[i], "--seed") and i + 1 <= args.len) { + child_argv.items[seed_arg_index] = args[i + 1]; + i += 2; + } else { + child_argv.appendAssumeCapacity(args[i]); + i += 1; + } + } + } else { + child_argv.appendSliceAssumeCapacity(args); + } if (process.can_execv and options.capture == null) { const err = process.execv(gpa, child_argv.items); @@ -6241,7 +5587,7 @@ fn cmdAstCheck( break :mode .zig; }; - file.mod = try Package.Module.createLimited(arena, .{ + file.mod = try Module.createLimited(arena, .{ .root = Path.cwd(), .root_src_path = file.sub_file_path, .fully_qualified_name = "root", @@ -6647,7 +5993,7 @@ fn cmdChangelist( .mod = undefined, }; - file.mod = try Package.Module.createLimited(arena, .{ + file.mod = try Module.createLimited(arena, .{ .root = Path.cwd(), .root_src_path = file.sub_file_path, .fully_qualified_name = "root", @@ -7008,613 +6354,6 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes { fatal("unsupported rc includes type: '{s}'", .{arg}); } -const usage_fetch = - \\Usage: zig fetch [options] - \\Usage: zig fetch [options] - \\ - \\ Copy a package into the global cache and print its hash. - \\ must point to one of the following: - \\ - A git+http / git+https server for the package - \\ - A tarball file (with or without compression) containing - \\ package source - \\ - A git bundle file containing package source - \\ - \\Examples: - \\ - \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git - \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz - \\ - \\Options: - \\ -h, --help Print this help and exit - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --debug-hash Print verbose hash information to stdout - \\ --save Add the fetched package to build.zig.zon - \\ --save=[name] Add the fetched package to build.zig.zon as name - \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim - \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim - \\ -; - -fn cmdFetch( - gpa: Allocator, - arena: Allocator, - args: []const []const u8, -) !void { - dev.check(.fetch_command); - - const color: Color = .auto; - const work_around_btrfs_bug = native_os == .linux and - EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); - var opt_path_or_url: ?[]const u8 = null; - var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); - var debug_hash: bool = false; - var save: union(enum) { - no, - yes: ?[]const u8, - exact: ?[]const u8, - } = .no; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - const stdout = io.getStdOut().writer(); - try stdout.writeAll(usage_fetch); - return cleanExit(); - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--debug-hash")) { - debug_hash = true; - } else if (mem.eql(u8, arg, "--save")) { - save = .{ .yes = null }; - } else if (mem.startsWith(u8, arg, "--save=")) { - save = .{ .yes = arg["--save=".len..] }; - } else if (mem.eql(u8, arg, "--save-exact")) { - save = .{ .exact = null }; - } else if (mem.startsWith(u8, arg, "--save-exact=")) { - save = .{ .exact = arg["--save-exact=".len..] }; - } else { - fatal("unrecognized parameter: '{s}'", .{arg}); - } - } else if (opt_path_or_url != null) { - fatal("unexpected extra parameter: '{s}'", .{arg}); - } else { - opt_path_or_url = arg; - } - } - } - - const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); - - var thread_pool: ThreadPool = undefined; - try thread_pool.init(.{ .allocator = gpa }); - defer thread_pool.deinit(); - - var http_client: std.http.Client = .{ .allocator = gpa }; - defer http_client.deinit(); - - try http_client.initDefaultProxies(arena); - - var root_prog_node = std.Progress.start(.{ - .root_name = "Fetch", - }); - defer root_prog_node.end(); - - var global_cache_directory: Directory = l: { - const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); - break :l .{ - .handle = try fs.cwd().makeOpenPath(p, .{}), - .path = p, - }; - }; - defer global_cache_directory.handle.close(); - - var job_queue: Package.Fetch.JobQueue = .{ - .http_client = &http_client, - .thread_pool = &thread_pool, - .global_cache = global_cache_directory, - .recursive = false, - .read_only = false, - .debug_hash = debug_hash, - .work_around_btrfs_bug = work_around_btrfs_bug, - }; - defer job_queue.deinit(); - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .path_or_url = path_or_url }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .parent_package_root = undefined, - .parent_manifest_ast = null, - .prog_node = root_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .allow_missing_fingerprint = true, - .allow_name_string = true, - .use_latest_commit = true, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = null, - .manifest_ast = undefined, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - defer fetch.deinit(); - - fetch.run() catch |err| switch (err) { - error.OutOfMemory => fatal("out of memory", .{}), - error.FetchFailed => {}, // error bundle checked below - }; - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStdErr(color.renderOptions()); - process.exit(1); - } - - const package_hash = fetch.computedPackageHash(); - const package_hash_slice = package_hash.toSlice(); - - root_prog_node.end(); - root_prog_node = .{ .index = .none }; - - const name = switch (save) { - .no => { - try io.getStdOut().writer().print("{s}\n", .{package_hash_slice}); - return cleanExit(); - }, - .yes, .exact => |name| name: { - if (name) |n| break :name n; - const fetched_manifest = fetch.manifest orelse - fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); - break :name fetched_manifest.name; - }, - }; - - const cwd_path = try process.getCwdAlloc(arena); - - var build_root = try findBuildRoot(arena, .{ - .cwd_path = cwd_path, - }); - defer build_root.deinit(); - - // The name to use in case the manifest file needs to be created now. - const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path); - var manifest, var ast = try loadManifest(gpa, arena, .{ - .root_name = try sanitizeExampleName(arena, init_root_name), - .dir = build_root.directory.handle, - .color = color, - }); - defer { - manifest.deinit(gpa); - ast.deinit(gpa); - } - - var fixups: Ast.Fixups = .{}; - defer fixups.deinit(gpa); - - var saved_path_or_url = path_or_url; - - if (fetch.latest_commit) |latest_commit| resolved: { - const latest_commit_hex = try std.fmt.allocPrint(arena, "{}", .{latest_commit}); - - var uri = try std.Uri.parse(path_or_url); - - if (uri.fragment) |fragment| { - const target_ref = try fragment.toRawMaybeAlloc(arena); - - // the refspec may already be fully resolved - if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; - - std.log.info("resolved ref '{s}' to commit {s}", .{ target_ref, latest_commit_hex }); - - // include the original refspec in a query parameter, could be used to check for updates - uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={%}", .{fragment}) }; - } else { - std.log.info("resolved to commit {s}", .{latest_commit_hex}); - } - - // replace the refspec with the resolved commit SHA - uri.fragment = .{ .raw = latest_commit_hex }; - - switch (save) { - .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{}", .{uri}), - .no, .exact => {}, // keep the original URL - } - } - - const new_node_init = try std.fmt.allocPrint(arena, - \\.{{ - \\ .url = "{}", - \\ .hash = "{}", - \\ }} - , .{ - std.zig.fmtEscapes(saved_path_or_url), - std.zig.fmtEscapes(package_hash_slice), - }); - - const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{ - std.zig.fmtId(name), new_node_init, - }); - - const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ - new_node_text, - }); - - const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ - dependencies_init, - }); - - if (manifest.dependencies.get(name)) |dep| { - if (dep.hash) |h| { - switch (dep.location) { - .url => |u| { - if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { - std.log.info("existing dependency named '{s}' is up-to-date", .{name}); - process.exit(0); - } - }, - .path => {}, - } - } - - const location_replace = try std.fmt.allocPrint( - arena, - "\"{}\"", - .{std.zig.fmtEscapes(saved_path_or_url)}, - ); - const hash_replace = try std.fmt.allocPrint( - arena, - "\"{}\"", - .{std.zig.fmtEscapes(package_hash_slice)}, - ); - - warn("overwriting existing dependency named '{s}'", .{name}); - try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); - if (dep.hash_node.unwrap()) |hash_node| { - try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); - } else { - // https://github.com/ziglang/zig/issues/21690 - } - } else if (manifest.dependencies.count() > 0) { - // Add fixup for adding another dependency. - const deps = manifest.dependencies.values(); - const last_dep_node = deps[deps.len - 1].node; - try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); - } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { - // Add fixup for replacing the entire dependencies struct. - try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); - } else { - // Add fixup for adding dependencies struct. - try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); - } - - var rendered = std.ArrayList(u8).init(gpa); - defer rendered.deinit(); - try ast.renderToArrayList(&rendered, fixups); - - build_root.directory.handle.writeFile(.{ .sub_path = Package.Manifest.basename, .data = rendered.items }) catch |err| { - fatal("unable to write {s} file: {s}", .{ Package.Manifest.basename, @errorName(err) }); - }; - - return cleanExit(); -} - -fn createEmptyDependenciesModule( - arena: Allocator, - main_mod: *Package.Module, - global_cache_directory: Cache.Directory, - local_cache_directory: Cache.Directory, - builtin_mod: *Package.Module, - global_options: Compilation.Config, -) !void { - var source = std.ArrayList(u8).init(arena); - try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); - _ = try createDependenciesModule( - arena, - source.items, - main_mod, - global_cache_directory, - local_cache_directory, - builtin_mod, - global_options, - ); -} - -/// Creates the dependencies.zig file and corresponding `Package.Module` for the -/// build runner to obtain via `@import("@dependencies")`. -fn createDependenciesModule( - arena: Allocator, - source: []const u8, - main_mod: *Package.Module, - global_cache_directory: Cache.Directory, - local_cache_directory: Cache.Directory, - builtin_mod: *Package.Module, - global_options: Compilation.Config, -) !*Package.Module { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - { - var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(.{ .sub_path = basename, .data = source }); - } - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(source); - const hex_digest = hh.final(); - - const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); - try Package.Fetch.renameTmpIntoCache( - local_cache_directory.handle, - tmp_dir_sub_path, - o_dir_sub_path, - ); - - const deps_mod = try Package.Module.create(arena, .{ - .global_cache_directory = global_cache_directory, - .paths = .{ - .root = .{ - .root_dir = local_cache_directory, - .sub_path = o_dir_sub_path, - }, - .root_src_path = basename, - }, - .fully_qualified_name = "root.@dependencies", - .parent = main_mod, - .cc_argv = &.{}, - .inherited = .{}, - .global = global_options, - .builtin_mod = builtin_mod, - .builtin_modules = null, // `builtin_mod` is specified - }); - try main_mod.deps.put(arena, "@dependencies", deps_mod); - return deps_mod; -} - -const BuildRoot = struct { - directory: Cache.Directory, - build_zig_basename: []const u8, - cleanup_build_dir: ?fs.Dir, - - fn deinit(br: *BuildRoot) void { - if (br.cleanup_build_dir) |*dir| dir.close(); - br.* = undefined; - } -}; - -const FindBuildRootOptions = struct { - build_file: ?[]const u8 = null, - cwd_path: ?[]const u8 = null, -}; - -fn findBuildRoot(arena: Allocator, options: FindBuildRootOptions) !BuildRoot { - const cwd_path = options.cwd_path orelse try process.getCwdAlloc(arena); - const build_zig_basename = if (options.build_file) |bf| - fs.path.basename(bf) - else - Package.build_zig_basename; - - if (options.build_file) |bf| { - if (fs.path.dirname(bf)) |dirname| { - const dir = fs.cwd().openDir(dirname, .{}) catch |err| { - fatal("unable to open directory to build file from argument 'build-file', '{s}': {s}", .{ dirname, @errorName(err) }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = dirname, .handle = dir }, - .cleanup_build_dir = dir, - }; - } - - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = null, .handle = fs.cwd() }, - .cleanup_build_dir = null, - }; - } - // Search up parent directories until we find build.zig. - var dirname: []const u8 = cwd_path; - while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); - if (fs.cwd().access(joined_path, .{})) |_| { - const dir = fs.cwd().openDir(dirname, .{}) catch |err| { - fatal("unable to open directory while searching for build.zig file, '{s}': {s}", .{ dirname, @errorName(err) }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ - .path = dirname, - .handle = dir, - }, - .cleanup_build_dir = dir, - }; - } else |err| switch (err) { - error.FileNotFound => { - dirname = fs.path.dirname(dirname) orelse { - std.log.info("initialize {s} template file with 'zig init'", .{ - Package.build_zig_basename, - }); - std.log.info("see 'zig --help' for more options", .{}); - fatal("no build.zig file found, in the current directory or any parent directories", .{}); - }; - continue; - }, - else => |e| return e, - } - } -} - -const LoadManifestOptions = struct { - root_name: []const u8, - dir: fs.Dir, - color: Color, -}; - -fn loadManifest( - gpa: Allocator, - arena: Allocator, - options: LoadManifestOptions, -) !struct { Package.Manifest, Ast } { - const manifest_bytes = while (true) { - break options.dir.readFileAllocOptions( - arena, - Package.Manifest.basename, - Package.Manifest.max_bytes, - null, - 1, - 0, - ) catch |err| switch (err) { - error.FileNotFound => { - const fingerprint: Package.Fingerprint = .generate(options.root_name); - var templates = findTemplates(gpa, arena); - defer templates.deinit(); - templates.write(arena, options.dir, options.root_name, Package.Manifest.basename, fingerprint) catch |e| { - fatal("unable to write {s}: {s}", .{ - Package.Manifest.basename, @errorName(e), - }); - }; - continue; - }, - else => |e| fatal("unable to load {s}: {s}", .{ - Package.Manifest.basename, @errorName(e), - }), - }; - }; - var ast = try Ast.parse(gpa, manifest_bytes, .zon); - errdefer ast.deinit(gpa); - - if (ast.errors.len > 0) { - try std.zig.printAstErrorsToStderr(gpa, ast, Package.Manifest.basename, options.color); - process.exit(2); - } - - var manifest = try Package.Manifest.parse(gpa, ast, .{}); - errdefer manifest.deinit(gpa); - - if (manifest.errors.len > 0) { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - const src_path = try wip_errors.addString(Package.Manifest.basename); - try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); - - var error_bundle = try wip_errors.toOwnedBundle(""); - defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(options.color.renderOptions()); - - process.exit(2); - } - return .{ manifest, ast }; -} - -const Templates = struct { - zig_lib_directory: Cache.Directory, - dir: fs.Dir, - buffer: std.ArrayList(u8), - - fn deinit(templates: *Templates) void { - templates.zig_lib_directory.handle.close(); - templates.dir.close(); - templates.buffer.deinit(); - templates.* = undefined; - } - - fn write( - templates: *Templates, - arena: Allocator, - out_dir: fs.Dir, - root_name: []const u8, - template_path: []const u8, - fingerprint: Package.Fingerprint, - ) !void { - if (fs.path.dirname(template_path)) |dirname| { - out_dir.makePath(dirname) catch |err| { - fatal("unable to make path '{s}': {s}", .{ dirname, @errorName(err) }); - }; - } - - const max_bytes = 10 * 1024 * 1024; - const contents = templates.dir.readFileAlloc(arena, template_path, max_bytes) catch |err| { - fatal("unable to read template file '{s}': {s}", .{ template_path, @errorName(err) }); - }; - templates.buffer.clearRetainingCapacity(); - try templates.buffer.ensureUnusedCapacity(contents.len); - var i: usize = 0; - while (i < contents.len) { - if (contents[i] == '.') { - if (std.mem.startsWith(u8, contents[i..], ".LITNAME")) { - try templates.buffer.append('.'); - try templates.buffer.appendSlice(root_name); - i += ".LITNAME".len; - continue; - } else if (std.mem.startsWith(u8, contents[i..], ".NAME")) { - try templates.buffer.appendSlice(root_name); - i += ".NAME".len; - continue; - } else if (std.mem.startsWith(u8, contents[i..], ".FINGERPRINT")) { - try templates.buffer.writer().print("0x{x}", .{fingerprint.int()}); - i += ".FINGERPRINT".len; - continue; - } else if (std.mem.startsWith(u8, contents[i..], ".ZIGVER")) { - try templates.buffer.appendSlice(build_options.version); - i += ".ZIGVER".len; - continue; - } - } - try templates.buffer.append(contents[i]); - i += 1; - } - - return out_dir.writeFile(.{ - .sub_path = template_path, - .data = templates.buffer.items, - .flags = .{ .exclusive = true }, - }); - } -}; - -fn findTemplates(gpa: Allocator, arena: Allocator) Templates { - const self_exe_path = introspect.findZigExePath(arena) catch |err| { - fatal("unable to find self exe path: {s}", .{@errorName(err)}); - }; - var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { - fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) }); - }; - - const s = fs.path.sep_str; - const template_sub_path = "init"; - const template_dir = zig_lib_directory.handle.openDir(template_sub_path, .{}) catch |err| { - const path = zig_lib_directory.path orelse "."; - fatal("unable to open zig project template directory '{s}{s}{s}': {s}", .{ - path, s, template_sub_path, @errorName(err), - }); - }; - - return .{ - .zig_lib_directory = zig_lib_directory, - .dir = template_dir, - .buffer = std.ArrayList(u8).init(gpa), - }; -} - fn parseOptimizeMode(s: []const u8) std.builtin.OptimizeMode { return std.meta.stringToEnum(std.builtin.OptimizeMode, s) orelse fatal("unrecognized optimization mode: '{s}'", .{s}); @@ -7640,7 +6379,7 @@ fn handleModArg( mod_name: []const u8, opt_root_src_orig: ?[]const u8, create_module: *CreateModule, - mod_opts: *Package.Module.CreateOptions.Inherited, + mod_opts: *Module.CreateOptions.Inherited, cc_argv: *std.ArrayListUnmanaged([]const u8), target_arch_os_abi: *?[]const u8, target_mcpu: *?[]const u8,