From 0c978ba957ad1d44f5a4952b2d6dce121a5b05e6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 22 Jun 2026 17:32:23 -0700 Subject: [PATCH] WIP: migrate build and fetch commands to Maker process --- build.zig | 3 + lib/compiler/Maker.zig | 1419 +++++++++++++- {src/Package => lib/compiler/Maker}/Fetch.zig | 41 +- .../compiler/Maker}/Fetch/git.zig | 0 .../Fetch/git/testdata/testrepo-sha1.idx | Bin .../Fetch/git/testdata/testrepo-sha1.pack | Bin .../Fetch/git/testdata/testrepo-sha256.idx | Bin .../Fetch/git/testdata/testrepo-sha256.pack | Bin {src => lib/compiler/Maker}/Package.zig | 2 - .../compiler/Maker}/Package/Manifest.zig | 0 lib/compiler/configurer.zig | 5 +- lib/std/Build/Cache.zig | 2 +- lib/std/Build/Configuration.zig | 2 +- lib/std/zig.zig | 383 +++- src/Compilation.zig | 158 +- src/{Package => }/Module.zig | 34 +- src/Zcu.zig | 1 - src/Zcu/PerThread.zig | 1 - src/dev.zig | 2 - src/introspect.zig | 220 --- src/main.zig | 1688 +---------------- src/print_env.zig | 3 +- src/print_targets.zig | 3 +- 23 files changed, 1784 insertions(+), 2183 deletions(-) rename {src/Package => lib/compiler/Maker}/Fetch.zig (99%) rename {src/Package => lib/compiler/Maker}/Fetch/git.zig (100%) rename {src/Package => lib/compiler/Maker}/Fetch/git/testdata/testrepo-sha1.idx (100%) rename {src/Package => lib/compiler/Maker}/Fetch/git/testdata/testrepo-sha1.pack (100%) rename {src/Package => lib/compiler/Maker}/Fetch/git/testdata/testrepo-sha256.idx (100%) rename {src/Package => lib/compiler/Maker}/Fetch/git/testdata/testrepo-sha256.pack (100%) rename {src => lib/compiler/Maker}/Package.zig (98%) rename {src => lib/compiler/Maker}/Package/Manifest.zig (100%) rename src/{Package => }/Module.zig (97%) delete mode 100644 src/introspect.zig diff --git a/build.zig b/build.zig index e97fac39188412a77f57f3559247cf07a40c25ed..19bf41dc607d3b93e96bf53598d9945f8bd5bcf4 100644 --- a/build.zig +++ b/build.zig @@ -175,6 +175,9 @@ pub fn build(b: *std.Build) !void { ".tar", // exclude files from lib/std/zip/testdata ".zip", + // exclude files from lib/compiler/Maker/Fetch/git/testdata + ".idx", + ".pack", // others "README.md", }, diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 4d4ceceea255b4c94681843e9e7f4c23c823c4e8..6fe474bee810bc95c0b6e6bcd6bfa063b60e8686 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -17,6 +17,10 @@ const log = std.log; const mem = std.mem; const process = std.process; const Color = std.zig.Color; +const EnvVar = std.zig.EnvVar; +const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename; +const allocPrint = std.fmt.allocPrint; +const stringToEnum = std.meta.stringToEnum; const Fuzz = @import("Maker/Fuzz.zig"); const Graph = @import("Maker/Graph.zig"); @@ -25,10 +29,11 @@ const Watch = @import("Maker/Watch.zig"); const WebServer = @import("Maker/WebServer.zig"); const ScannedConfig = @import("Maker/ScannedConfig.zig"); const PkgConfig = @import("Maker/PkgConfig.zig"); +const Fetch = @import("Maker/Fetch.zig"); +const Package = @import("Maker/Package.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, - .http_disable_tls = true, }; gpa: Allocator, @@ -100,6 +105,15 @@ const ErrorStyle = enum { const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; +/// Used to build the -M flags to pass to build-exe. +const CliModule = struct { + name: []const u8, + root_path: []const u8, + deps: Deps = .empty, + + const Deps = std.array_hash_map.String(*CliModule); +}; + pub fn main(init: process.Init.Minimal) !void { // The build runner is long-lived in the following use cases: // * `--watch` mode @@ -124,70 +138,54 @@ pub fn main(init: process.Init.Minimal) !void { const arena = arena_instance.allocator(); const args = try init.args.toSlice(arena); - - // skip my own exe name - var arg_idx: usize = 1; - - const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); - const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); - const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); - const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); - const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); - const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); + var arg_i: usize = 1; + const cmd_name = nextArgOrFatal(args, &arg_i); + const zig_lib_arg = prefixedArgOrFatal(args, &arg_i, "--zig-lib="); + const zig_exe_arg = prefixedArgOrFatal(args, &arg_i, "--zig="); + const global_cache_arg = prefixedArgOrFatal(args, &arg_i, "--global-cache="); + const seed_arg = prefixedArgOrFatal(args, &arg_i, "--seed="); const cwd: Dir = .cwd(); const zig_lib_directory: Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try cwd.openDir(io, zig_lib_dir, .{}), - }; - - const build_root_directory: Cache.Directory = .{ - .path = build_root, - .handle = try cwd.openDir(io, build_root, .{}), - }; - - const local_cache_directory: Cache.Directory = .{ - .path = local_cache_root, - .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), + .path = zig_lib_arg, + .handle = try cwd.openDir(io, zig_lib_arg, .{}), }; const global_cache_directory: Cache.Directory = .{ - .path = global_cache_root, - .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + .path = global_cache_arg, + .handle = try cwd.createDirPathOpen(io, global_cache_arg, .{}), }; var graph: Graph = .{ .io = io, .arena = arena, - .cache = .{ - .io = io, - .gpa = gpa, - .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), - .cwd = try process.currentPathAlloc(io, arena), - }, - .zig_exe = zig_exe, + .cache = undefined, + .zig_exe = zig_exe_arg, .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, - .local_cache_root = local_cache_directory, + .local_cache_root = undefined, .zig_lib_directory = zig_lib_directory, - .build_root_directory = build_root_directory, + .build_root_directory = undefined, + .random_seed = parseRandomSeed(seed_arg), }; - graph.cache.addPrefix(.{ .path = null, .handle = cwd }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); + const cmd = stringToEnum(enum { fetch, build }, cmd_name) orelse fatal("bad command name: {q}", .{ cmd_name }); + switch (cmd) { + .fetch => return cmdFetch( gpa, &graph, args[arg_i..]), + .build => {}, + } var step_names: std.ArrayList([]const u8) = .empty; var help_menu = false; var steps_menu = false; - var print_configuration = false; + var print_configuration: enum {none, zon, path} = .none; var override_install_prefix: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = null; var override_bin_dir: ?[]const u8 = null; var override_include_dir: ?[]const u8 = null; + var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(&graph.environ_map); + var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(&graph.environ_map); var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; var summary: ?Summary = null; @@ -201,39 +199,120 @@ pub fn main(init: process.Init.Minimal) !void { var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config = false; var run_args: ?[]const []const u8 = null; + var build_file: ?[]const u8 = null; - if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + var configure_argv: std.ArrayList([]const u8) = .empty; + var cached_passthru_configure: std.ArrayList(u32) = .empty; + var forks: std.ArrayList(Fork) = .empty; + var system_pkg_dir_path: ?[]const u8 = null; + var fetch_only = false; + var fetch_mode: Fetch.JobQueue.Mode = .needed; + var debug_target: ?[]const u8 = null; + var cache_poison: std.Build.Graph.CachePoison = .pure; + + if (EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (stringToEnum(ErrorStyle, str)) |style| { error_style = style; } } - if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + if (EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (stringToEnum(MultilineErrors, str)) |style| { multiline_errors = style; } } - while (nextArg(args, &arg_idx)) |arg| { + try configure_argv.ensureUnusedCapacity(arena, 16); + try cached_passthru_configure.ensureUnusedCapacity(arena, 16); + + _ = configure_argv.addOneAssumeCapacity(); // configurer executable + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", graph.zig_exe }; + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; + const conf_argv_index_build_root = configure_argv.items.len - 1; + + while (nextArg(args, &arg_i)) |arg| { if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try configure_argv.ensureUnusedCapacity(arena, 2); + if (mem.startsWith(u8, arg, "-D") or + mem.startsWith(u8, arg, "-fsys=") or + mem.startsWith(u8, arg, "-fno-sys=") or + mem.startsWith(u8, arg, "--release=") or + mem.eql(u8, arg, "--release")) + { + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--system")) { + system_pkg_dir_path = nextArgOrFatal(args, &arg_i); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. + continue; + } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { + color = stringToEnum(Color, rest) orelse + fatal("expected --color=[auto|on|off]; found {q}", .{arg}); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--cache-poison")) { + cache_poison = .poisoned; + configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); + continue; + } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { + // Allow the configurer process to report parse failure. + if (stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { + cache_poison = poison; + } + configure_argv.appendAssumeCapacity(arg); + continue; + } else if (mem.eql(u8, arg, "--verbose")) { + // Intentionally is added both to make and configure but + // does not go into the cache hash. + configure_argv.appendAssumeCapacity(arg); + } else if (mem.eql(u8, arg, "--search-prefix")) { + const prefix = nextArgOrFatal(args, &arg_i); + // This argument is cache poisonous: it does not go into + // the cache and configurer must set the poison bit when + // choosing to observe it. + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, prefix }; + continue; + } else if (mem.eql(u8, arg, "--cache-dir")) { + override_local_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--pkg-dir")) { + override_pkg_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--fetch")) { + fetch_only = true; + } else if (mem.cutPrefix(u8, arg, "--fetch=")) |rest| { + fetch_only = true; + fetch_mode = stringToEnum(Fetch.JobQueue.Mode, rest) orelse + fatal("expected [needed|all] after \"--fetch=\", found {q}", .{rest}); + } else if (mem.cutPrefix(u8, arg, "--fork=")) |rest| { + try forks.append(arena, .init(rest)); + } else if (mem.eql(u8, arg, "--fork")) { + try forks.append(arena, .init(nextArgOrFatal(args, &arg_i))); + } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { help_menu = true; } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { steps_menu = true; } else if (mem.eql(u8, arg, "--print-configuration")) { - print_configuration = true; + print_configuration = .zon; + } else if (mem.eql(u8, arg, "--print-configuration-path")) { + print_configuration = .path; } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - override_install_prefix = nextArgOrFatal(args, &arg_idx); + override_install_prefix = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--build-file")) { + build_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - override_lib_dir = nextArgOrFatal(args, &arg_idx); + override_lib_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - override_bin_dir = nextArgOrFatal(args, &arg_idx); + override_bin_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - override_include_dir = nextArgOrFatal(args, &arg_idx); + override_include_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--sysroot")) { - graph.sysroot = nextArgOrFatal(args, &arg_idx); + graph.sysroot = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--maxrss")) { - const max_rss_text = nextArgOrFatal(args, &arg_idx); + const max_rss_text = nextArgOrFatal(args, &arg_i); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| fatal("invalid byte size {q}: {t}", .{ max_rss_text, err }); } else if (mem.eql(u8, arg, "--skip-oom-steps")) { @@ -253,7 +332,7 @@ pub fn main(init: process.Init.Minimal) !void { .{ "h", std.time.ns_per_hour }, .{ "hour", std.time.ns_per_hour }, }; - const timeout_str = nextArgOrFatal(args, &arg_idx); + const timeout_str = nextArgOrFatal(args, &arg_i); const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( "invalid timeout {q}: expected unit (ns, us, ms, s, m, h)", .{timeout_str}, @@ -274,50 +353,46 @@ pub fn main(init: process.Init.Minimal) !void { ); test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); } else if (mem.eql(u8, arg, "--search-prefix")) { - try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); + try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_i)); } else if (mem.eql(u8, arg, "--libc")) { - graph.libc_file = nextArgOrFatal(args, &arg_idx); + graph.libc_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected [auto|on|off] after {q}", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { + color = stringToEnum(Color, next_arg) orelse { fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ arg, next_arg, }); }; } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); - error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + error_style = stringToEnum(ErrorStyle, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); - multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + multiline_errors = stringToEnum(MultilineErrors, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected [all|new|failures|line|none] after {q}", .{arg}); - summary = std.meta.stringToEnum(Summary, next_arg) orelse { + summary = stringToEnum(Summary, next_arg) orelse { fatalWithHint("expected [all|new|failures|line|none] after {q}, found {q}", .{ arg, next_arg, }); }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after {q}", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed {q} as unsigned 32-bit integer: {t}", .{ next_arg, err }); - }; + } else if (mem.cutPrefix(u8, arg, "--seed=")) |rest| { + graph.random_seed = parseRandomSeed(rest); } else if (mem.eql(u8, arg, "--build-id")) { graph.build_id = .fast; } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { graph.build_id = std.zig.BuildId.parse(style) catch |err| fatal("unable to parse --build-id style {q}: {t}", .{ style, err }); } else if (mem.eql(u8, arg, "--debounce")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected u16 after {q}", .{arg}); debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { fatal("unable to parse debounce interval {q} as unsigned 16-bit integer: {t}", .{ @@ -333,8 +408,7 @@ pub fn main(init: process.Init.Minimal) !void { fatal("invalid web UI address {q}: {t}", .{ addr_str, err }); }; } else if (mem.eql(u8, arg, "--debug-log")) { - const next_arg = nextArgOrFatal(args, &arg_idx); - try graph.debug_log_scopes.append(arena, next_arg); + try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i)); } else if (mem.eql(u8, arg, "--debug-compile-errors")) { graph.debug_compile_errors = true; } else if (mem.eql(u8, arg, "--debug-incremental")) { @@ -344,19 +418,21 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = .Debug; } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + graph.debug_compiler_runtime_libs = stringToEnum(std.builtin.OptimizeMode, rest) orelse fatal("unrecognized optimization mode: {s}", .{rest}); } else if (is_debug_mode and mem.eql(u8, arg, "--debug-maker-leaks")) { debug_maker_leaks = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--verbose")) { graph.verbose = true; } else if (mem.eql(u8, arg, "--verbose-air")) { graph.verbose_air = true; } else if (mem.eql(u8, arg, "--verbose-cc")) { graph.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-link")) { + graph.verbose_link = true; } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { graph.verbose_llvm_ir = true; } else if (mem.eql(u8, arg, "--watch")) { @@ -439,7 +515,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-reference-trace")) { graph.reference_trace = null; } else if (mem.eql(u8, arg, "--error-limit")) { - const next_arg = nextArgOrFatal(args, &arg_idx); + const next_arg = nextArgOrFatal(args, &arg_i); graph.error_limit = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| fatal("unable to parse error limit {q}: {t}", .{ next_arg, err }); } else if (mem.cutPrefix(u8, arg, "-j")) |text| { @@ -449,7 +525,7 @@ pub fn main(init: process.Init.Minimal) !void { threaded.setAsyncLimit(.limited(n)); graph.max_jobs = n; } else if (mem.eql(u8, arg, "--")) { - run_args = argsRest(args, arg_idx); + run_args = argsRest(args, arg_i); break; } else { fatalWithHint("unrecognized argument: {s}", .{arg}); @@ -459,8 +535,40 @@ pub fn main(init: process.Init.Minimal) !void { } } - const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); - const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| + fatal("resolving current directory path failed: {t}", .{err}); + + const build_root = try findBuildRoot(arena, io, .{ + .cwd_path = cwd_path, + .build_file = build_file, + }); + + graph.build_root_directory = build_root.directory; + graph.local_cache_root = if (override_local_cache_dir) |unresolved_path| std.zig.Directories.openUnresolved( + arena, + io, + cwd_path, + unresolved_path, + .@"local_cache", + ) else .{ + .path = try Dir.path.join(arena, &.{build_root.directory.path orelse ".", default_local_zig_cache_basename}), + .handle = try build_root.directory.handle.createDirPathOpen(io, default_local_zig_cache_basename, .{}), + }; + graph.cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try graph.local_cache_root.handle.createDirPathOpen(io, "h", .{}), + .cwd = cwd_path, + }; + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(graph.build_root_directory); + graph.cache.addPrefix(zig_lib_directory); + graph.cache.addPrefix(graph.local_cache_root); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + const NO_COLOR = EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); graph.stderr_mode = switch (color) { .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), @@ -468,6 +576,555 @@ pub fn main(init: process.Init.Minimal) !void { .off => .no_color, }; + const main_progress_node = std.Progress.start(io, .{ + .disable_printing = (graph.stderr_mode.? == .no_color), + }); + defer main_progress_node.end(); + + { + // Cache lookup for configure options. If we get a match, we can skip + // execution of the configure script. If not, we get the file path to pass + // to the configure process. + var config_man = graph.cache.obtain(); + defer config_man.deinit(); + + for (cached_passthru_configure.items) |i| + config_man.hash.addBytes(configure_argv.items[i]); + + // Prevents a `zig build` from getting a false positive cache hit following + // a `zig build --cache-poison=ignored`. + config_man.hash.add(cache_poison == .ignored); + + // Normally the build runner is compiled for the host target but here is + // some code to help when debugging edits to the build runner so that you + // can make sure it compiles successfully on other targets. + const resolved_target: Package.Module.ResolvedTarget = t: { + if (debug_target) |triple| { + const target_query = try std.Target.Query.parse(.{ .arch_os_abi = triple }); + config_man.hash.addBytes(triple); + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, target_query), + .is_native_os = false, + .is_native_abi = false, + .is_explicit_dynamic_linker = false, + }; + } + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, .{}), + .is_native_os = true, + .is_native_abi = true, + .is_explicit_dynamic_linker = false, + }; + }; + + const pkg_root: Path = if (override_pkg_dir) |p| + .initCwd(p) + else if (system_pkg_dir_path) |p| + .initCwd(p) + else + .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }; + + configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; + + var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; + var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; + + { + // Populate fork_set. + var group: Io.Group = .init; + defer group.cancel(io); + + for (forks.items) |*fork| + group.async(io, Fork.load, .{ io, gpa, fork, color }); + + try group.await(io); + + for (forks.items) |*fork| { + if (fork.failed) process.exit(1); + try fork_set.put(arena, .{ + .path = fork.path, + .manifest_ast = fork.manifest_ast, + .manifest = fork.manifest, + .uses = 0, + }, {}); + } + } + defer Fork.deinitList(forks.items); + + var file_system_inputs: std.ArrayList(u8) = .empty; + defer file_system_inputs.deinit(gpa); + + var build_configurer_argv: std.ArrayList(u8) = .empty; + defer build_configurer_argv.deinit(gpa); + + var dependencies_source: std.ArrayList(u8) = .empty; + defer dependencies_source.deinit(gpa); + + const configurer_root_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "lib/compiler/configurer.zig", + }; + + const root_build_src_path: Cache.Path = .{ + .root_dir = build_root.directory, + .sub_path = build_root.build_zig_basename, + }; + + try build_configurer_argv.appendSlice(gpa, &.{ + graph.zig_exe, "build-exe", // + "--cache-dir", graph.local_cache_root.path orelse ".", // + "--global-cache-dir", graph.global_cache_root.path orelse ".", // + "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // + "--name", "configurer", // + "-fsingle-threaded", // + }); + if (graph.libc_file) |libc_file| { + try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file}); + } + if (graph.reference_trace) |n| { + try build_configurer_argv.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{n})); + } + if (graph.debug_compile_errors) { + try build_configurer_argv.append(gpa, "--debug-compile-errors"); + } + try build_configurer_argv.appendSlice(gpa, &.{ + "--dep", "@build", // + "--dep", "@dependencies", // + try allocPrint(arena, "-Mroot={f}", .{configurer_root_src_path}), // + try allocPrint(arena, "-M@build={f}", .{root_build_src_path}), // + }); + + // In the loop below, after doing the fetch operation, the argv will be + // truncated at this point, dependencies added, and then the + // "--listen=-" arg appended at the end. + const argv_deps_index = build_configurer_argv.items.len - 1; + + //const root_mod = try arena.create(CliModule); + //root_mod.* = .{ + // .name = "root", + // .root_path = try configurer_root_src_path.toString(arena), + //}; + + const build_mod = try arena.create(CliModule); + build_mod.* = .{ + .name = "@build", + .root_path = try root_build_src_path.toString(arena), + }; + defer build_mod.deps.deinit(gpa); + + const deps_mod = try arena.create(CliModule); + deps_mod.* = .{ + .name = "@dependencies", + .root_path = undefined, + }; + defer deps_mod.deps.deinit(gpa); + + // This loop is re-evaluated when the build script exits with an indication that it + // could not continue due to missing lazy dependencies. + const configuration_path: Path, const poisoned: bool = cp: while (true) { + //root_mod.deps.clearRetainingCapacity(); + build_mod.deps.clearRetainingCapacity(); + deps_mod.deps.clearRetainingCapacity(); + + // We want to release all the locks before executing the child process, so we make a nice + // big block here to ensure the cleanup gets run when we extract out our argv. + { + + + + { + const fetch_prog_node = main_progress_node.start("Fetch Packages", 0); + defer fetch_prog_node.end(); + + // Reset fork match counts. + for (fork_set.keys()) |*fork| fork.uses = 0; + + var job_queue: Package.Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = graph.global_cache_root, + .local_storage = &.{ + .cache_root = .{ .root_dir = graph.local_cache_root }, + .pkg_root = pkg_root, + }, + .recursive = true, + .debug_hash = false, + .unlazy_set = unlazy_set, + .fork_set = fork_set, + .mode = fetch_mode, + .prog_node = fetch_prog_node, + .read_only = system_pkg_dir_path != null, + }; + defer job_queue.deinit(); + + if (system_pkg_dir_path == null) { + try http_client.initDefaultProxies(arena, &graph.environ_map); + } + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); + + const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = phantom_package_root }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = phantom_package_root, + .parent_package_root = phantom_package_root, + .parent_manifest_ast = null, + .prog_node = fetch_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = true, + .oom_flag = false, + .latest_commit = null, + + .cli_module = build_mod, + }; + + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), + &fetch, + ); + + job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); + try job_queue.group.await(io); + + { + // Ensure that forks were actually used. This is done + // before printing manifest errors because using a fork can + // prevent them. + var any_unused = false; + for (fork_set.keys()) |*fork| { + if (fork.uses == 0) { + std.log.err("fork {f} matched no {s} packages", .{ + fork.path, fork.manifest.name, + }); + any_unused = true; + } else { + std.log.info("fork {f} matched {d} {s} packages", .{ + fork.path, fork.uses, fork.manifest.name, + }); + } + } + if (any_unused) process.exit(1); + } + + try job_queue.consolidateErrors(); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + // TODO when watching, watch and rebuild configure script rather than exit here + errors.renderToStderr(io, .{}, color) catch {}; + process.exit(1); + } + + if (fetch_only) return cleanExit(io); + + // Create the dependencies.zig file for configurer to + // obtain via `@import("@dependencies")`. + { + { + dependencies_source.clearRetainingCapacity(); + var source_writer: Io.Writer.Allocating = .fromArrayList(&dependencies_source); + defer dependencies_source = source_writer.toArrayList(); + job_queue.createDependenciesSource(&dependencies_source) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; + } + // Atomically create the file in a directory named after the hash of its contents. + var hh: Cache.HashHelper = .{}; + hh.addBytes(builtin.zig_version_string); + hh.addBytes(dependencies_source.items); + const hex_digest = hh.final(); + const dependencies_zig_path: Path = .{ + .root_dir = graph.local_cache_root, + .sub_path = try allocPrint(arena, "o/{s}/dependencies.zig", .{ &hex_digest }), + }; + var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic( + io, + dependencies_zig_path.sub_path, .{ .make_path = true, .replace = true }, + ); + defer atomic_file.deinit(io); + atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch |err| + fatal("writing dependencies.zig contents: {t}", .{err}); + atomic_file.replace(io) catch |err| + fatal("replacing {f}: {t}", .{dependencies_zig_path, err}); + + deps_mod.root_path = try dependencies_zig_path.toString(arena); + } + + { + // Add a CliModule for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(gpa, @intCast(hashes.len)); + for (hashes, fetches) |*hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + if (!f.has_build_zig) + continue; + const hash_slice = try arena.dupe(u8, hash.toSlice()); + + const m = try arena.create(CliModule); + m.* = .{ + .root_path = try f.package_root.toString(arena), + .name = hash_slice, + }; + deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m); + f.cli_module = m; + } + + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.cli_module orelse continue; + if (!f.have_manifest) continue; + const man = &f.manifest; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(gpa, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + global_cache_directory, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } + } + } + + // Lower module dependencies to CLI argv. + build_configurer_argv.shrinkRetainingCapacity(argv_deps_index); + for (deps_mod.deps.values()) |dep| { + try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1); + for (dep.deps.values()) |sub| { + build_configurer_argv.appendAssumeCapacity("--dep"); + build_configurer_argv.appendAssumeCapacity(sub.name); + } + build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}", .{ + dep.name, dep.root_path, + })); + } + try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * deps_mod.deps.count() + 1); + for (deps_mod.deps.values()) |dep| { + build_configurer_argv.appendAssumeCapacity("--dep"); + build_configurer_argv.appendAssumeCapacity(dep.name); + } + build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M@dependencies={s}", .{ + deps_mod.root_path, + })); + } + + const compile_prog_node = main_progress_node.start("Compile Configure Script", 0); + defer compile_prog_node.end(); + + try build_configurer_argv.append(gpa, "--listen=-"); + + file_system_inputs.clearRetainingCapacity(); + execute_child(build_configurer_argv, &file_system_inputs); + + const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); + const exe_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), + }; + _ = try config_man.addFilePath(exe_path, null); + configure_argv.items[0] = try exe_path.toString(arena); + + switch (cache_poison) { + .pure, .disallowed, .ignored => if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .{ + .root_dir = dirs.local_cache, + .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), + }, + false, + }; + }, + .poisoned => {}, // Don't bother checking for cache hit. + } + } + + if (!process.can_spawn) { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); + } + + const rand_int = randInt(io, u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + const config_tmp_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = tmp_dir_sub_path, + }; + const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( + io, + config_tmp_path.sub_path, + .{ .read = true, .exclusive = true }, + ); + defer config_tmp_file.close(io); + + const term = term: { + const child_node = main_progress_node.start("Run Configure Script", 0); + defer child_node.end(); + var child = std.process.spawn(io, .{ + .argv = configure_argv.items, + .stdout = .{ .file = config_tmp_file }, + .progress_node = child_node, + }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err }); + defer child.kill(io); + break :term child.wait(io) catch |err| + fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err }); + }; + if (!term.success()) { + // Failure to produce the configuration file. + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command {f}:\n{s}", .{ term, cmd }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); + + if (configuration.unlazy_deps.len != 0) { + if (!dev.env.supports(.fetch_command)) process.exit(1); + var any_errors = false; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); + any_errors = true; + continue; + } + try unlazy_set.put(arena, .fromSlice(hash), {}); + } + if (any_errors) process.exit(1); + if (system_pkg_dir_path) |p| { + // In this mode, the system needs to provide these packages; they + // cannot be fetched by Zig. + const s = fs.path.sep_str; + for (unlazy_set.keys()) |*hash| { + std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); + } + std.log.info("remote package fetching disabled due to --system mode", .{}); + std.log.info("dependencies might be avoidable depending on build configuration", .{}); + process.exit(1); + } + continue :cp; + } + + for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { + const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; + try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + } + + // We need to add to the configuration cache the source files of + // configurer itself, so that the maker process can watch the file system + // for those changes and restart itself. By doing this, we make it + // possible to bypass creating a Compilation for configurer on + // Configuration cache hit. + { + var it = mem.splitScalar(u8, file_system_inputs.items, 0); + while (it.next()) |input| { + _ = try config_man.addPrefixedPathPost(.{ + .prefix = input[0], + .sub_path = input[1..], + }); + } + } + + // If it is poisoned, there is no point in moving it to cached + // location. Just leave it in the tmp directory. + if (configuration.poisoned) { + break :cp .{ config_tmp_path, true }; + } else { + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), + }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| retry: { + const e = switch (err) { + error.FileNotFound => e: { + const dir_path = final_path.dirname().?; + dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| + fatal("failed to create directory {f}: {t}", .{ dir_path, e }); + if (Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + )) |_| break :retry else |e| break :e e; + }, + else => |e| e, + }; + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, e, + }); + }; + config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + break :cp .{ final_path, false }; + } + }; + + { + // Release all file system locks just before running the maker process. + var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; + defer if (configuration_lock) |*l| l.release(io); + + if (print_configuration_path) { + var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); + stdout_writer.interface.print("{f}\n", .{configuration_path}) catch + fatal("failed printing cache file path: {t}", .{stdout_writer.err.?}); + stdout_writer.flush() catch |err| + fatal("failed printing cache file path: {t}", .{err}); + return cleanExit(io); + } + const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); + + make_argv.items[0] = try make_runner.exe_path.toString(arena); + make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); + } + } + const scanned_config: ScannedConfig = sc: { const configuration = c: { var file = cwd.openFile(io, configure_path, .{}) catch |err| @@ -505,7 +1162,7 @@ pub fn main(init: process.Init.Minimal) !void { }; if (help_menu) { - var w = initStdoutWriter(io); + const w = initStdoutWriter(io); scanned_config.printUsage(&graph, w) catch |err| switch (err) { error.WriteFailed => return stdout_writer_allocation.err.?, else => |e| return e, @@ -513,18 +1170,24 @@ pub fn main(init: process.Init.Minimal) !void { w.flush() catch return stdout_writer_allocation.err.?; return cleanExit(io, &scanned_config); } else if (steps_menu) { - var w = initStdoutWriter(io); + const w = initStdoutWriter(io); scanned_config.printSteps(&graph, w) catch |err| switch (err) { error.WriteFailed => return stdout_writer_allocation.err.?, else => |e| return e, }; w.flush() catch return stdout_writer_allocation.err.?; return cleanExit(io, &scanned_config); - } else if (print_configuration) { - var w = initStdoutWriter(io); - scanned_config.print(w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return cleanExit(io, &scanned_config); + } else switch (print_configuration) { + .none => {}, + .zon => { + const w = initStdoutWriter(io); + scanned_config.print(w) catch return stdout_writer_allocation.err.?; + w.flush() catch return stdout_writer_allocation.err.?; + return cleanExit(io, &scanned_config); + }, + .path => { + @panic("TODO"); + }, } if (webui_listen != null) { @@ -532,11 +1195,6 @@ pub fn main(init: process.Init.Minimal) !void { if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); } - const main_progress_node = std.Progress.start(io, .{ - .disable_printing = (graph.stderr_mode.? == .no_color), - }); - defer main_progress_node.end(); - const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ .root_dir = .cwd(), .sub_path = try Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), @@ -706,6 +1364,338 @@ pub fn main(init: process.Init.Minimal) !void { } } +fn cmdFetch( + gpa: Allocator, + graph: *Graph, + args: []const []const u8 +) !void { + const environ_map = &graph.environ_map; + const io = graph.io; + const arena = graph.arena; + + const color: Color = Color.settingFromEnvironment(environ_map); + var opt_path_or_url: ?[]const u8 = null; + var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); + var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); + var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); + var debug_hash: bool = false; + var save: union(enum) { + no, + yes: ?[]const u8, + exact: ?[]const u8, + } = .no; + + var arg_i: usize = 0; + while (nextArg(args, &arg_i)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try Io.File.stdout().writeStreamingAll(io, usage_fetch); + return cleanExit(io); + } else if (mem.eql(u8, arg, "--global-cache-dir")) { + override_global_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--cache-dir")) { + override_local_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--pkg-dir")) { + override_pkg_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--debug-hash")) { + debug_hash = true; + } else if (mem.eql(u8, arg, "--debug-log")) { + try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i)); + } else if (mem.eql(u8, arg, "--save")) { + save = .{ .yes = null }; + } else if (mem.cutPrefix(u8, arg, "--save=")) |rest| { + save = .{ .yes = rest }; + } else if (mem.eql(u8, arg, "--save-exact")) { + save = .{ .exact = null }; + } else if (mem.cutPrefix(u8, arg, "--save-exact=")) |rest| { + save = .{ .exact = rest }; + } else { + fatal("unrecognized parameter: {q}", .{arg}); + } + } else if (opt_path_or_url != null) { + fatal("unexpected extra parameter: {q}", .{arg}); + } else { + opt_path_or_url = arg; + } + } + + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); + + var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + try http_client.initDefaultProxies(arena, environ_map); + + var root_prog_node = std.Progress.start(io, .{ + .root_name = "Fetch", + }); + defer root_prog_node.end(); + + var local_storage: Fetch.LocalStorage = undefined; + var build_root: BuildRoot = undefined; + var build_root_initialized = false; + defer if (build_root_initialized) build_root.deinit(io); + + const cwd_path = try std.zig.getResolvedCwd(io, arena); + + const local_storage_ptr = switch (save) { + .no => null, + .yes, .exact => ls: { + build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path }); + build_root_initialized = true; + + local_storage = .{ + .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{ + .root_dir = build_root.directory, + .sub_path = ".zig-cache", + }, + .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }, + }; + + break :ls &local_storage; + }, + }; + + var job_queue: Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = graph.global_cache_root, + .local_storage = local_storage_ptr, + .recursive = false, + .read_only = false, + .debug_hash = debug_hash, + .mode = .all, + .prog_node = root_prog_node, + }; + defer job_queue.deinit(); + + var fetch: Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = undefined, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + + .module = null, + }; + defer fetch.deinit(); + + fetch.run() catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.FetchFailed => {}, // error bundle checked below + }; + + try job_queue.group.await(io); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + process.exit(1); + } + + const package_hash = fetch.computedPackageHash(); + const package_hash_slice = package_hash.toSlice(); + + root_prog_node.end(); + root_prog_node = .{ .index = .none }; + + const name = switch (save) { + .no => { + var data: [2][]const u8 = .{ package_hash_slice, "\n" }; + const w = initStdoutWriter(); + try w.writeVecAll(&data); + try w.flush(); + return cleanExit(io); + }, + .yes, .exact => |name| name: { + if (name) |n| break :name n; + if (!fetch.have_manifest) + fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); + break :name fetch.manifest.name; + }, + }; + + // The name to use in case the manifest file needs to be created now. + const init_root_name = Dir.path.basename(build_root.directory.path orelse cwd_path); + var manifest, var ast = try loadManifest(gpa, arena, io, .{ + .root_name = try sanitizeExampleName(arena, init_root_name), + .dir = build_root.directory.handle, + .color = color, + }); + defer { + manifest.deinit(gpa); + ast.deinit(gpa); + } + + var fixups: Ast.Render.Fixups = .{}; + defer fixups.deinit(gpa); + + var saved_path_or_url = path_or_url; + + if (fetch.latest_commit) |latest_commit| resolved: { + const latest_commit_hex = try allocPrint(arena, "{f}", .{latest_commit}); + + var uri = try std.Uri.parse(path_or_url); + + if (uri.fragment) |fragment| { + const target_ref = try fragment.toRawMaybeAlloc(arena); + + // the refspec may already be fully resolved + if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; + + std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex }); + + // include the original refspec in a query parameter, could be used to check for updates + uri.query = .{ .percent_encoded = try allocPrint(arena, "ref={f}", .{ + std.fmt.alt(fragment, .formatEscaped), + }) }; + } else { + std.log.info("resolved to commit {s}", .{latest_commit_hex}); + } + + // replace the refspec with the resolved commit SHA + uri.fragment = .{ .raw = latest_commit_hex }; + + switch (save) { + .yes => saved_path_or_url = try allocPrint(arena, "{f}", .{uri}), + .no, .exact => {}, // keep the original URL + } + } + + const new_node_init = try allocPrint(arena, + \\.{{ + \\ .url = "{f}", + \\ .hash = "{f}", + \\ }} + , .{ + std.zig.fmtString(saved_path_or_url), + std.zig.fmtString(package_hash_slice), + }); + + const new_node_text = try allocPrint(arena, ".{f} = {s},\n", .{ + std.zig.fmtIdPU(name), new_node_init, + }); + + const dependencies_init = try allocPrint(arena, ".{{\n {s} }}", .{ + new_node_text, + }); + + const dependencies_text = try allocPrint(arena, ".dependencies = {s},\n", .{ + dependencies_init, + }); + + if (manifest.dependencies.get(name)) |dep| { + if (dep.hash) |h| { + switch (dep.location) { + .url => |u| { + if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { + std.log.info("existing dependency named {q} is up-to-date", .{name}); + process.exit(0); + } + }, + .path => {}, + } + } + + const location_replace = try allocPrint( + arena, + "\"{f}\"", + .{std.zig.fmtString(saved_path_or_url)}, + ); + const hash_replace = try allocPrint( + arena, + "\"{f}\"", + .{std.zig.fmtString(package_hash_slice)}, + ); + + warn("overwriting existing dependency named {q}", .{name}); + try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); + if (dep.hash_node.unwrap()) |hash_node| { + try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); + } else { + // https://github.com/ziglang/zig/issues/21690 + } + } else if (manifest.dependencies.count() > 0) { + // Add fixup for adding another dependency. + const deps = manifest.dependencies.values(); + const last_dep_node = deps[deps.len - 1].node; + try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); + } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { + // Add fixup for replacing the entire dependencies struct. + try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); + } else { + // Add fixup for adding dependencies struct. + try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); + } + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + try ast.render(gpa, &aw.writer, fixups); + const rendered = aw.written(); + + build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch |err| { + fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err }); + }; + + return cleanExit(io); +} + +const usage_fetch = + \\Usage: zig fetch [options] + \\Usage: zig fetch [options] + \\ + \\ Copy a package into the global cache and print its hash. + \\ must point to one of the following: + \\ - A git+http / git+https server for the package + \\ - A tarball file (with or without compression) containing + \\ package source + \\ - A git bundle file containing package source + \\ + \\Examples: + \\ + \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git + \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --cache-dir [path] Override path to local cache directory + \\ --pkg-dir [path] Override path to local package directory + \\ --debug-hash Print verbose hash information to stdout + \\ --debug-log [scope] Enable printing debug/info log messages for scope + \\ --save Add the fetched package to build.zig.zon + \\ --save=[name] Add the fetched package to build.zig.zon as name + \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim + \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim + \\ +; + +fn cmdBuild() !void { + +} + fn markFailedStepsDirty(maker: *Maker) void { const all_steps = maker.step_stack.keys(); @@ -1677,16 +2667,13 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { } fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); - }; + return nextArg(args, idx) orelse fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); } -fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { - const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); - if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); - const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); - return arg; +fn prefixedArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, prefix: []const u8) []const u8 { + const arg = args[index_ptr.*]; + if (mem.cutPrefix(u8, arg, prefix)) |rest| return rest; + fatal("expected {q} to begin with {q}", .{arg, prefix}); } fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { @@ -2006,11 +2993,11 @@ pub fn installSymLinks( const name = conf_comp.root_name.slice(c); const filename_major_only, const filename_name_only = if (os_tag.isDarwin()) .{ - try std.fmt.allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }), - try std.fmt.allocPrint(arena, "lib{s}.dylib", .{name}), + try allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }), + try allocPrint(arena, "lib{s}.dylib", .{name}), } else .{ - try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }), - try std.fmt.allocPrint(arena, "lib{s}.so", .{name}), + try allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }), + try allocPrint(arena, "lib{s}.so", .{name}), }; return installSymLinksInner(maker, arena, output_path, asking_step_index, filename_major_only, filename_name_only); @@ -2059,3 +3046,229 @@ inline fn debugMakerLeaks() bool { if (!is_debug_mode) return false; return debug_maker_leaks; } + +const BuildRoot = struct { + directory: Cache.Directory, + build_zig_basename: []const u8, + cleanup_build_dir: ?Io.Dir, + + fn deinit(br: *BuildRoot, io: Io) void { + if (br.cleanup_build_dir) |*dir| dir.close(io); + br.* = undefined; + } +}; + +const FindBuildRootOptions = struct { + build_file: ?[]const u8 = null, + cwd_path: ?[]const u8 = null, +}; + +fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot { + const cwd_path = options.cwd_path orelse try std.zig.getResolvedCwd(io, arena); + const build_zig_basename = if (options.build_file) |bf| + Dir.path.basename(bf) + else + std.zig.build_zig_basename; + + if (options.build_file) |bf| { + if (Dir.path.dirname(bf)) |dirname| { + const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { + fatal("failed opening directory containing {q}: {t}", .{ bf, err }); + }; + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ .path = dirname, .handle = dir }, + .cleanup_build_dir = dir, + }; + } + + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ .path = null, .handle = Io.Dir.cwd() }, + .cleanup_build_dir = null, + }; + } + // Search up parent directories until we find build.zig. + var dirname: []const u8 = cwd_path; + while (true) { + const joined_path = try Dir.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); + if (Io.Dir.cwd().access(io, joined_path, .{})) |_| { + const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { + fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err }); + }; + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ + .path = dirname, + .handle = dir, + }, + .cleanup_build_dir = dir, + }; + } else |err| switch (err) { + error.FileNotFound => { + dirname = Dir.path.dirname(dirname) orelse { + std.log.info("initialize {s} template file with \"zig init\"", .{ std.zig.build_zig_basename }); + std.log.info("see \"zig --help\" for more options", .{}); + fatal("no build.zig file found, in the current directory or any parent directories", .{}); + }; + continue; + }, + else => |e| return e, + } + } +} + +const Fork = struct { + path: Path, + manifest_ast: std.zig.Ast, + manifest: Package.Manifest, + error_bundle: std.zig.ErrorBundle.Wip, + failed: bool, + arena_allocator: std.heap.ArenaAllocator, + + fn init(cwd_relative_path: []const u8) Fork { + return .{ + .manifest_ast = undefined, + .manifest = undefined, + .error_bundle = undefined, + .arena_allocator = undefined, + .path = .{ + .root_dir = .cwd(), + .sub_path = cwd_relative_path, + }, + .failed = false, + }; + } + + fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { + loadFallible(io, gpa, fork, color) catch |err| switch (err) { + error.Canceled => |e| return e, + error.AlreadyReported => fork.failed = true, + else => |e| { + std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); + fork.failed = true; + }, + }; + } + + fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void { + fork.arena_allocator = .init(gpa); + const arena = fork.arena_allocator.allocator(); + + var error_bundle: std.zig.ErrorBundle.Wip = undefined; + try error_bundle.init(gpa); + defer error_bundle.deinit(); + + const manifest_path = try fork.path.join(arena, Package.Manifest.basename); + + Package.Manifest.load( + io, + arena, + manifest_path, + &fork.manifest_ast, + &error_bundle, + &fork.manifest, + true, + ) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ErrorsBundled => { + assert(error_bundle.root_list.items.len > 0); + var errors = try error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + return error.AlreadyReported; + }, + else => |e| { + std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); + return error.AlreadyReported; + }, + }; + } + + fn deinitList(forks: []Fork) void { + for (forks) |*fork| fork.arena_allocator.deinit(); + } +}; + +fn parseRandomSeed(arg: []const u8) u32 { + return std.fmt.parseUnsigned(u32, arg, 0) catch |err| + fatal("failed parsing random seed {q} as unsigned 32-bit integer: {t}", .{ arg, err }); +} + +fn randInt(io: Io, comptime T: type) T { + var x: T = undefined; + io.random(@ptrCast(&x)); + return x; +} + +const LoadManifestOptions = struct { + root_name: []const u8, + dir: Io.Dir, + color: Color, +}; + +fn loadManifest( + gpa: Allocator, + arena: Allocator, + io: Io, + options: LoadManifestOptions, +) !struct { Package.Manifest, std.zig.Ast } { + const rng: std.Random.IoSource = .{ .io = io }; + + const manifest_bytes = while (true) { + break options.dir.readFileAllocOptions( + io, + Package.Manifest.basename, + arena, + .limited(Package.Manifest.max_bytes), + .@"1", + 0, + ) catch |err| switch (err) { + error.FileNotFound => { + writeSimpleTemplateFile(io, Package.Manifest.basename, + \\.{{ + \\ .name = .{s}, + \\ .version = "{s}", + \\ .paths = .{{""}}, + \\ .fingerprint = 0x{x}, + \\}} + \\ + , .{ + options.root_name, + build_options.version, + Package.Fingerprint.generate(rng.interface(), options.root_name).int(), + }) catch |e| { + fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e }); + }; + continue; + }, + else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }), + }; + }; + var ast = try Ast.parse(gpa, manifest_bytes, .zon); + errdefer ast.deinit(gpa); + + if (ast.errors.len > 0) { + try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color); + process.exit(2); + } + + var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); + errdefer manifest.deinit(gpa); + + if (manifest.errors.len > 0) { + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + + const src_path = try wip_errors.addString(Package.Manifest.basename); + try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); + + var error_bundle = try wip_errors.toOwnedBundle(""); + defer error_bundle.deinit(gpa); + error_bundle.renderToStderr(io, .{}, options.color) catch {}; + + process.exit(2); + } + return .{ manifest, ast }; +} + diff --git a/src/Package/Fetch.zig b/lib/compiler/Maker/Fetch.zig similarity index 99% rename from src/Package/Fetch.zig rename to lib/compiler/Maker/Fetch.zig index 20e19cdaa7ad2e55b9ddb07f490df1ffdaa26eea..b6093f59b6992efba93f8327348b1145c905c41b 100644 --- a/src/Package/Fetch.zig +++ b/lib/compiler/Maker/Fetch.zig @@ -96,7 +96,10 @@ latest_commit: ?git.Oid, /// The module for this `Fetch` tasks's package, which exposes `build.zig` as /// the root source file. -module: ?*Package.Module, +/// +/// This could be an opaque "userdata" field because this code does not observe +/// this data in any way but let's have some type safety because we can. +cli_module: ?*@import("../Maker.zig").CliModule, pub const LazyStatus = enum { /// Not lazy. @@ -227,16 +230,16 @@ pub const JobQueue = struct { /// Creates the dependencies.zig source code for the build runner to obtain /// via `@import("@dependencies")`. - pub fn createDependenciesSource(jq: *JobQueue, buf: *std.array_list.Managed(u8)) Allocator.Error!void { + pub fn createDependenciesSource(jq: *JobQueue, w: *Io.Writer) Io.Writer.Error!void { const keys = jq.table.keys(); assert(keys.len != 0); // caller should have added the first one if (keys.len == 1) { // This is the first one. It must have no dependencies. - return createEmptyDependenciesSource(buf); + return createEmptyDependenciesSource(w); } - try buf.appendSlice("pub const packages = struct {\n"); + try w.writeAll("pub const packages = struct {\n"); // Ensure the generated .zig file is deterministic. jq.table.sortUnstable(@as(struct { @@ -254,7 +257,7 @@ pub const JobQueue = struct { const hash_slice = hash.toSlice(); - try buf.print( + try w.print( \\ pub const {f} = struct {{ \\ , .{std.zig.fmtId(hash_slice)}); @@ -263,14 +266,14 @@ pub const JobQueue = struct { switch (fetch.lazy_status) { .eager => break :lazy, .available => { - try buf.appendSlice( + try w.writeAll( \\ pub const available = true; \\ ); break :lazy; }, .unavailable => { - try buf.appendSlice( + try w.writeAll( \\ pub const available = false; \\ }; \\ @@ -280,13 +283,13 @@ pub const JobQueue = struct { } } - try buf.print( + try w.print( \\ pub const build_root = "{f}"; \\ , .{std.fmt.alt(fetch.package_root, .formatEscapeString)}); if (fetch.has_build_zig) { - try buf.print( + try w.print( \\ pub const build_zig = @import("{f}"); \\ , .{std.zig.fmtString(hash_slice)}); @@ -294,25 +297,25 @@ pub const JobQueue = struct { if (fetch.have_manifest) { const manifest = &fetch.manifest; - try buf.appendSlice( + try w.writeAll( \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ \\ ); for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.print( + try w.print( " .{{ \"{f}\", \"{f}\" }},\n", .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, ); } - try buf.appendSlice( + try w.writeAll( \\ }; \\ }; \\ ); } else { - try buf.appendSlice( + try w.writeAll( \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; \\ }; \\ @@ -320,7 +323,7 @@ pub const JobQueue = struct { } } - try buf.appendSlice( + try w.writeAll( \\}; \\ \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ @@ -333,16 +336,16 @@ pub const JobQueue = struct { for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.print( + try w.print( " .{{ \"{f}\", \"{f}\" }},\n", .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, ); } - try buf.appendSlice("};\n"); + try w.appendSlice("};\n"); } - pub fn createEmptyDependenciesSource(buf: *std.array_list.Managed(u8)) Allocator.Error!void { - try buf.appendSlice( + pub fn createEmptyDependenciesSource(w: *Io.Writer) Io.Writer!void { + try w.writeAll( \\pub const packages = struct {}; \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; \\ @@ -1020,7 +1023,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .oom_flag = false, .latest_commit = null, - .module = null, + .cli_module = null, }; } diff --git a/src/Package/Fetch/git.zig b/lib/compiler/Maker/Fetch/git.zig similarity index 100% rename from src/Package/Fetch/git.zig rename to lib/compiler/Maker/Fetch/git.zig diff --git a/src/Package/Fetch/git/testdata/testrepo-sha1.idx b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.idx similarity index 100% rename from src/Package/Fetch/git/testdata/testrepo-sha1.idx rename to lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.idx diff --git a/src/Package/Fetch/git/testdata/testrepo-sha1.pack b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.pack similarity index 100% rename from src/Package/Fetch/git/testdata/testrepo-sha1.pack rename to lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.pack diff --git a/src/Package/Fetch/git/testdata/testrepo-sha256.idx b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.idx similarity index 100% rename from src/Package/Fetch/git/testdata/testrepo-sha256.idx rename to lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.idx diff --git a/src/Package/Fetch/git/testdata/testrepo-sha256.pack b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.pack similarity index 100% rename from src/Package/Fetch/git/testdata/testrepo-sha256.pack rename to lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.pack diff --git a/src/Package.zig b/lib/compiler/Maker/Package.zig similarity index 98% rename from src/Package.zig rename to lib/compiler/Maker/Package.zig index 8fb9995bd81315343e9b1da8d1741774ae4e9d82..01bcf01036acc109c288727cf6099e9f0d65e94c 100644 --- a/src/Package.zig +++ b/lib/compiler/Maker/Package.zig @@ -1,9 +1,7 @@ const std = @import("std"); const assert = std.debug.assert; -pub const Module = @import("Package/Module.zig"); pub const Fetch = @import("Package/Fetch.zig"); -pub const build_zig_basename = "build.zig"; pub const Manifest = @import("Package/Manifest.zig"); pub const Fingerprint = packed struct(u64) { diff --git a/src/Package/Manifest.zig b/lib/compiler/Maker/Package/Manifest.zig similarity index 100% rename from src/Package/Manifest.zig rename to lib/compiler/Maker/Package/Manifest.zig diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 809bb9e6263c3a9c0e128bb6a677589fef6054c9..a7b7ae44274356702ffe34923d39405d23715f11 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -633,7 +633,8 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { var s: Serialize = .{ .wc = wc, .arena = arena }; try wc.path_deps.ensureTotalCapacityPrecise(gpa, graph.configure_dependencies.items.len); - for ( + // TODO remove this + if (false) for ( graph.configure_dependencies.items, wc.path_deps.addManyAsSliceAssumeCapacity(graph.configure_dependencies.items.len), ) |src, *dest| { @@ -661,7 +662,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { .dependency => |d| .init(try s.builderToPackage(d.dependency.builder)), }, }; - } + }; // Starting from all top-level steps in `b`, traverse the entire step graph // and add all step dependencies implied by module graphs. diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 0880d2abeaa9e4fa9580673b3a16f028c2b3d0f4..68f5cfc49af5e8e48724db42455115313108f9e1 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1035,7 +1035,7 @@ pub const Manifest = struct { pub fn addPathPost(man: *Manifest, path: Path) !void { _ = man; _ = path; - @panic("TODO"); + std.log.err("TODO Build.Cache.addPathPost", .{}); } /// Like `addFilePost` but when the file contents have already been loaded from disk. diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 8e36cad974a76f24ef6e12f08802995c2b748544..72b876e577296b359ba86cee7df7d2fe65a8bb92 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -1881,7 +1881,7 @@ pub const PathDep = extern struct { _ = c; _ = arena; _ = path; - @panic("TODO"); + std.log.err("TODO Configuration.PathDep.toCachePath", .{}); } }; diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 05891c582e38646da978177e34a3e8b26ac7f0af..2e1e0769436f003a829af900cde18c3b9f440236 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -2,12 +2,17 @@ //! source lives here. These APIs are provided as-is and have absolutely no API //! guarantees whatsoever. +const builtin = @import("builtin"); + const std = @import("std.zig"); const assert = std.debug.assert; const mem = std.mem; const Allocator = std.mem.Allocator; const Io = std.Io; const Writer = std.Io.Writer; +const Cache = std.Build.Cache; +const fatal = std.process.fatal; +const Dir = std.Io.Dir; const tokenizer = @import("zig/tokenizer.zig"); @@ -47,6 +52,9 @@ pub const c_translation = struct { pub const helpers = @import("zig/c_translation/helpers.zig"); }; +pub const default_local_zig_cache_basename = ".zig-cache"; +pub const build_zig_basename = "build.zig"; + pub const SrcHasher = std.crypto.hash.Blake3; pub const SrcHash = [16]u8; @@ -70,7 +78,7 @@ pub const Color = enum { /// CLICOLOR_FORCE environment variables. Color is always disabled on WASI per /// https://github.com/WebAssembly/WASI/issues/162 pub fn settingFromEnvironment(environ_map: *const std.process.Environ.Map) Color { - return if (@import("builtin").os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map)) + return if (builtin.os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map)) .off else if (EnvVar.CLICOLOR_FORCE.isSet(environ_map)) .on @@ -163,8 +171,8 @@ pub const BinNameOptions = struct { os_tag: std.Target.Os.Tag, ofmt: std.Target.ObjectFormat, abi: std.Target.Abi, - output_mode: std.builtin.OutputMode, - link_mode: ?std.builtin.LinkMode = null, + output_mode: std.lang.OutputMode, + link_mode: ?std.lang.LinkMode = null, version: ?std.SemanticVersion = null, }; @@ -512,7 +520,7 @@ pub const FormatId = struct { pub fn format(ctx: FormatId, writer: *Writer) Writer.Error!void { const bytes = ctx.bytes; if (isValidId(bytes) and - (ctx.flags.allow_primitive or !std.zig.isPrimitive(bytes)) and + (ctx.flags.allow_primitive or !isPrimitive(bytes)) and (ctx.flags.allow_underscore or !isUnderscore(bytes))) { return writer.writeAll(bytes); @@ -592,7 +600,7 @@ pub fn isValidId(bytes: []const u8) bool { else => return false, } } - return std.zig.Token.getKeyword(bytes) == null; + return Token.getKeyword(bytes) == null; } test isValidId { @@ -658,7 +666,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *Io.File.Reader) ![ } pub fn printAstErrorsToStderr(gpa: Allocator, io: Io, tree: Ast, path: []const u8, color: Color) !void { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; + var wip_errors: ErrorBundle.Wip = undefined; try wip_errors.init(gpa); defer wip_errors.deinit(); @@ -673,7 +681,7 @@ pub fn putAstErrorsIntoBundle( gpa: Allocator, tree: Ast, path: []const u8, - wip_errors: *std.zig.ErrorBundle.Wip, + wip_errors: *ErrorBundle.Wip, ) Allocator.Error!void { switch (tree.mode) { .zig => { @@ -692,7 +700,7 @@ pub fn putAstErrorsIntoBundle( } pub fn resolveTargetQueryOrFatal(io: Io, target_query: std.Target.Query) std.Target { - return std.zig.system.resolveTargetQuery(io, target_query) catch |err| + return system.resolveTargetQuery(io, target_query) catch |err| std.process.fatal("unable to resolve target: {t}", .{err}); } @@ -1242,6 +1250,365 @@ pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPri return aw.toOwnedSlice(); } +/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This +/// means the path has no repeated separators, no "." or ".." components, and no trailing separator. +/// On WASI, "" is returned instead of ".". +pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 { + if (builtin.os.tag == .wasi) { + if (std.debug.runtime_safety) { + const cwd = try std.process.currentPathAlloc(io, gpa); + defer gpa.free(cwd); + assert(mem.eql(u8, cwd, ".")); + } + return ""; + } + const cwd = try std.process.currentPathAlloc(io, gpa); + defer gpa.free(cwd); + const resolved = try Dir.path.resolve(gpa, &.{cwd}); + assert(Dir.path.isAbsolute(resolved)); + return resolved; +} + +pub const Directories = struct { + /// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path, + /// but on WASI is the empty string "" instead, because WASI does not have absolute paths. + cwd: []const u8, + /// The Zig 'lib' directory. + /// `zig_lib.path` is resolved (`resolvePath`) or `null` for cwd. + /// Guaranteed to be a different path from `global_cache` and `local_cache`. + zig_lib: Cache.Directory, + /// The global Zig cache directory. + /// `global_cache.path` is resolved (`resolvePath`) or `null` for cwd. + global_cache: Cache.Directory, + /// The local Zig cache directory. + /// `local_cache.path` is resolved (`resolvePath`) or `null` for cwd. + /// This may be the same as `global_cache`. + local_cache: Cache.Directory, + + pub fn deinit(dirs: *Directories, io: Io) void { + // The local and global caches could be the same. + const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle; + + dirs.global_cache.handle.close(io); + if (close_local) dirs.local_cache.handle.close(io); + dirs.zig_lib.handle.close(io); + } + + /// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for + /// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it + /// shares handles with `dirs`. + pub fn withoutLocalCache(dirs: Directories) Directories { + return .{ + .cwd = dirs.cwd, + .zig_lib = dirs.zig_lib, + .global_cache = dirs.global_cache, + .local_cache = dirs.global_cache, + }; + } + + const LocalCacheStrategy = union(enum) { + override: []const u8, + search, + global, + }; + + /// Uses `std.process.fatal` on error conditions. + pub fn init( + arena: Allocator, + io: Io, + override_zig_lib: ?[]const u8, + override_global_cache: ?[]const u8, + local_cache_strat: LocalCacheStrategy, + preopens: std.process.Preopens, + self_exe_path: switch (builtin.target.os.tag) { + .wasi => void, + else => []const u8, + }, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, + ) Directories { + const wasi = builtin.target.os.tag == .wasi; + + const zig_lib: Cache.Directory = d: { + if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); + if (wasi) break :d getPreopen(preopens, "/lib"); + break :d findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch |err| { + fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err }); + }; + }; + + const global_cache: Cache.Directory = d: { + if (override_global_cache) |path| break :d openUnresolved(arena, io, cwd, path, .@"global cache"); + if (wasi) break :d getPreopen(preopens, "/cache"); + const path = resolveGlobalCacheDir(arena, environ_map) catch |err| { + fatal("unable to resolve zig cache directory: {t}", .{err}); + }; + break :d openUnresolved(arena, io, cwd, path, .@"global cache"); + }; + + const local_cache = getLocalCacheDirectory(arena, io, cwd, global_cache, local_cache_strat); + + if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) { + fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache }); + } + if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) { + fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache }); + } + + return .{ + .cwd = cwd, + .zig_lib = zig_lib, + .global_cache = global_cache, + .local_cache = local_cache, + }; + } + + fn getLocalCacheDirectory( + arena: Allocator, + io: Io, + cwd: []const u8, + global_cache: Cache.Directory, + local_cache_strat: LocalCacheStrategy, + ) Cache.Directory { + return switch (local_cache_strat) { + .override => |path| openUnresolved(arena, io, cwd, path, .@"local cache"), + .search => d: { + const maybe_path = resolveSuitableLocalCacheDir(arena, io, cwd) catch |err| + fatal("unable to resolve zig cache directory: {t}", .{err}); + const path = maybe_path orelse break :d global_cache; + break :d openUnresolved(arena, io, cwd, path, .@"local cache"); + }, + .global => global_cache, + }; + } + + fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory { + return .{ + .path = if (std.mem.eql(u8, name, ".")) null else name, + .handle = switch (preopens.get(name) orelse fatal("preopen not found: {q}", .{name})) { + .file => fatal("preopen {q} is not a directory", .{name}), + .dir => |d| d, + }, + }; + } + fn openUnresolved( + arena: Allocator, + io: Io, + cwd: []const u8, + unresolved_path: []const u8, + thing: enum { @"zig lib", @"global cache", @"local cache" }, + ) Cache.Directory { + const path = resolvePath(arena, cwd, &.{unresolved_path}) catch |err| { + fatal("unable to resolve {t} directory: {t}", .{ thing, err }); + }; + const nonempty_path = if (path.len == 0) "." else path; + const handle_or_err = switch (thing) { + .@"zig lib" => Dir.cwd().openDir(io, nonempty_path, .{}), + .@"global cache", .@"local cache" => Dir.cwd().createDirPathOpen(io, nonempty_path, .{}), + }; + return .{ + .path = if (path.len == 0) null else path, + .handle = handle_or_err catch |err| { + const extra_str: []const u8 = e: { + if (thing == .@"global cache") switch (err) { + error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++ + "If this location is not writable then consider specifying an alternative with " ++ + "the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.", + else => {}, + }; + break :e ""; + }; + fatal("unable to open {t} directory {q}: {t}{s}", .{ thing, nonempty_path, err, extra_str }); + }, + }; + } +}; + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory { + const cwd_path = try getResolvedCwd(io, gpa); + defer gpa.free(cwd_path); + const self_exe_path = try std.process.executablePathAlloc(io, gpa); + defer gpa.free(self_exe_path); + + return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path); +} + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDirFromSelfExe( + allocator: Allocator, + io: Io, + /// The return value of `getResolvedCwd`. + /// Passed as an argument to avoid pointlessly repeating the call. + cwd_path: []const u8, + self_exe_path: []const u8, +) error{ OutOfMemory, FileNotFound }!Cache.Directory { + const cwd = Dir.cwd(); + var cur_path: []const u8 = self_exe_path; + while (Dir.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { + var base_dir = cwd.openDir(io, dirname, .{}) catch continue; + defer base_dir.close(io); + + const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue; + const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? }); + defer allocator.free(p); + + const resolved = try resolvePath(allocator, cwd_path, &.{p}); + return .{ + .handle = sub_directory.handle, + .path = if (resolved.len == 0) null else resolved, + }; + } + return error.FileNotFound; +} + +/// Returns the sub_path that worked, or `null` if none did. +/// The path of the returned Directory is relative to `base`. +/// The handle of the returned Directory is open. +fn testZigInstallPrefix(io: Io, base_dir: Dir) ?Cache.Directory { + const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig"; + + zig_dir: { + // Try lib/zig/std/std.zig + const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig"; + var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir; + const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { + test_zig_dir.close(io); + break :zig_dir; + }; + file.close(io); + return .{ .handle = test_zig_dir, .path = lib_zig }; + } + + // Try lib/std/std.zig + var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null; + const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { + test_zig_dir.close(io); + return null; + }; + file.close(io); + return .{ .handle = test_zig_dir, .path = "lib" }; +} + +pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 { + if (EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) |value| return value; + + const app_name = "zig"; + + switch (builtin.os.tag) { + .wasi => @compileError("on WASI the global cache dir must be resolved with preopens"), + .windows => { + const local_app_data_dir = EnvVar.LOCALAPPDATA.get(environ_map) orelse + return error.AppDataDirUnavailable; + return Dir.path.join(arena, &.{ local_app_data_dir, app_name }); + }, + else => { + if (EnvVar.XDG_CACHE_HOME.get(environ_map)) |cache_root| { + if (cache_root.len > 0) { + return Dir.path.join(arena, &.{ cache_root, app_name }); + } + } + if (EnvVar.HOME.get(environ_map)) |home| { + if (home.len > 0) { + return Dir.path.join(arena, &.{ home, ".cache", app_name }); + } + } + return error.AppDataDirUnavailable; + }, + } +} + +/// Searches upwards from `cwd` for a directory containing a `build.zig` file. +/// If such a directory is found, returns the path to it joined to the `.zig_cache` name. +/// Otherwise, returns `null`, indicating no suitable local cache location. +pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 { + var cur_dir = cwd; + while (true) { + const joined = try Dir.path.join(arena, &.{ cur_dir, build_zig_basename }); + if (Dir.cwd().access(io, joined, .{})) |_| { + return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename }); + } else |err| switch (err) { + error.FileNotFound => { + cur_dir = Dir.path.dirname(cur_dir) orelse return null; + continue; + }, + else => return null, + } + } +} + +/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would +/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd +/// returns the empty string ("") instead of ".". +pub fn resolvePath( + gpa: Allocator, + /// The return value of `getResolvedCwd`. + /// Passed as an argument to avoid pointlessly repeating the call. + cwd_resolved: []const u8, + paths: []const []const u8, +) Allocator.Error![]u8 { + if (builtin.target.os.tag == .wasi) { + assert(mem.eql(u8, cwd_resolved, "")); + const res = try Dir.path.resolve(gpa, paths); + if (mem.eql(u8, res, ".")) { + gpa.free(res); + return ""; + } + return res; + } + + // Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`. + for (paths) |p| { + if (Dir.path.isAbsolute(p)) break; // absolute path + if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir + } else { + // no absolute path, no "..". + const res = try Dir.path.resolve(gpa, paths); + if (mem.eql(u8, res, ".")) { + gpa.free(res); + return ""; + } + assert(!Dir.path.isAbsolute(res)); + assert(!isUpDir(res)); + return res; + } + + // The fast path failed; resolve the whole thing. + // Optimization: `paths` often has just one element. + const path_resolved = switch (paths.len) { + 0 => unreachable, + 1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }), + else => r: { + const all_paths = try gpa.alloc([]const u8, paths.len + 1); + defer gpa.free(all_paths); + all_paths[0] = cwd_resolved; + @memcpy(all_paths[1..], paths); + break :r try Dir.path.resolve(gpa, all_paths); + }, + }; + errdefer gpa.free(path_resolved); + + assert(Dir.path.isAbsolute(path_resolved)); + assert(Dir.path.isAbsolute(cwd_resolved)); + + if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd + if (path_resolved.len == cwd_resolved.len) { + // equal to cwd + gpa.free(path_resolved); + return ""; + } + if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs) + + // in cwd; extract sub path + const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]); + gpa.free(path_resolved); + return sub_path; +} + +pub fn isUpDir(p: []const u8) bool { + return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep); +} + test { _ = Ast; _ = AstRlAnnotate; diff --git a/src/Compilation.zig b/src/Compilation.zig index 70ed4c7f9a55d8b1695520be78c264925f39928b..b130d4917c17a86a856f0ab02ee4552ddd179bda 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -17,7 +17,6 @@ const Value = @import("Value.zig"); const Type = @import("Type.zig"); const target_util = @import("target.zig"); const Package = @import("Package.zig"); -const introspect = @import("introspect.zig"); const link = @import("link.zig"); const tracy = @import("tracy.zig"); const trace = tracy.trace; @@ -190,7 +189,7 @@ parent_whole_cache: ?ParentWholeCache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, /// Owned by the caller of `Compilation.create`. -dirs: Directories, +dirs: std.zig.Directories, libc_include_dir_list: []const []const u8, libc_framework_dir_list: []const []const u8, rc_includes: std.zig.RcIncludes, @@ -431,7 +430,7 @@ pub const Path = struct { } /// Given a `Path`, returns the directory handle and sub path to be used to open the path. - pub fn openInfo(p: Path, dirs: Directories) struct { Io.Dir, []const u8 } { + pub fn openInfo(p: Path, dirs: std.zig.Directories) struct { Io.Dir, []const u8 } { const dir = switch (p.root) { .none => { const cwd_sub_path = absToCwdRelative(p.sub_path, dirs.cwd); @@ -492,7 +491,7 @@ pub const Path = struct { /// From an unresolved path (which can be made of multiple not-yet-joined strings), construct a /// canonical `Path`. pub fn fromUnresolved(gpa: Allocator, dirs: Compilation.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path { - const resolved = try introspect.resolvePath(gpa, dirs.cwd, unresolved_parts); + const resolved = try std.zig.resolvePath(gpa, dirs.cwd, unresolved_parts); errdefer gpa.free(resolved); // If, for instance, `dirs.local_cache.path` is within the lib dir, it must take priority, @@ -626,7 +625,7 @@ pub const Path = struct { }); } - pub fn toCachePath(p: Path, dirs: Directories) Cache.Path { + pub fn toCachePath(p: Path, dirs: std.zig.Directories) Cache.Path { const root_dir: Cache.Directory = switch (p.root) { .zig_lib => dirs.zig_lib, .global_cache => dirs.global_cache, @@ -649,7 +648,7 @@ pub const Path = struct { /// This should not be used for most of the compiler pipeline, but is useful when emitting /// paths from the compilation (e.g. in debug info), because they will not depend on the cwd. /// The returned path is owned by the caller and allocated into `gpa`. - pub fn toAbsolute(p: Path, dirs: Directories, gpa: Allocator) Allocator.Error![]u8 { + pub fn toAbsolute(p: Path, dirs: std.zig.Directories, gpa: Allocator) Allocator.Error![]u8 { const root_path: []const u8 = switch (p.root) { .zig_lib => dirs.zig_lib.path orelse "", .global_cache => dirs.global_cache.path orelse "", @@ -680,7 +679,7 @@ pub const Path = struct { /// Returns whether this `Path` is illegal to have as a user-imported `Zcu.File` (including /// as the root of a module). Such paths exist in directories which the Zig compiler treats /// specially, like 'global_cache/b/', which stores 'builtin.zig' files. - pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: Directories) Allocator.Error!bool { + pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: std.zig.Directories) Allocator.Error!bool { const zig_builtin_dir: Path = try .fromRoot(gpa, dirs, .global_cache, "b"); defer zig_builtin_dir.deinit(gpa); return switch (p.isNested(zig_builtin_dir)) { @@ -690,149 +689,6 @@ pub const Path = struct { } }; -pub const Directories = struct { - /// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path, - /// but on WASI is the empty string "" instead, because WASI does not have absolute paths. - cwd: []const u8, - /// The Zig 'lib' directory. - /// `zig_lib.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - /// Guaranteed to be a different path from `global_cache` and `local_cache`. - zig_lib: Cache.Directory, - /// The global Zig cache directory. - /// `global_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - global_cache: Cache.Directory, - /// The local Zig cache directory. - /// `local_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - /// This may be the same as `global_cache`. - local_cache: Cache.Directory, - - pub fn deinit(dirs: *Directories, io: Io) void { - // The local and global caches could be the same. - const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle; - - dirs.global_cache.handle.close(io); - if (close_local) dirs.local_cache.handle.close(io); - dirs.zig_lib.handle.close(io); - } - - /// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for - /// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it - /// shares handles with `dirs`. - pub fn withoutLocalCache(dirs: Directories) Directories { - return .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }; - } - - /// Uses `std.process.fatal` on error conditions. - pub fn init( - arena: Allocator, - io: Io, - override_zig_lib: ?[]const u8, - override_global_cache: ?[]const u8, - local_cache_strat: union(enum) { - override: []const u8, - search, - global, - }, - preopens: std.process.Preopens, - self_exe_path: switch (builtin.target.os.tag) { - .wasi => void, - else => []const u8, - }, - environ_map: *const std.process.Environ.Map, - cwd: []const u8, - ) Directories { - const wasi = builtin.target.os.tag == .wasi; - - const zig_lib: Cache.Directory = d: { - if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); - if (wasi) break :d getPreopen(preopens, "/lib"); - break :d introspect.findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch |err| { - fatal("unable to find zig installation directory '{s}': {t}", .{ self_exe_path, err }); - }; - }; - - const global_cache: Cache.Directory = d: { - if (override_global_cache) |path| break :d openUnresolved(arena, io, cwd, path, .@"global cache"); - if (wasi) break :d getPreopen(preopens, "/cache"); - const path = introspect.resolveGlobalCacheDir(arena, environ_map) catch |err| { - fatal("unable to resolve zig cache directory: {t}", .{err}); - }; - break :d openUnresolved(arena, io, cwd, path, .@"global cache"); - }; - - const local_cache: Cache.Directory = switch (local_cache_strat) { - .override => |path| openUnresolved(arena, io, cwd, path, .@"local cache"), - .search => d: { - const maybe_path = introspect.resolveSuitableLocalCacheDir(arena, io, cwd) catch |err| { - fatal("unable to resolve zig cache directory: {t}", .{err}); - }; - const path = maybe_path orelse break :d global_cache; - break :d openUnresolved(arena, io, cwd, path, .@"local cache"); - }, - .global => global_cache, - }; - - if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) { - fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache }); - } - if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) { - fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache }); - } - - return .{ - .cwd = cwd, - .zig_lib = zig_lib, - .global_cache = global_cache, - .local_cache = local_cache, - }; - } - fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory { - return .{ - .path = if (std.mem.eql(u8, name, ".")) null else name, - .handle = switch (preopens.get(name) orelse fatal("preopen not found: '{s}'", .{name})) { - .file => fatal("preopen {s} is not a directory", .{name}), - .dir => |d| d, - }, - }; - } - fn openUnresolved( - arena: Allocator, - io: Io, - cwd: []const u8, - unresolved_path: []const u8, - thing: enum { @"zig lib", @"global cache", @"local cache" }, - ) Cache.Directory { - const path = introspect.resolvePath(arena, cwd, &.{unresolved_path}) catch |err| { - fatal("unable to resolve {s} directory: {s}", .{ @tagName(thing), @errorName(err) }); - }; - const nonempty_path = if (path.len == 0) "." else path; - const handle_or_err = switch (thing) { - .@"zig lib" => Io.Dir.cwd().openDir(io, nonempty_path, .{}), - .@"global cache", .@"local cache" => Io.Dir.cwd().createDirPathOpen(io, nonempty_path, .{}), - }; - return .{ - .path = if (path.len == 0) null else path, - .handle = handle_or_err catch |err| { - const extra_str: []const u8 = e: { - if (thing == .@"global cache") switch (err) { - error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++ - "If this location is not writable then consider specifying an alternative with " ++ - "the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.", - else => {}, - }; - break :e ""; - }; - fatal("unable to open {s} directory '{s}': {s}{s}", .{ @tagName(thing), nonempty_path, @errorName(err), extra_str }); - }, - }; - } -}; - /// This small wrapper function just checks whether debug extensions are enabled before checking /// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller, /// preventing debugging features from making it into release builds of the compiler. @@ -1549,7 +1405,7 @@ const CacheUse = union(CacheMode) { }; pub const CreateOptions = struct { - dirs: Directories, + dirs: std.zig.Directories, thread_limit: usize, self_exe_path: ?[]const u8 = null, diff --git a/src/Package/Module.zig b/src/Module.zig similarity index 97% rename from src/Package/Module.zig rename to src/Module.zig index 0c7e4166adf7d6c290cbb330bb12857c1cf21d90..02c65b09fcb13eb9ead45c4e9ce925d01b627e43 100644 --- a/src/Package/Module.zig +++ b/src/Module.zig @@ -1,4 +1,15 @@ //! Corresponds to something that Zig source code can `@import`. +const Module = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const assert = std.debug.assert; + +const target_util = @import("../target.zig"); +const Builtin = @import("../Builtin.zig"); +const Compilation = @import("../Compilation.zig"); +const File = @import("../Zcu.zig").File; /// The root directory of the module. Only files inside this directory can be imported. root: Compilation.Path, @@ -37,11 +48,6 @@ no_builtin: bool, pub const Deps = std.array_hash_map.String(*Module); -pub const Tree = struct { - /// Each `Package` exposes a `Module` with build.zig as its root source file. - build_module_table: std.array_hash_map.Auto(MultiHashHexDigest, *Module), -}; - pub const CreateOptions = struct { paths: Paths, fully_qualified_name: []const u8, @@ -50,7 +56,7 @@ pub const CreateOptions = struct { inherited: Inherited, global: Compilation.Config, /// If this is null then `resolved_target` must be non-null. - parent: ?*Package.Module, + parent: ?*Module, pub const Paths = struct { root: Compilation.Path, @@ -107,7 +113,7 @@ pub const CreateError = error{ }; /// At least one of `parent` and `resolved_target` must be non-null. -pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { +pub fn create(arena: Allocator, options: CreateOptions) !*Module { if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); if (options.inherited.fuzz == true) assert(options.global.any_fuzz); if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); @@ -420,7 +426,7 @@ pub const LimitedOptions = struct { /// This one can only be used if the Module will only be used for AstGen and earlier in /// the pipeline. Illegal behavior occurs if a limited module touches Sema. -pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { +pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Module { const mod = try gpa.create(Module); mod.* = .{ .root = options.root, @@ -515,15 +521,3 @@ pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin { .wasi_exec_model = global.wasi_exec_model, }; } - -const Module = @This(); -const Package = @import("../Package.zig"); -const std = @import("std"); -const Allocator = std.mem.Allocator; -const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; -const target_util = @import("../target.zig"); -const Cache = std.Build.Cache; -const Builtin = @import("../Builtin.zig"); -const assert = std.debug.assert; -const Compilation = @import("../Compilation.zig"); -const File = @import("../Zcu.zig").File; diff --git a/src/Zcu.zig b/src/Zcu.zig index e3a36fe31b987dce03f89d1f4694cf718d72d3dc..41758eaf3c3b2ae6ef9ce3adfef9ccb734bda5f4 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -34,7 +34,6 @@ const AstGen = std.zig.AstGen; const Sema = @import("Sema.zig"); const target_util = @import("target.zig"); const build_options = @import("build_options"); -const isUpDir = @import("introspect.zig").isUpDir; const InternPool = @import("InternPool.zig"); const Alignment = InternPool.Alignment; const AnalUnit = InternPool.AnalUnit; diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index e1cda016c163cdfde8cb693c5f143f2f86057980..f9aede4f72b4c0a4e32b06f02ac509ef62d30d09 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -23,7 +23,6 @@ const builtin = @import("builtin"); const dev = @import("../dev.zig"); const InternPool = @import("../InternPool.zig"); const AnalUnit = InternPool.AnalUnit; -const introspect = @import("../introspect.zig"); const Module = @import("../Package.zig").Module; const Sema = @import("../Sema.zig"); const target_util = @import("../target.zig"); diff --git a/src/dev.zig b/src/dev.zig index a6f97799157440329f3b248959e47f1585cfc1b3..8d3723ec101b8984b9d33381754ec3388374d3f6 100644 --- a/src/dev.zig +++ b/src/dev.zig @@ -112,7 +112,6 @@ pub const Env = enum { .translate_c_command, .fmt_command, .jit_command, - .fetch_command, .init_command, .targets_command, .version_command, @@ -252,7 +251,6 @@ pub const Feature = enum { translate_c_command, fmt_command, jit_command, - fetch_command, init_command, targets_command, version_command, diff --git a/src/introspect.zig b/src/introspect.zig deleted file mode 100644 index 13d00520936973fccc37ede11256842e0826f423..0000000000000000000000000000000000000000 --- a/src/introspect.zig +++ /dev/null @@ -1,220 +0,0 @@ -const builtin = @import("builtin"); - -const std = @import("std"); -const Io = std.Io; -const Dir = std.Io.Dir; -const mem = std.mem; -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const assert = std.debug.assert; - -const build_options = @import("build_options"); - -const Compilation = @import("Compilation.zig"); -const Package = @import("Package.zig"); - -/// Returns the sub_path that worked, or `null` if none did. -/// The path of the returned Directory is relative to `base`. -/// The handle of the returned Directory is open. -fn testZigInstallPrefix(io: Io, base_dir: Io.Dir) ?Cache.Directory { - const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig"; - - zig_dir: { - // Try lib/zig/std/std.zig - const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig"; - var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir; - const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { - test_zig_dir.close(io); - break :zig_dir; - }; - file.close(io); - return .{ .handle = test_zig_dir, .path = lib_zig }; - } - - // Try lib/std/std.zig - var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null; - const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { - test_zig_dir.close(io); - return null; - }; - file.close(io); - return .{ .handle = test_zig_dir, .path = "lib" }; -} - -/// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory { - const cwd_path = try getResolvedCwd(io, gpa); - defer gpa.free(cwd_path); - const self_exe_path = try std.process.executablePathAlloc(io, gpa); - defer gpa.free(self_exe_path); - - return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path); -} - -/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This -/// means the path has no repeated separators, no "." or ".." components, and no trailing separator. -/// On WASI, "" is returned instead of ".". -pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 { - if (builtin.target.os.tag == .wasi) { - if (std.debug.runtime_safety) { - const cwd = try std.process.currentPathAlloc(io, gpa); - defer gpa.free(cwd); - assert(mem.eql(u8, cwd, ".")); - } - return ""; - } - const cwd = try std.process.currentPathAlloc(io, gpa); - defer gpa.free(cwd); - const resolved = try Dir.path.resolve(gpa, &.{cwd}); - assert(Dir.path.isAbsolute(resolved)); - return resolved; -} - -/// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDirFromSelfExe( - allocator: Allocator, - io: Io, - /// The return value of `getResolvedCwd`. - /// Passed as an argument to avoid pointlessly repeating the call. - cwd_path: []const u8, - self_exe_path: []const u8, -) error{ OutOfMemory, FileNotFound }!Cache.Directory { - const cwd = Io.Dir.cwd(); - var cur_path: []const u8 = self_exe_path; - while (Dir.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { - var base_dir = cwd.openDir(io, dirname, .{}) catch continue; - defer base_dir.close(io); - - const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue; - const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? }); - defer allocator.free(p); - - const resolved = try resolvePath(allocator, cwd_path, &.{p}); - return .{ - .handle = sub_directory.handle, - .path = if (resolved.len == 0) null else resolved, - }; - } - return error.FileNotFound; -} - -pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 { - if (std.zig.EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) |value| return value; - - const app_name = "zig"; - - switch (builtin.os.tag) { - .wasi => @compileError("on WASI the global cache dir must be resolved with preopens"), - .windows => { - const local_app_data_dir = std.zig.EnvVar.LOCALAPPDATA.get(environ_map) orelse - return error.AppDataDirUnavailable; - return Dir.path.join(arena, &.{ local_app_data_dir, app_name }); - }, - else => { - if (std.zig.EnvVar.XDG_CACHE_HOME.get(environ_map)) |cache_root| { - if (cache_root.len > 0) { - return Dir.path.join(arena, &.{ cache_root, app_name }); - } - } - if (std.zig.EnvVar.HOME.get(environ_map)) |home| { - if (home.len > 0) { - return Dir.path.join(arena, &.{ home, ".cache", app_name }); - } - } - return error.AppDataDirUnavailable; - }, - } -} - -/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would -/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd -/// returns the empty string ("") instead of ".". -pub fn resolvePath( - gpa: Allocator, - /// The return value of `getResolvedCwd`. - /// Passed as an argument to avoid pointlessly repeating the call. - cwd_resolved: []const u8, - paths: []const []const u8, -) Allocator.Error![]u8 { - if (builtin.target.os.tag == .wasi) { - assert(mem.eql(u8, cwd_resolved, "")); - const res = try Dir.path.resolve(gpa, paths); - if (mem.eql(u8, res, ".")) { - gpa.free(res); - return ""; - } - return res; - } - - // Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`. - for (paths) |p| { - if (Dir.path.isAbsolute(p)) break; // absolute path - if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir - } else { - // no absolute path, no "..". - const res = try Dir.path.resolve(gpa, paths); - if (mem.eql(u8, res, ".")) { - gpa.free(res); - return ""; - } - assert(!Dir.path.isAbsolute(res)); - assert(!isUpDir(res)); - return res; - } - - // The fast path failed; resolve the whole thing. - // Optimization: `paths` often has just one element. - const path_resolved = switch (paths.len) { - 0 => unreachable, - 1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }), - else => r: { - const all_paths = try gpa.alloc([]const u8, paths.len + 1); - defer gpa.free(all_paths); - all_paths[0] = cwd_resolved; - @memcpy(all_paths[1..], paths); - break :r try Dir.path.resolve(gpa, all_paths); - }, - }; - errdefer gpa.free(path_resolved); - - assert(Dir.path.isAbsolute(path_resolved)); - assert(Dir.path.isAbsolute(cwd_resolved)); - - if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd - if (path_resolved.len == cwd_resolved.len) { - // equal to cwd - gpa.free(path_resolved); - return ""; - } - if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs) - - // in cwd; extract sub path - const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]); - gpa.free(path_resolved); - return sub_path; -} - -pub fn isUpDir(p: []const u8) bool { - return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep); -} - -pub const default_local_zig_cache_basename = ".zig-cache"; - -/// Searches upwards from `cwd` for a directory containing a `build.zig` file. -/// If such a directory is found, returns the path to it joined to the `.zig_cache` name. -/// Otherwise, returns `null`, indicating no suitable local cache location. -pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 { - var cur_dir = cwd; - while (true) { - const joined = try Dir.path.join(arena, &.{ cur_dir, Package.build_zig_basename }); - if (Io.Dir.cwd().access(io, joined, .{})) |_| { - return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename }); - } else |err| switch (err) { - error.FileNotFound => { - cur_dir = Dir.path.dirname(cur_dir) orelse return null; - continue; - }, - else => return null, - } - } -} diff --git a/src/main.zig b/src/main.zig index 84b6553ff64750279537a242729f6ecf9c807639..4e255a4d496a9c7271c4d91b555b0e77647e5b81 100644 --- a/src/main.zig +++ b/src/main.zig @@ -21,13 +21,13 @@ const AstGen = std.zig.AstGen; const ZonGen = std.zig.ZonGen; const Server = std.zig.Server; const stringToEnum = std.meta.stringToEnum; +const allocPrint = std.fmt.allocPrint; pub const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); const link = @import("link.zig"); const Package = @import("Package.zig"); const build_options = @import("build_options"); -const introspect = @import("introspect.zig"); const wasi_libc = @import("libs/wasi_libc.zig"); const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); @@ -353,9 +353,15 @@ fn mainArgs( dev.check(.ar_command); return process.exit(try llvmArMain(arena, args)); }, - .build => { - dev.check(.build_command); - return cmdBuild(gpa, arena, io, cmd_args, environ_map); + .build, .fetch => { + return jitCmd(gpa, arena, io, args, environ_map, .{ + .cmd_name = "maker", + .root_src_path = "Maker.zig", + .prepend_zig_lib_dir_path = true, + .prepend_global_cache_path = true, + .prepend_zig_exe_path = true, + .prepend_seed = true, + }); }, .clang, .@"-cc1", .@"-cc1as" => { dev.check(.clang_command); @@ -385,7 +391,6 @@ fn mainArgs( .depend_on_aro = true, .prepend_zig_lib_dir_path = true, .server = use_server, - .color = Color.settingFromEnvironment(environ_map), }); }, .fmt => { @@ -396,25 +401,19 @@ fn mainArgs( return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "objcopy", .root_src_path = "objcopy.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, .objdump => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "objdump", .root_src_path = "objdump.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, - .fetch => { - return cmdFetch(gpa, arena, io, cmd_args, environ_map); - }, .libc => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "libc", .root_src_path = "libc.zig", .prepend_zig_lib_dir_path = true, - .color = Color.settingFromEnvironment(environ_map), }); }, .std => { @@ -424,7 +423,6 @@ fn mainArgs( .prepend_zig_lib_dir_path = true, .prepend_zig_exe_path = true, .prepend_global_cache_path = true, - .color = Color.settingFromEnvironment(environ_map), }); }, .init => { @@ -461,7 +459,6 @@ fn mainArgs( return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "reduce", .root_src_path = "reduce.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, .zen => { @@ -2977,7 +2974,7 @@ fn buildOutputType( while (preprocessor_args_it.next()) |arg| { if (mem.eql(u8, arg, "-MD") or mem.eql(u8, arg, "-MMD") or mem.eql(u8, arg, "-MT")) { disable_c_depfile = true; - const cc_arg = try std.fmt.allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() }); + const cc_arg = try allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() }); try cc_argv.append(arena, cc_arg); } else { fatal("unsupported preprocessor arg: {s}", .{arg}); @@ -3222,7 +3219,7 @@ fn buildOutputType( else => process.executablePathAlloc(io, arena) catch |err| fatal("unable to find zig self exe path: {t}", .{err}), }; - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( @@ -3421,9 +3418,9 @@ fn buildOutputType( .yes_default_value => if (create_module.resolved_options.output_mode == .Lib and create_module.resolved_options.link_mode == .dynamic and target.ofmt == .elf) if (have_version) - try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major }) + try allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major }) else - try std.fmt.allocPrint(arena, "lib{s}.so", .{root_name}) + try allocPrint(arena, "lib{s}.so", .{root_name}) else null, }; @@ -3433,7 +3430,7 @@ fn buildOutputType( .yes_default_path => emit: { if (output_to_cache != null) break :emit .yes_cache; const name = switch (clang_preprocessor_mode) { - .pch => try std.fmt.allocPrint(arena, "{s}.pch", .{root_name}), + .pch => try allocPrint(arena, "{s}.pch", .{root_name}), else => try std.zig.binNameAlloc(arena, .{ .root_name = root_name, .cpu_arch = target.cpu.arch, @@ -3469,16 +3466,16 @@ fn buildOutputType( }, }; - const default_h_basename = try std.fmt.allocPrint(arena, "{s}.h", .{root_name}); + const default_h_basename = try allocPrint(arena, "{s}.h", .{root_name}); const emit_h_resolved = emit_h.resolve(io, default_h_basename, output_to_cache); - const default_asm_basename = try std.fmt.allocPrint(arena, "{s}.s", .{root_name}); + const default_asm_basename = try allocPrint(arena, "{s}.s", .{root_name}); const emit_asm_resolved = emit_asm.resolve(io, default_asm_basename, output_to_cache); - const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{s}.ll", .{root_name}); + const default_llvm_ir_basename = try allocPrint(arena, "{s}.ll", .{root_name}); const emit_llvm_ir_resolved = emit_llvm_ir.resolve(io, default_llvm_ir_basename, output_to_cache); - const default_llvm_bc_basename = try std.fmt.allocPrint(arena, "{s}.bc", .{root_name}); + const default_llvm_bc_basename = try allocPrint(arena, "{s}.bc", .{root_name}); const emit_llvm_bc_resolved = emit_llvm_bc.resolve(io, default_llvm_bc_basename, output_to_cache); const emit_docs_resolved = emit_docs.resolve(io, "docs", output_to_cache); @@ -3499,7 +3496,7 @@ fn buildOutputType( fatal("the argument -femit-implib is allowed only when building a Windows DLL", .{}); } } - const default_implib_basename = try std.fmt.allocPrint(arena, "{s}.lib", .{root_name}); + const default_implib_basename = try allocPrint(arena, "{s}.lib", .{root_name}); const emit_implib_resolved: Compilation.CreateOptions.Emit = switch (emit_implib) { .no => .no, .yes => emit_implib.resolve(io, default_implib_basename, output_to_cache), @@ -3528,7 +3525,7 @@ fn buildOutputType( // "-" is stdin. Dump it to a real file. const sep = fs.path.sep_str; - const dump_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{ + const dump_path = try allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{ randInt(io, u64), ext.canonicalName(target), }); try dirs.local_cache.handle.createDirPath(io, "tmp"); @@ -3557,7 +3554,7 @@ fn buildOutputType( const bin_digest: Cache.BinDigest = hasher.hasher.finalResult(); - const sub_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{ + const sub_path = try allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{ &bin_digest, ext.canonicalName(target), }); try dirs.local_cache.handle.rename(dump_path, dirs.local_cache.handle, sub_path, io); @@ -4586,7 +4583,7 @@ fn runOrTest( try argv.append(exe_path); if (arg_mode == .zig_test) { try argv.append( - try std.fmt.allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}), + try allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}), ); } } else { @@ -4794,7 +4791,7 @@ fn cmdTranslateC( assert(comp.c_source_files.len == 1); const c_source_file = comp.c_source_files[0]; - const translated_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{comp.root_name}); + const translated_basename = try allocPrint(arena, "{s}.zig", .{comp.root_name}); var man: Cache.Manifest = comp.obtainCObjectCacheManifest(comp.root_mod); man.want_shared_lock = false; @@ -4872,7 +4869,6 @@ pub fn translateC( .root_src_path = "translate-c/main.zig", .depend_on_aro = true, .capture = capture, - .color = Color.settingFromEnvironment(environ_map), }); } @@ -4912,7 +4908,7 @@ fn cmdInit(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8) ! } } - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); const cwd_basename = fs.path.basename(cwd_path); const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename); @@ -5027,1065 +5023,17 @@ test sanitizeExampleName { try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); } -fn cmdBuild( - gpa: Allocator, - arena: Allocator, - io: Io, - args: []const []const u8, - environ_map: *process.Environ.Map, -) !void { - var build_file: ?[]const u8 = null; - var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); - var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) - .Debug - else - .ReleaseSafe; - var configure_argv: std.ArrayList([]const u8) = .empty; - var make_argv: std.ArrayList([]const u8) = .empty; - var cached_passthru_configure: std.ArrayList(u32) = .empty; - var forks: std.ArrayList(Fork) = .empty; - var reference_trace: ?u32 = null; - var debug_compile_errors = false; - var verbose_link = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_LINK.isSet(environ_map); - var verbose_cc = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_CC.isSet(environ_map); - var verbose_air = false; - var verbose_intern_pool = false; - var verbose_generic_instances = false; - var verbose_llvm_ir: ?[]const u8 = null; - var verbose_llvm_bc: ?[]const u8 = null; - var verbose_llvm_cpu_features = false; - var fetch_only = false; - var fetch_mode: Package.Fetch.JobQueue.Mode = .needed; - var system_pkg_dir_path: ?[]const u8 = null; - var debug_target: ?[]const u8 = null; - var debug_libc_paths_file: ?[]const u8 = null; - var cache_poison: std.Build.Graph.CachePoison = .pure; - var print_configuration_path: bool = false; - - const self_exe_path = try process.executablePathAlloc(io, arena); - const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); - - try configure_argv.ensureUnusedCapacity(arena, 16); - try make_argv.ensureUnusedCapacity(arena, 16); - try cached_passthru_configure.ensureUnusedCapacity(arena, 16); - - _ = configure_argv.addOneAssumeCapacity(); // configurer executable - _ = make_argv.addOneAssumeCapacity(); // maker executable - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined }; - const make_argv_index_zig_lib_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const make_argv_index_build_root = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined }; - const make_argv_index_cache_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined }; - const make_argv_index_global_cache_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined }; - const argv_index_configuration_file = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed }; - const argv_index_seed = make_argv.items.len - 1; - - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const conf_argv_index_build_root = configure_argv.items.len - 1; - - var color: Color = Color.settingFromEnvironment(environ_map); - var n_jobs: ?u32 = null; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - try configure_argv.ensureUnusedCapacity(arena, 2); - - if (mem.startsWith(u8, arg, "-D") or - mem.startsWith(u8, arg, "-fsys=") or - mem.startsWith(u8, arg, "-fno-sys=") or - mem.startsWith(u8, arg, "--release=") or - mem.eql(u8, arg, "--release")) - { - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--system")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - system_pkg_dir_path = args[i]; - - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. - continue; - } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { - color = stringToEnum(Color, rest) orelse - fatal("expected --color=[auto|on|off]; found {q}", .{arg}); - - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--cache-poison")) { - cache_poison = .poisoned; - configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); - continue; - } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { - // Allow the configurer process to report parse failure. - if (stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { - cache_poison = poison; - } - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--verbose")) { - // Intentionally is added both to make and configure but - // does not go into the cache hash. - configure_argv.appendAssumeCapacity(arg); - } else if (mem.eql(u8, arg, "--search-prefix")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - // This argument is cache poisonous: it does not go into - // the cache and configurer must set the poison bit when - // choosing to observe it. - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, args[i] }; - (try make_argv.addManyAsArray(arena, 2)).* = .{ arg, args[i] }; - continue; - } else if (mem.eql(u8, arg, "--build-file")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - build_file = args[i]; - continue; - } else if (mem.eql(u8, arg, "--zig-lib-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_lib_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--pkg-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_pkg_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--print-configuration-path")) { - print_configuration_path = true; - continue; - } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.eql(u8, arg, "--fetch")) { - fetch_only = true; - } else if (mem.cutPrefix(u8, arg, "--fetch=")) |sub_arg| { - fetch_only = true; - fetch_mode = stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse - fatal("expected [needed|all] after \"--fetch=\", found: {s}", .{sub_arg}); - } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| { - try forks.append(arena, .init(sub_arg)); - continue; - } else if (mem.eql(u8, arg, "--fork")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - try forks.append(arena, .init(args[i])); - continue; - } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - fatal("unable to parse reference_trace count {q}: {t}", .{ num, err }); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; - } else if (mem.cutPrefix(u8, arg, "--maker-opt=")) |rest| { - maker_optimize_mode = parseOptimizeMode(rest); - continue; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - try make_argv.appendSlice(arena, args[i .. i + 2]); - i += 1; - try addDebugLog(arena, args[i]); - continue; - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - if (build_options.enable_debug_extensions) { - debug_compile_errors = true; - } else { - warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{}); - } - } else if (mem.eql(u8, arg, "--debug-target")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_target = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{}); - } - continue; - } else if (mem.eql(u8, arg, "--debug-libc")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_libc_paths_file = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-libc has no effect.", .{}); - } - continue; - } else if (mem.eql(u8, arg, "--verbose-link")) { - verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-intern-pool")) { - verbose_intern_pool = true; - } else if (mem.eql(u8, arg, "--verbose-generic-instances")) { - verbose_generic_instances = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - verbose_llvm_ir = "-"; - } else if (mem.cutPrefix(u8, arg, "--verbose-llvm-ir=")) |rest| { - verbose_llvm_ir = rest; - } else if (mem.cutPrefix(u8, arg, "--verbose-llvm-bc=")) |rest| { - verbose_llvm_bc = rest; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - verbose_llvm_cpu_features = true; - } else if (mem.cutPrefix(u8, arg, "-j")) |str| { - const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| - fatal("unable to parse jobs count {s}: {t}", .{ str, err }); - if (num < 1) { - fatal("number of jobs must be at least 1", .{}); - } - n_jobs = num; - } else if (mem.eql(u8, arg, "--seed")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - make_argv.items[argv_index_seed] = args[i]; - continue; - } else if (mem.eql(u8, arg, "--")) { - try make_argv.appendSlice(arena, args[i..]); - break; - } - } - try make_argv.append(arena, arg); - } - } - - const root_prog_node = std.Progress.start(io, .{ - .disable_printing = (color == .off), - .root_name = "", - }); - defer root_prog_node.end(); - - process.raiseFileDescriptorLimit(); - - const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| - fatal("failed to get current directory path: {t}", .{err}); - - const build_root = try findBuildRoot(arena, io, .{ - .cwd_path = cwd_path, - .build_file = build_file, - }); - - { - // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( - arena, - io, - override_lib_dir, - override_global_cache_dir, - .{ .override = path: { - if (override_local_cache_dir) |d| break :path d; - break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); - } }, - .empty, - self_exe_path, - environ_map, - cwd_path, - ); - defer dirs.deinit(io); - - const thread_limit = @min( - @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), - std.math.maxInt(Zcu.PerThread.IdBacking), - ); - try setThreadLimit(arena, thread_limit); - - // Cache lookup for configure options. If we get a match, we can skip - // execution of the configure script. If not, we get the file path to pass - // to the configure process. - var local_cache: Cache = .{ - .gpa = gpa, - .io = io, - .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), - .cwd = cwd_path, - }; - local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); - local_cache.addPrefix(dirs.zig_lib); - local_cache.addPrefix(dirs.local_cache); - local_cache.addPrefix(dirs.global_cache); - defer local_cache.manifest_dir.close(io); - - var config_man = local_cache.obtain(); - defer config_man.deinit(); - config_man.hash.addBytes(build_options.version); - - for (cached_passthru_configure.items) |i| - config_man.hash.addBytes(configure_argv.items[i]); - - // Prevents a `zig build` from getting a false positive cache hit following - // a `zig build --cache-poison=ignored`. - config_man.hash.add(cache_poison == .ignored); - - // Normally the build runner is compiled for the host target but here is - // some code to help when debugging edits to the build runner so that you - // can make sure it compiles successfully on other targets. - const resolved_target: Package.Module.ResolvedTarget = t: { - if (build_options.enable_debug_extensions) { - if (debug_target) |triple| { - const target_query = try std.Target.Query.parse(.{ - .arch_os_abi = triple, - }); - config_man.hash.addBytes(triple); - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, target_query), - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - }; - } - } - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, .{}), - .is_native_os = true, - .is_native_abi = true, - .is_explicit_dynamic_linker = false, - }; - }; - - // Likewise, `--debug-libc` allows overriding the libc installation. - const libc_installation: ?*const LibCInstallation = lci: { - const paths_file = debug_libc_paths_file orelse break :lci null; - if (!build_options.enable_debug_extensions) unreachable; - const lci = try arena.create(LibCInstallation); - lci.* = try .parse(arena, io, paths_file, &resolved_target.result); - LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); - break :lci lci; - }; - - // Kick off an optimized compilation of the make runner. - var make_runner_task = if (print_configuration_path) undefined else io.async(compileMakeRunner, .{ gpa, arena, io, .{ - .dirs = .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }, - .environ_map = environ_map, - .parent_prog_node = root_prog_node, - .resolved_target = resolved_target, - .libc_installation = libc_installation, - .thread_limit = thread_limit, - .self_exe_path = self_exe_path, - .color = color, - .reference_trace = reference_trace, - .optimize_mode = maker_optimize_mode, - } }); - defer _ = if (!print_configuration_path) make_runner_task.cancel(io) catch {}; - - const pkg_root: Path = if (override_pkg_dir) |p| - .initCwd(p) - else if (system_pkg_dir_path) |p| - .initCwd(p) - else - .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }; - - make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; - make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - - configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; - - // Dummy http client that is not actually used when fetch_command is unsupported. - // Prevents bootstrap from depending on a bunch of unnecessary stuff. - var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { - allocator: Allocator, - io: Io, - fn deinit(_: @This()) void {} - } = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; - var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; - - { - // Populate fork_set. - var group: Io.Group = .init; - defer group.cancel(io); - - for (forks.items) |*fork| - group.async(io, Fork.load, .{ io, gpa, fork, color }); - - try group.await(io); - - for (forks.items) |*fork| { - if (fork.failed) process.exit(1); - try fork_set.put(arena, .{ - .path = fork.path, - .manifest_ast = fork.manifest_ast, - .manifest = fork.manifest, - .uses = 0, - }, {}); - } - } - defer Fork.deinitList(forks.items); - - var file_system_inputs: std.ArrayList(u8) = .empty; - defer file_system_inputs.deinit(gpa); - - // This loop is re-evaluated when the build script exits with an indication that it - // could not continue due to missing lazy dependencies. - const configuration_path: Path, const poisoned: bool = cp: while (true) { - // We want to release all the locks before executing the child process, so we make a nice - // big block here to ensure the cleanup gets run when we extract out our argv. - { - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "configurer.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .resolved_target = resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = resolved_target, - .single_threaded = true, - }, - .global = config, - .parent = null, - }); - - const build_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}), - .root_src_path = build_root.build_zig_basename, - }, - .fully_qualified_name = "root.@build", - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - - if (dev.env.supports(.fetch_command)) { - const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); - defer fetch_prog_node.end(); - - // Reset fork match counts. - for (fork_set.keys()) |*fork| fork.uses = 0; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = dirs.global_cache, - .local_storage = &.{ - .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" }, - .pkg_root = pkg_root, - }, - .recursive = true, - .debug_hash = false, - .unlazy_set = unlazy_set, - .fork_set = fork_set, - .mode = fetch_mode, - .prog_node = fetch_prog_node, - .read_only = system_pkg_dir_path != null, - }; - defer job_queue.deinit(); - - if (system_pkg_dir_path == null) { - try http_client.initDefaultProxies(arena, environ_map); - } - - try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); - try job_queue.table.ensureUnusedCapacity(gpa, 1); - - const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = phantom_package_root }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = phantom_package_root, - .parent_package_root = phantom_package_root, - .parent_manifest_ast = null, - .prog_node = fetch_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = true, - .oom_flag = false, - .latest_commit = null, - - .module = build_mod, - }; - - job_queue.all_fetches.appendAssumeCapacity(&fetch); - - job_queue.table.putAssumeCapacityNoClobber( - Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache), - &fetch, - ); - - job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); - try job_queue.group.await(io); - - { - // Ensure that forks were actually used. This is done - // before printing manifest errors because using a fork can - // prevent them. - var any_unused = false; - for (fork_set.keys()) |*fork| { - if (fork.uses == 0) { - std.log.err("fork {f} matched no {s} packages", .{ - fork.path, fork.manifest.name, - }); - any_unused = true; - } else { - std.log.info("fork {f} matched {d} {s} packages", .{ - fork.path, fork.uses, fork.manifest.name, - }); - } - } - if (any_unused) process.exit(1); - } - - try job_queue.consolidateErrors(); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } - - if (fetch_only) return cleanExit(io); - - var source_buf = std.array_list.Managed(u8).init(gpa); - defer source_buf.deinit(); - try job_queue.createDependenciesSource(&source_buf); - const deps_mod = try createDependenciesModule( - arena, - io, - source_buf.items, - root_mod, - dirs, - config, - ); - - { - // We need a Module for each package's build.zig. - const hashes = job_queue.table.keys(); - const fetches = job_queue.table.values(); - try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); - for (hashes, fetches) |*hash, f| { - if (f == &fetch) { - // The first one is a dummy package for the current project. - continue; - } - if (!f.has_build_zig) - continue; - const hash_slice = hash.toSlice(); - const mod_root_path = try f.package_root.toString(arena); - const m = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{mod_root_path}), - .root_src_path = Package.build_zig_basename, - }, - .fully_qualified_name = try std.fmt.allocPrint( - arena, - "root.@dependencies.{s}", - .{hash_slice}, - ), - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - const hash_cloned = try arena.dupe(u8, hash_slice); - deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); - f.module = m; - } - - // Each build.zig module needs access to each of its - // dependencies' build.zig modules by name. - for (fetches) |f| { - const mod = f.module orelse continue; - if (!f.have_manifest) continue; - const man = &f.manifest; - const dep_names = man.dependencies.keys(); - try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); - for (dep_names, man.dependencies.values()) |name, dep| { - const dep_digest = Package.Fetch.depDigest( - f.package_root, - dirs.global_cache, - dep, - ) orelse continue; - const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; - const name_cloned = try arena.dupe(u8, name); - mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); - } - } - } - } else try createEmptyDependenciesModule( - arena, - io, - root_mod, - dirs, - config, - ); - - const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); - defer compile_prog_node.end(); - - try root_mod.deps.put(arena, "@build", build_mod); - - file_system_inputs.clearRetainingCapacity(); - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .libc_installation = libc_installation, - .dirs = dirs, - .root_name = "configure", - .config = config, - .root_mod = root_mod, - .main_mod = build_mod, - .emit_bin = .yes_cache, - .self_exe_path = self_exe_path, - .thread_limit = thread_limit, - .verbose_cc = verbose_cc, - .verbose_link = verbose_link, - .verbose_air = verbose_air, - .verbose_intern_pool = verbose_intern_pool, - .verbose_generic_instances = verbose_generic_instances, - .verbose_llvm_ir = verbose_llvm_ir, - .verbose_llvm_bc = verbose_llvm_bc, - .verbose_llvm_cpu_features = verbose_llvm_cpu_features, - .cache_mode = .whole, - .reference_trace = reference_trace, - .debug_compile_errors = debug_compile_errors, - .environ_map = environ_map, - .file_system_inputs = &file_system_inputs, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - updateModule(comp, color, compile_prog_node) catch |err| switch (err) { - error.CompileErrorsReported => process.exit(2), - else => |e| return e, - }; - - // Since incremental compilation isn't done yet, we use cache_mode = whole - // above, and thus the output file is already closed. - //try comp.makeBinFileExecutable(); - const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); - const exe_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), - }; - _ = try config_man.addFilePath(exe_path, null); - configure_argv.items[0] = try exe_path.toString(arena); - - switch (cache_poison) { - .pure, .disallowed, .ignored => if (try config_man.hit()) { - const digest = config_man.final(); - break :cp .{ - .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), - }, - false, - }; - }, - .poisoned => {}, // Don't bother checking for cache hit. - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); - } - - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - const config_tmp_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( - io, - config_tmp_path.sub_path, - .{ .read = true, .exclusive = true }, - ); - defer config_tmp_file.close(io); - - const term = term: { - const child_node = root_prog_node.start("Run Configure Script", 0); - defer child_node.end(); - var child = std.process.spawn(io, .{ - .argv = configure_argv.items, - .stdout = .{ .file = config_tmp_file }, - .progress_node = child_node, - }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err }); - }; - if (!term.success()) { - // Failure to produce the configuration file. - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following configure command {f}:\n{s}", .{ term, cmd }); - } - // Even though the file is designed to be sent directly to make - // runner, we must load it now because: - // * If it contains additional file dependencies, we need to - // add them to `config_man` before obtaining the final digest. - // * If it contains a set of lazy packages that need to be - // fetched, we need to fetch those now and re-run configure. - var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - - if (configuration.unlazy_deps.len != 0) { - if (!dev.env.supports(.fetch_command)) process.exit(1); - var any_errors = false; - for (configuration.unlazy_deps) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); - any_errors = true; - continue; - } - try unlazy_set.put(arena, .fromSlice(hash), {}); - } - if (any_errors) process.exit(1); - if (system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. - const s = fs.path.sep_str; - for (unlazy_set.keys()) |*hash| { - std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); - } - std.log.info("remote package fetching disabled due to --system mode", .{}); - std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(1); - } - continue :cp; - } - - for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; - try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); - } - - // We need to add to the configuration cache the source files of - // configurer itself, so that the maker process can watch the file system - // for those changes and restart itself. By doing this, we make it - // possible to bypass creating a Compilation for configurer on - // Configuration cache hit. - { - var it = mem.splitScalar(u8, file_system_inputs.items, 0); - while (it.next()) |input| { - _ = try config_man.addPrefixedPathPost(.{ - .prefix = input[0], - .sub_path = input[1..], - }); - } - } - - // If it is poisoned, there is no point in moving it to cached - // location. Just leave it in the tmp directory. - if (configuration.poisoned) { - break :cp .{ config_tmp_path, true }; - } else { - const digest = config_man.final(); - const final_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), - }; - Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - ) catch |err| retry: { - const e = switch (err) { - error.FileNotFound => e: { - const dir_path = final_path.dirname().?; - dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| - fatal("failed to create directory {f}: {t}", .{ dir_path, e }); - if (Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - )) |_| break :retry else |e| break :e e; - }, - else => |e| e, - }; - fatal("failed to rename configuration file from {f} into {f}: {t}", .{ - config_tmp_path, final_path, e, - }); - }; - config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); - break :cp .{ final_path, false }; - } - }; - - { - // Release all file system locks just before running the maker process. - var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; - defer if (configuration_lock) |*l| l.release(io); - - if (print_configuration_path) { - var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); - stdout_writer.interface.print("{f}\n", .{configuration_path}) catch - fatal("failed printing cache file path: {t}", .{stdout_writer.err.?}); - stdout_writer.flush() catch |err| - fatal("failed printing cache file path: {t}", .{err}); - return cleanExit(io); - } - const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); - - make_argv.items[0] = try make_runner.exe_path.toString(arena); - make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ - native_os, cmd, - }); - } - - const term = term: { - _ = try io.lockStderr(&.{}, .no_color); - defer io.unlockStderr(); - var child = std.process.spawn(io, .{ - .argv = make_argv.items, - }) catch |err| fatal("failed spawning maker {s}: {t}", .{ make_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed waiting on maker {s}: {t}", .{ make_argv.items[0], err }); - }; - if (term.success()) return cleanExit(io); - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following maker command {f}:\n{s}", .{ term, cmd }); -} - -const MakeRunner = struct { - exe_path: Path, - - const Options = struct { - environ_map: *const process.Environ.Map, - dirs: Compilation.Directories, - parent_prog_node: std.Progress.Node, - resolved_target: Package.Module.ResolvedTarget, - libc_installation: ?*const LibCInstallation, - self_exe_path: []const u8, - thread_limit: usize, - color: Color, - reference_trace: ?u32, - optimize_mode: std.builtin.OptimizeMode, - }; -}; - -fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner { - const compile_prog_node = options.parent_prog_node.start("Compiling Maker (first time setup)", 0); - defer compile_prog_node.end(); - - const strip = options.optimize_mode != .Debug; - - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), - .root_src_path = "Maker.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .root_strip = strip, - .root_optimize_mode = options.optimize_mode, - .resolved_target = options.resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = options.resolved_target, - .optimize_mode = options.optimize_mode, - .strip = strip, - }, - .global = config, - .parent = null, - }); - - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .dirs = options.dirs, - .root_name = "maker", - .config = config, - .root_mod = root_mod, - .main_mod = root_mod, - .emit_bin = .yes_cache, - .self_exe_path = options.self_exe_path, - .thread_limit = options.thread_limit, - .cache_mode = .whole, - .environ_map = options.environ_map, - .reference_trace = options.reference_trace, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - error.Canceled => |e| return e, - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - try updateModule(comp, options.color, compile_prog_node); - - const exe_path: Path = .{ - .root_dir = options.dirs.global_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ - &Cache.binToHex(comp.digest.?), comp.emit_bin.?, - }), - }; - - return .{ - .exe_path = exe_path, - }; -} - -const Fork = struct { - path: Path, - manifest_ast: std.zig.Ast, - manifest: Package.Manifest, - error_bundle: std.zig.ErrorBundle.Wip, - failed: bool, - arena_allocator: std.heap.ArenaAllocator, - - fn init(cwd_relative_path: []const u8) Fork { - return .{ - .manifest_ast = undefined, - .manifest = undefined, - .error_bundle = undefined, - .arena_allocator = undefined, - .path = .{ - .root_dir = .cwd(), - .sub_path = cwd_relative_path, - }, - .failed = false, - }; - } - - fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { - loadFallible(io, gpa, fork, color) catch |err| switch (err) { - error.Canceled => |e| return e, - error.AlreadyReported => fork.failed = true, - else => |e| { - std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); - fork.failed = true; - }, - }; - } - - fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void { - fork.arena_allocator = .init(gpa); - const arena = fork.arena_allocator.allocator(); - - var error_bundle: std.zig.ErrorBundle.Wip = undefined; - try error_bundle.init(gpa); - defer error_bundle.deinit(); - - const manifest_path = try fork.path.join(arena, Package.Manifest.basename); - - Package.Manifest.load( - io, - arena, - manifest_path, - &fork.manifest_ast, - &error_bundle, - &fork.manifest, - true, - ) catch |err| switch (err) { - error.Canceled => |e| return e, - error.ErrorsBundled => { - assert(error_bundle.root_list.items.len > 0); - var errors = try error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - return error.AlreadyReported; - }, - else => |e| { - std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); - return error.AlreadyReported; - }, - }; - } - - fn deinitList(forks: []Fork) void { - for (forks) |*fork| fork.arena_allocator.deinit(); - } -}; - const JitCmdOptions = struct { cmd_name: []const u8, root_src_path: []const u8, prepend_zig_lib_dir_path: bool = false, prepend_global_cache_path: bool = false, prepend_zig_exe_path: bool = false, + prepend_seed: bool = false, depend_on_aro: bool = false, capture: ?*[]u8 = null, /// Send error bundles via std.zig.Server over stdout server: bool = false, - color: Color = .auto, }; fn jitCmd( @@ -6098,8 +5046,10 @@ fn jitCmd( ) !void { dev.check(.jit_command); + const color = Color.settingFromEnvironment(environ_map); + const root_prog_node = std.Progress.start(io, .{ - .disable_printing = (options.color == .off), + .disable_printing = (color == .off), }); defer root_prog_node.end(); @@ -6141,7 +5091,7 @@ fn jitCmdInner( const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( @@ -6158,7 +5108,7 @@ fn jitCmdInner( defer dirs.deinit(io); var child_argv: std.ArrayList([]const u8) = .empty; - try child_argv.ensureUnusedCapacity(arena, args.len + 4); + try child_argv.ensureUnusedCapacity(arena, args.len + 5); // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. @@ -6244,7 +5194,8 @@ fn jitCmdInner( process.exit(2); } } else { - updateModule(comp, options.color, root_prog_node) catch |err| switch (err) { + const color = Color.settingFromEnvironment(environ_map); + updateModule(comp, color, root_prog_node) catch |err| switch (err) { error.CompileErrorsReported => process.exit(2), else => |e| return e, }; @@ -6259,11 +5210,13 @@ fn jitCmdInner( } if (options.prepend_zig_lib_dir_path) - child_argv.appendAssumeCapacity(dirs.zig_lib.path.?); + child_argv.appendAssumeCapacity(try allocPrint(arena, "--zig-lib={s}", .{dirs.zig_lib.path.?})); if (options.prepend_zig_exe_path) - child_argv.appendAssumeCapacity(self_exe_path); + child_argv.appendAssumeCapacity(try allocPrint(arena, "--zig={s}", .{self_exe_path})); if (options.prepend_global_cache_path) - child_argv.appendAssumeCapacity(dirs.global_cache.path.?); + child_argv.appendAssumeCapacity(try allocPrint(arena, "--global-cache={s}", .{dirs.global_cache.path.?})); + if (options.prepend_seed) + child_argv.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)})); child_argv.appendSliceAssumeCapacity(args); @@ -7199,567 +6152,6 @@ fn parseRcIncludes(arg: []const u8) std.zig.RcIncludes { fatal("unsupported rc includes type: {q}", .{arg}); } -const usage_fetch = - \\Usage: zig fetch [options] - \\Usage: zig fetch [options] - \\ - \\ Copy a package into the global cache and print its hash. - \\ must point to one of the following: - \\ - A git+http / git+https server for the package - \\ - A tarball file (with or without compression) containing - \\ package source - \\ - A git bundle file containing package source - \\ - \\Examples: - \\ - \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git - \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz - \\ - \\Options: - \\ -h, --help Print this help and exit - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --cache-dir [path] Override path to local cache directory - \\ --pkg-dir [path] Override path to local package directory - \\ --debug-hash Print verbose hash information to stdout - \\ --debug-log [scope] Enable printing debug/info log messages for scope - \\ --save Add the fetched package to build.zig.zon - \\ --save=[name] Add the fetched package to build.zig.zon as name - \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim - \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim - \\ -; - -fn cmdFetch( - gpa: Allocator, - arena: Allocator, - io: Io, - args: []const []const u8, - environ_map: *process.Environ.Map, -) !void { - dev.check(.fetch_command); - - const color: Color = Color.settingFromEnvironment(environ_map); - var opt_path_or_url: ?[]const u8 = null; - var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var debug_hash: bool = false; - var save: union(enum) { - no, - yes: ?[]const u8, - exact: ?[]const u8, - } = .no; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - try Io.File.stdout().writeStreamingAll(io, usage_fetch); - return cleanExit(io); - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--pkg-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_pkg_dir = args[i]; - } else if (mem.eql(u8, arg, "--debug-hash")) { - debug_hash = true; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - try addDebugLog(arena, args[i]); - } else if (mem.eql(u8, arg, "--save")) { - save = .{ .yes = null }; - } else if (mem.cutPrefix(u8, arg, "--save=")) |rest| { - save = .{ .yes = rest }; - } else if (mem.eql(u8, arg, "--save-exact")) { - save = .{ .exact = null }; - } else if (mem.cutPrefix(u8, arg, "--save-exact=")) |rest| { - save = .{ .exact = rest }; - } else { - fatal("unrecognized parameter: {q}", .{arg}); - } - } else if (opt_path_or_url != null) { - fatal("unexpected extra parameter: {q}", .{arg}); - } else { - opt_path_or_url = arg; - } - } - } - - const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); - - var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - try http_client.initDefaultProxies(arena, environ_map); - - var root_prog_node = std.Progress.start(io, .{ - .root_name = "Fetch", - }); - defer root_prog_node.end(); - - var global_cache_directory: Directory = l: { - const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena, environ_map); - break :l .{ - .handle = try Io.Dir.cwd().createDirPathOpen(io, p, .{}), - .path = p, - }; - }; - defer global_cache_directory.handle.close(io); - - var local_storage: Package.Fetch.LocalStorage = undefined; - var build_root: BuildRoot = undefined; - var build_root_initialized = false; - defer if (build_root_initialized) build_root.deinit(io); - - const cwd_path = try introspect.getResolvedCwd(io, arena); - - const local_storage_ptr = switch (save) { - .no => null, - .yes, .exact => ls: { - build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path }); - build_root_initialized = true; - - local_storage = .{ - .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{ - .root_dir = build_root.directory, - .sub_path = ".zig-cache", - }, - .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }, - }; - - break :ls &local_storage; - }, - }; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = global_cache_directory, - .local_storage = local_storage_ptr, - .recursive = false, - .read_only = false, - .debug_hash = debug_hash, - .mode = .all, - .prog_node = root_prog_node, - }; - defer job_queue.deinit(); - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .path_or_url = path_or_url }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = undefined, - .parent_package_root = undefined, - .parent_manifest_ast = null, - .prog_node = root_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = true, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - defer fetch.deinit(); - - fetch.run() catch |err| switch (err) { - error.OutOfMemory, error.Canceled => |e| return e, - error.FetchFailed => {}, // error bundle checked below - }; - - try job_queue.group.await(io); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } - - const package_hash = fetch.computedPackageHash(); - const package_hash_slice = package_hash.toSlice(); - - root_prog_node.end(); - root_prog_node = .{ .index = .none }; - - const name = switch (save) { - .no => { - var stdout = Io.File.stdout().writerStreaming(io, &stdout_buffer); - try stdout.interface.print("{s}\n", .{package_hash_slice}); - try stdout.interface.flush(); - return cleanExit(io); - }, - .yes, .exact => |name| name: { - if (name) |n| break :name n; - if (!fetch.have_manifest) - fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); - break :name fetch.manifest.name; - }, - }; - - // The name to use in case the manifest file needs to be created now. - const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path); - var manifest, var ast = try loadManifest(gpa, arena, io, .{ - .root_name = try sanitizeExampleName(arena, init_root_name), - .dir = build_root.directory.handle, - .color = color, - }); - defer { - manifest.deinit(gpa); - ast.deinit(gpa); - } - - var fixups: Ast.Render.Fixups = .{}; - defer fixups.deinit(gpa); - - var saved_path_or_url = path_or_url; - - if (fetch.latest_commit) |latest_commit| resolved: { - const latest_commit_hex = try std.fmt.allocPrint(arena, "{f}", .{latest_commit}); - - var uri = try std.Uri.parse(path_or_url); - - if (uri.fragment) |fragment| { - const target_ref = try fragment.toRawMaybeAlloc(arena); - - // the refspec may already be fully resolved - if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; - - std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex }); - - // include the original refspec in a query parameter, could be used to check for updates - uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={f}", .{ - std.fmt.alt(fragment, .formatEscaped), - }) }; - } else { - std.log.info("resolved to commit {s}", .{latest_commit_hex}); - } - - // replace the refspec with the resolved commit SHA - uri.fragment = .{ .raw = latest_commit_hex }; - - switch (save) { - .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{f}", .{uri}), - .no, .exact => {}, // keep the original URL - } - } - - const new_node_init = try std.fmt.allocPrint(arena, - \\.{{ - \\ .url = "{f}", - \\ .hash = "{f}", - \\ }} - , .{ - std.zig.fmtString(saved_path_or_url), - std.zig.fmtString(package_hash_slice), - }); - - const new_node_text = try std.fmt.allocPrint(arena, ".{f} = {s},\n", .{ - std.zig.fmtIdPU(name), new_node_init, - }); - - const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ - new_node_text, - }); - - const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ - dependencies_init, - }); - - if (manifest.dependencies.get(name)) |dep| { - if (dep.hash) |h| { - switch (dep.location) { - .url => |u| { - if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { - std.log.info("existing dependency named {q} is up-to-date", .{name}); - process.exit(0); - } - }, - .path => {}, - } - } - - const location_replace = try std.fmt.allocPrint( - arena, - "\"{f}\"", - .{std.zig.fmtString(saved_path_or_url)}, - ); - const hash_replace = try std.fmt.allocPrint( - arena, - "\"{f}\"", - .{std.zig.fmtString(package_hash_slice)}, - ); - - warn("overwriting existing dependency named {q}", .{name}); - try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); - if (dep.hash_node.unwrap()) |hash_node| { - try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); - } else { - // https://github.com/ziglang/zig/issues/21690 - } - } else if (manifest.dependencies.count() > 0) { - // Add fixup for adding another dependency. - const deps = manifest.dependencies.values(); - const last_dep_node = deps[deps.len - 1].node; - try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); - } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { - // Add fixup for replacing the entire dependencies struct. - try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); - } else { - // Add fixup for adding dependencies struct. - try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); - } - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - try ast.render(gpa, &aw.writer, fixups); - const rendered = aw.written(); - - build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch |err| { - fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err }); - }; - - return cleanExit(io); -} - -fn createEmptyDependenciesModule( - arena: Allocator, - io: Io, - main_mod: *Package.Module, - dirs: Compilation.Directories, - global_options: Compilation.Config, -) !void { - var source = std.array_list.Managed(u8).init(arena); - try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); - _ = try createDependenciesModule( - arena, - io, - source.items, - main_mod, - dirs, - global_options, - ); -} - -/// Creates the dependencies.zig file and corresponding `Package.Module` for the -/// build runner to obtain via `@import("@dependencies")`. -fn createDependenciesModule( - arena: Allocator, - io: Io, - source: []const u8, - main_mod: *Package.Module, - dirs: Compilation.Directories, - global_options: Compilation.Config, -) !*Package.Module { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - { - var tmp_dir = try dirs.local_cache.handle.createDirPathOpen(io, tmp_dir_sub_path, .{}); - defer tmp_dir.close(io); - try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source }); - } - const tmp_dir_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(source); - const hex_digest = hh.final(); - - const o_dir_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest), - }; - try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path); - - const deps_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path), - .root_src_path = basename, - }, - .fully_qualified_name = "root.@dependencies", - .parent = main_mod, - .cc_argv = &.{}, - .inherited = .{}, - .global = global_options, - }); - try main_mod.deps.put(arena, "@dependencies", deps_mod); - return deps_mod; -} - -const BuildRoot = struct { - directory: Cache.Directory, - build_zig_basename: []const u8, - cleanup_build_dir: ?Io.Dir, - - fn deinit(br: *BuildRoot, io: Io) void { - if (br.cleanup_build_dir) |*dir| dir.close(io); - br.* = undefined; - } -}; - -const FindBuildRootOptions = struct { - build_file: ?[]const u8 = null, - cwd_path: ?[]const u8 = null, -}; - -fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot { - const cwd_path = options.cwd_path orelse try introspect.getResolvedCwd(io, arena); - const build_zig_basename = if (options.build_file) |bf| - fs.path.basename(bf) - else - Package.build_zig_basename; - - if (options.build_file) |bf| { - if (fs.path.dirname(bf)) |dirname| { - const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { - fatal("unable to open directory to build file from argument 'build-file', {q}: {t}", .{ dirname, err }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = dirname, .handle = dir }, - .cleanup_build_dir = dir, - }; - } - - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = null, .handle = Io.Dir.cwd() }, - .cleanup_build_dir = null, - }; - } - // Search up parent directories until we find build.zig. - var dirname: []const u8 = cwd_path; - while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); - if (Io.Dir.cwd().access(io, joined_path, .{})) |_| { - const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { - fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ - .path = dirname, - .handle = dir, - }, - .cleanup_build_dir = dir, - }; - } else |err| switch (err) { - error.FileNotFound => { - dirname = fs.path.dirname(dirname) orelse { - std.log.info("initialize {s} template file with 'zig init'", .{ - Package.build_zig_basename, - }); - std.log.info("see 'zig --help' for more options", .{}); - fatal("no build.zig file found, in the current directory or any parent directories", .{}); - }; - continue; - }, - else => |e| return e, - } - } -} - -const LoadManifestOptions = struct { - root_name: []const u8, - dir: Io.Dir, - color: Color, -}; - -fn loadManifest( - gpa: Allocator, - arena: Allocator, - io: Io, - options: LoadManifestOptions, -) !struct { Package.Manifest, Ast } { - const rng: std.Random.IoSource = .{ .io = io }; - - const manifest_bytes = while (true) { - break options.dir.readFileAllocOptions( - io, - Package.Manifest.basename, - arena, - .limited(Package.Manifest.max_bytes), - .@"1", - 0, - ) catch |err| switch (err) { - error.FileNotFound => { - writeSimpleTemplateFile(io, Package.Manifest.basename, - \\.{{ - \\ .name = .{s}, - \\ .version = "{s}", - \\ .paths = .{{""}}, - \\ .fingerprint = 0x{x}, - \\}} - \\ - , .{ - options.root_name, - build_options.version, - Package.Fingerprint.generate(rng.interface(), options.root_name).int(), - }) catch |e| { - fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e }); - }; - continue; - }, - else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }), - }; - }; - var ast = try Ast.parse(gpa, manifest_bytes, .zon); - errdefer ast.deinit(gpa); - - if (ast.errors.len > 0) { - try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color); - process.exit(2); - } - - var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); - errdefer manifest.deinit(gpa); - - if (manifest.errors.len > 0) { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - const src_path = try wip_errors.addString(Package.Manifest.basename); - try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); - - var error_bundle = try wip_errors.toOwnedBundle(""); - defer error_bundle.deinit(gpa); - error_bundle.renderToStderr(io, .{}, options.color) catch {}; - - process.exit(2); - } - return .{ manifest, ast }; -} - const Templates = struct { zig_lib_directory: Cache.Directory, dir: Io.Dir, @@ -7834,13 +6226,13 @@ fn writeSimpleTemplateFile(io: Io, file_name: []const u8, comptime fmt: []const } fn findTemplates(gpa: Allocator, arena: Allocator, io: Io) Templates { - const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| { + const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| { fatal("unable to get cwd: {t}", .{err}); }; const self_exe_path = process.executablePathAlloc(io, arena) catch |err| { fatal("unable to find self exe path: {t}", .{err}); }; - var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch |err| { + var zig_lib_directory = std.zig.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch |err| { fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err }); }; diff --git a/src/print_env.zig b/src/print_env.zig index 9370006c72df78adc2725178d2644ca44fbbfcae..93e14781a184db46288b9c5e8c3d77e598820f2c 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -8,7 +8,6 @@ const fatal = std.process.fatal; const build_options = @import("build_options"); const Compilation = @import("Compilation.zig"); -const introspect = @import("introspect.zig"); pub fn cmdEnv( arena: Allocator, @@ -29,7 +28,7 @@ pub fn cmdEnv( }, }; - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); var dirs: Compilation.Directories = .init( arena, diff --git a/src/print_targets.zig b/src/print_targets.zig index c3a9ff44584ee2eb304cb63531fbda56ee1c98a1..702a684de3e829b8e7d475c265a5ed2e584caa1e 100644 --- a/src/print_targets.zig +++ b/src/print_targets.zig @@ -9,7 +9,6 @@ const Target = std.Target; const assert = std.debug.assert; const glibc = @import("libs/glibc.zig"); -const introspect = @import("introspect.zig"); const target = @import("target.zig"); pub fn cmdTargets( @@ -20,7 +19,7 @@ pub fn cmdTargets( native_target: *const Target, ) !void { _ = args; - var zig_lib_directory = introspect.findZigLibDir(allocator, io) catch |err| + var zig_lib_directory = std.zig.findZigLibDir(allocator, io) catch |err| fatal("unable to find zig installation directory: {t}", .{err}); defer zig_lib_directory.handle.close(io); defer allocator.free(zig_lib_directory.path.?); -- 2.54.0