| author |  Andrew Kelley <andrew@ziglang.org> 2025-03-27 17:21:16-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2025-03-27 17:21:16-07:00 |
| log | b31c32879d9c8b368726012fc5b98fa1d90453ce |
| tree | b3a87dfe9a0229b3bda36b172ef8290f3a8e8c15 |
| parent | d0911786c95dfa7a63ec348bb4a9870da12f62e4 |
WIP
36 files changed, 8264 insertions(+), 8571 deletions(-)
CMakeLists.txt-5
| ... | ... | @@ -513,11 +513,6 @@ set(ZIG_STAGE2_SOURCES |
| 513 | 513 | src/InternPool.zig |
| 514 | 514 | src/Liveness.zig |
| 515 | 515 | src/Liveness/Verify.zig |
| 516 | src/Package.zig | |
| 517 | src/Package/Fetch.zig | |
| 518 | src/Package/Fetch/git.zig | |
| 519 | src/Package/Manifest.zig | |
| 520 | src/Package/Module.zig | |
| 521 | 516 | src/RangeSet.zig |
| 522 | 517 | src/Sema.zig |
| 523 | 518 | src/Sema/bitcast.zig |
lib/compiler/build.zig created+1790
| ... | ... | @@ -0,0 +1,1790 @@ |
| 1 | const builtin = @import("builtin"); | |
| 2 | const native_os = builtin.os.tag; | |
| 3 | ||
| 4 | const std = @import("std"); | |
| 5 | const assert = std.debug.assert; | |
| 6 | const io = std.io; | |
| 7 | const fmt = std.fmt; | |
| 8 | const mem = std.mem; | |
| 9 | const process = std.process; | |
| 10 | const ArrayList = std.ArrayList; | |
| 11 | const File = std.fs.File; | |
| 12 | const Step = std.Build.Step; | |
| 13 | const Watch = std.Build.Watch; | |
| 14 | const Fuzz = std.Build.Fuzz; | |
| 15 | const Allocator = std.mem.Allocator; | |
| 16 | const fatal = std.process.fatal; | |
| 17 | const Directory = std.Build.Cache.Directory; | |
| 18 | const Package = std.zig.Package; | |
| 19 | ||
| 20 | pub const std_options: std.Options = .{ | |
| 21 | .side_channels_mitigations = .none, | |
| 22 | .crypto_fork_safety = false, | |
| 23 | }; | |
| 24 | ||
| 25 | pub fn main() !void { | |
| 26 | // Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived, | |
| 27 | // one shot program. We don't need to waste time freeing memory and finding places to squish | |
| 28 | // bytes into. So we free everything all at once at the very end. | |
| 29 | var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | |
| 30 | defer single_threaded_arena.deinit(); | |
| 31 | ||
| 32 | var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ | |
| 33 | .child_allocator = single_threaded_arena.allocator(), | |
| 34 | }; | |
| 35 | const arena = thread_safe_arena.allocator(); | |
| 36 | const gpa = arena; | |
| 37 | ||
| 38 | const args = try process.argsAlloc(arena); | |
| 39 | ||
| 40 | // skip my own exe name | |
| 41 | var arg_idx: usize = 1; | |
| 42 | ||
| 43 | const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); | |
| 44 | const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); | |
| 45 | const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); | |
| 46 | const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); | |
| 47 | ||
| 48 | const zig_lib_directory: Directory = .{ | |
| 49 | .path = zig_lib_dir, | |
| 50 | .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), | |
| 51 | }; | |
| 52 | ||
| 53 | const local_cache_directory: Directory = .{ | |
| 54 | .path = cache_root, | |
| 55 | .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), | |
| 56 | }; | |
| 57 | ||
| 58 | const global_cache_directory: Directory = .{ | |
| 59 | .path = global_cache_root, | |
| 60 | .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), | |
| 61 | }; | |
| 62 | ||
| 63 | var graph: std.Build.Graph = .{ | |
| 64 | .arena = arena, | |
| 65 | .cache = .{ | |
| 66 | .gpa = gpa, | |
| 67 | .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), | |
| 68 | }, | |
| 69 | .zig_exe = zig_exe, | |
| 70 | .env_map = try process.getEnvMap(arena), | |
| 71 | .global_cache_root = global_cache_directory, | |
| 72 | .zig_lib_directory = zig_lib_directory, | |
| 73 | .host = .{ | |
| 74 | .query = .{}, | |
| 75 | .result = try std.zig.system.resolveTargetQuery(.{}), | |
| 76 | }, | |
| 77 | }; | |
| 78 | ||
| 79 | var targets = ArrayList([]const u8).init(arena); | |
| 80 | var debug_log_scopes = ArrayList([]const u8).init(arena); | |
| 81 | var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = gpa }; | |
| 82 | var options_args: std.ArrayListUnmanaged([]const u8) = .empty; | |
| 83 | ||
| 84 | var install_prefix: ?[]const u8 = null; | |
| 85 | var install_paths: std.Build.InstallPaths = .{}; | |
| 86 | var summary: ?Summary = null; | |
| 87 | var max_rss: u64 = 0; | |
| 88 | var skip_oom_steps = false; | |
| 89 | var color: Color = .auto; | |
| 90 | var prominent_compile_errors = false; | |
| 91 | var help_menu = false; | |
| 92 | var steps_menu = false; | |
| 93 | var watch = false; | |
| 94 | var fuzz = false; | |
| 95 | var debounce_interval_ms: u16 = 50; | |
| 96 | var listen_port: u16 = 0; | |
| 97 | var remaining_args: ?[]const []const u8 = null; | |
| 98 | ||
| 99 | var build_file: ?[]const u8 = null; | |
| 100 | var reference_trace: ?u32 = null; | |
| 101 | var debug_compile_errors = false; | |
| 102 | var verbose_link = (native_os != .wasi or builtin.link_libc) and std.zig.EnvVar.ZIG_VERBOSE_LINK.isSet(); | |
| 103 | var verbose_cc = (native_os != .wasi or builtin.link_libc) and std.zig.EnvVar.ZIG_VERBOSE_CC.isSet(); | |
| 104 | var verbose_air = false; | |
| 105 | var verbose_intern_pool = false; | |
| 106 | var verbose_generic_instances = false; | |
| 107 | var verbose_llvm_ir: ?[]const u8 = null; | |
| 108 | var verbose_llvm_bc: ?[]const u8 = null; | |
| 109 | var verbose_cimport = false; | |
| 110 | var verbose_llvm_cpu_features = false; | |
| 111 | var fetch_only = false; | |
| 112 | ||
| 113 | while (nextArg(args, &arg_idx)) |arg| { | |
| 114 | if (mem.startsWith(u8, arg, "-D")) { | |
| 115 | try options_args.append(arena, arg); | |
| 116 | } else if (mem.startsWith(u8, arg, "-")) { | |
| 117 | if (mem.eql(u8, arg, "--verbose")) { | |
| 118 | graph.verbose = true; | |
| 119 | } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { | |
| 120 | help_menu = true; | |
| 121 | } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { | |
| 122 | install_prefix = nextArgOrFatal(args, &arg_idx); | |
| 123 | } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { | |
| 124 | steps_menu = true; | |
| 125 | } else if (mem.startsWith(u8, arg, "-fsys=")) { | |
| 126 | const name = arg["-fsys=".len..]; | |
| 127 | graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); | |
| 128 | } else if (mem.startsWith(u8, arg, "-fno-sys=")) { | |
| 129 | const name = arg["-fno-sys=".len..]; | |
| 130 | graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); | |
| 131 | } else if (mem.eql(u8, arg, "--release")) { | |
| 132 | graph.release_mode = .any; | |
| 133 | } else if (mem.startsWith(u8, arg, "--release=")) { | |
| 134 | const text = arg["--release=".len..]; | |
| 135 | graph.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { | |
| 136 | fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ | |
| 137 | arg, text, | |
| 138 | }); | |
| 139 | }; | |
| 140 | } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { | |
| 141 | install_paths.lib_dir = nextArgOrFatal(args, &arg_idx); | |
| 142 | } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { | |
| 143 | install_paths.exe_dir = nextArgOrFatal(args, &arg_idx); | |
| 144 | } else if (mem.eql(u8, arg, "--prefix-include-dir")) { | |
| 145 | install_paths.include_dir = nextArgOrFatal(args, &arg_idx); | |
| 146 | } else if (mem.eql(u8, arg, "--sysroot")) { | |
| 147 | graph.sysroot = nextArgOrFatal(args, &arg_idx); | |
| 148 | } else if (mem.eql(u8, arg, "--maxrss")) { | |
| 149 | const max_rss_text = nextArgOrFatal(args, &arg_idx); | |
| 150 | max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { | |
| 151 | std.debug.print("invalid byte size: '{s}': {s}\n", .{ | |
| 152 | max_rss_text, @errorName(err), | |
| 153 | }); | |
| 154 | process.exit(1); | |
| 155 | }; | |
| 156 | } else if (mem.eql(u8, arg, "--skip-oom-steps")) { | |
| 157 | skip_oom_steps = true; | |
| 158 | } else if (mem.eql(u8, arg, "--search-prefix")) { | |
| 159 | const search_prefix = nextArgOrFatal(args, &arg_idx); | |
| 160 | graph.addSearchPrefix(search_prefix); | |
| 161 | } else if (mem.eql(u8, arg, "--libc")) { | |
| 162 | graph.libc_file = nextArgOrFatal(args, &arg_idx); | |
| 163 | } else if (mem.eql(u8, arg, "--build-file")) { | |
| 164 | build_file = nextArgOrFatal(args, &arg_idx); | |
| 165 | } else if (mem.eql(u8, arg, "--color")) { | |
| 166 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 167 | fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); | |
| 168 | color = std.meta.stringToEnum(Color, next_arg) orelse { | |
| 169 | fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ | |
| 170 | arg, next_arg, | |
| 171 | }); | |
| 172 | }; | |
| 173 | } else if (mem.eql(u8, arg, "--summary")) { | |
| 174 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 175 | fatalWithHint("expected [all|new|failures|none] after '{s}'", .{arg}); | |
| 176 | summary = std.meta.stringToEnum(Summary, next_arg) orelse { | |
| 177 | fatalWithHint("expected [all|new|failures|none] after '{s}', found '{s}'", .{ | |
| 178 | arg, next_arg, | |
| 179 | }); | |
| 180 | }; | |
| 181 | } else if (mem.eql(u8, arg, "--seed")) { | |
| 182 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 183 | fatalWithHint("expected u32 after '{s}'", .{arg}); | |
| 184 | graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { | |
| 185 | fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ | |
| 186 | next_arg, @errorName(err), | |
| 187 | }); | |
| 188 | }; | |
| 189 | } else if (mem.eql(u8, arg, "--debounce")) { | |
| 190 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 191 | fatalWithHint("expected u16 after '{s}'", .{arg}); | |
| 192 | debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { | |
| 193 | fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {s}\n", .{ | |
| 194 | next_arg, @errorName(err), | |
| 195 | }); | |
| 196 | }; | |
| 197 | } else if (mem.eql(u8, arg, "--port")) { | |
| 198 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 199 | fatalWithHint("expected u16 after '{s}'", .{arg}); | |
| 200 | listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| { | |
| 201 | fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{ | |
| 202 | next_arg, @errorName(err), | |
| 203 | }); | |
| 204 | }; | |
| 205 | } else if (mem.eql(u8, arg, "--debug-log")) { | |
| 206 | const next_arg = nextArgOrFatal(args, &arg_idx); | |
| 207 | try debug_log_scopes.append(next_arg); | |
| 208 | } else if (mem.eql(u8, arg, "--debug-pkg-config")) { | |
| 209 | graph.debug_pkg_config = true; | |
| 210 | } else if (mem.eql(u8, arg, "--debug-rt")) { | |
| 211 | graph.debug_compiler_runtime_libs = true; | |
| 212 | } else if (mem.eql(u8, arg, "--debug-compile-errors")) { | |
| 213 | graph.debug_compile_errors = true; | |
| 214 | } else if (mem.eql(u8, arg, "--system")) { | |
| 215 | graph.system_package_mode = nextArgOrFatal(args, &arg_idx); | |
| 216 | } else if (mem.eql(u8, arg, "--glibc-runtimes")) { | |
| 217 | graph.glibc_runtimes_dir = nextArgOrFatal(args, &arg_idx); | |
| 218 | } else if (mem.eql(u8, arg, "--verbose-link")) { | |
| 219 | graph.verbose_link = true; | |
| 220 | } else if (mem.eql(u8, arg, "--verbose-air")) { | |
| 221 | graph.verbose_air = true; | |
| 222 | } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { | |
| 223 | graph.verbose_llvm_ir = "-"; | |
| 224 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { | |
| 225 | graph.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; | |
| 226 | } else if (mem.eql(u8, arg, "--verbose-llvm-bc=")) { | |
| 227 | graph.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; | |
| 228 | } else if (mem.eql(u8, arg, "--verbose-cimport")) { | |
| 229 | graph.verbose_cimport = true; | |
| 230 | } else if (mem.eql(u8, arg, "--verbose-cc")) { | |
| 231 | graph.verbose_cc = true; | |
| 232 | } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { | |
| 233 | graph.verbose_llvm_cpu_features = true; | |
| 234 | } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { | |
| 235 | prominent_compile_errors = true; | |
| 236 | } else if (mem.eql(u8, arg, "--watch")) { | |
| 237 | watch = true; | |
| 238 | } else if (mem.eql(u8, arg, "--fuzz")) { | |
| 239 | fuzz = true; | |
| 240 | } else if (mem.eql(u8, arg, "--fetch")) { | |
| 241 | fetch_only = true; | |
| 242 | } else if (mem.eql(u8, arg, "-fincremental")) { | |
| 243 | graph.incremental = true; | |
| 244 | } else if (mem.eql(u8, arg, "-fno-incremental")) { | |
| 245 | graph.incremental = false; | |
| 246 | } else if (mem.eql(u8, arg, "-fwine")) { | |
| 247 | graph.enable_wine = true; | |
| 248 | } else if (mem.eql(u8, arg, "-fno-wine")) { | |
| 249 | graph.enable_wine = false; | |
| 250 | } else if (mem.eql(u8, arg, "-fqemu")) { | |
| 251 | graph.enable_qemu = true; | |
| 252 | } else if (mem.eql(u8, arg, "-fno-qemu")) { | |
| 253 | graph.enable_qemu = false; | |
| 254 | } else if (mem.eql(u8, arg, "-fwasmtime")) { | |
| 255 | graph.enable_wasmtime = true; | |
| 256 | } else if (mem.eql(u8, arg, "-fno-wasmtime")) { | |
| 257 | graph.enable_wasmtime = false; | |
| 258 | } else if (mem.eql(u8, arg, "-frosetta")) { | |
| 259 | graph.enable_rosetta = true; | |
| 260 | } else if (mem.eql(u8, arg, "-fno-rosetta")) { | |
| 261 | graph.enable_rosetta = false; | |
| 262 | } else if (mem.eql(u8, arg, "-fdarling")) { | |
| 263 | graph.enable_darling = true; | |
| 264 | } else if (mem.eql(u8, arg, "-fno-darling")) { | |
| 265 | graph.enable_darling = false; | |
| 266 | } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { | |
| 267 | graph.allow_so_scripts = true; | |
| 268 | } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { | |
| 269 | graph.allow_so_scripts = false; | |
| 270 | } else if (mem.eql(u8, arg, "-freference-trace")) { | |
| 271 | graph.reference_trace = 256; | |
| 272 | } else if (mem.startsWith(u8, arg, "-freference-trace=")) { | |
| 273 | const num = arg["-freference-trace=".len..]; | |
| 274 | graph.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 275 | std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); | |
| 276 | process.exit(1); | |
| 277 | }; | |
| 278 | } else if (mem.eql(u8, arg, "-fno-reference-trace")) { | |
| 279 | graph.reference_trace = null; | |
| 280 | } else if (mem.startsWith(u8, arg, "-j")) { | |
| 281 | const num = arg["-j".len..]; | |
| 282 | const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 283 | std.debug.print("unable to parse jobs count '{s}': {s}", .{ | |
| 284 | num, @errorName(err), | |
| 285 | }); | |
| 286 | process.exit(1); | |
| 287 | }; | |
| 288 | if (n_jobs < 1) { | |
| 289 | std.debug.print("number of jobs must be at least 1\n", .{}); | |
| 290 | process.exit(1); | |
| 291 | } | |
| 292 | thread_pool_options.n_jobs = n_jobs; | |
| 293 | } else if (mem.eql(u8, arg, "--")) { | |
| 294 | remaining_args = argsRest(args, arg_idx); | |
| 295 | break; | |
| 296 | } else { | |
| 297 | fatalWithHint("unrecognized argument: '{s}'", .{arg}); | |
| 298 | } | |
| 299 | } else { | |
| 300 | try targets.append(arg); | |
| 301 | } | |
| 302 | } | |
| 303 | graph.debug_log_scopes = debug_log_scopes.items; | |
| 304 | ||
| 305 | const cwd_path = try process.getCwdAlloc(arena); | |
| 306 | const build_root = try Package.findBuildRoot(arena, .{ | |
| 307 | .cwd_path = cwd_path, | |
| 308 | .build_file = build_file, | |
| 309 | }); | |
| 310 | ||
| 311 | graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); | |
| 312 | graph.cache.addPrefix(build_root.directory); | |
| 313 | graph.cache.addPrefix(local_cache_directory); | |
| 314 | graph.cache.addPrefix(global_cache_directory); | |
| 315 | graph.cache.hash.addBytes(builtin.zig_version_string); | |
| 316 | ||
| 317 | ||
| 318 | const stderr = std.io.getStdErr(); | |
| 319 | const ttyconf = get_tty_conf(color, stderr); | |
| 320 | switch (ttyconf) { | |
| 321 | .no_color => try graph.env_map.put("NO_COLOR", "1"), | |
| 322 | .escape_codes => try graph.env_map.put("CLICOLOR_FORCE", "1"), | |
| 323 | .windows_api => {}, | |
| 324 | } | |
| 325 | ||
| 326 | const main_progress_node = std.Progress.start(.{ | |
| 327 | .disable_printing = (color == .off), | |
| 328 | }); | |
| 329 | defer main_progress_node.end(); | |
| 330 | ||
| 331 | var thread_pool: std.Thread.Pool = undefined; | |
| 332 | try thread_pool.init(thread_pool_options); | |
| 333 | defer thread_pool.deinit(); | |
| 334 | ||
| 335 | ||
| 336 | { | |
| 337 | var compile_argv: std.ArrayListUnmanaged([]const u8) = .empty; | |
| 338 | defer compile_argv.deinit(gpa); | |
| 339 | ||
| 340 | var run_argv: std.ArrayListUnmanaged([]const u8) = .empty; | |
| 341 | defer run_argv.deinit(gpa); | |
| 342 | ||
| 343 | var cli_modules: std.StringArrayHashMapUnmanaged(CliModule) = .empty; | |
| 344 | defer cli_modules.deinit(gpa); | |
| 345 | ||
| 346 | const configure_runner_module = try std.fmt.allocPrint(arena, "-Mroot={s}/lib/configure_runner.zig", .{ | |
| 347 | zig_lib_dir, | |
| 348 | }); | |
| 349 | const build_zig_module = try std.fmt.allocPrint(arena, "-M@build={}/{s}", .{ | |
| 350 | build_root.directory, build_root.build_zig_basename, | |
| 351 | }); | |
| 352 | ||
| 353 | const exe_basename = try std.zig.binNameAlloc(arena, .{ | |
| 354 | .root_name = "configure", | |
| 355 | .target = graph.host.result, | |
| 356 | .output_mode = .Exe, | |
| 357 | }); | |
| 358 | var http_client: std.http.Client = .{ .allocator = gpa }; | |
| 359 | defer http_client.deinit(); | |
| 360 | ||
| 361 | var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; | |
| 362 | ||
| 363 | // This loop is re-evaluated when the build script exits with an indication that it | |
| 364 | // could not continue due to missing lazy dependencies. | |
| 365 | while (true) { | |
| 366 | // We want to release all the locks before executing the child process, so we make a nice | |
| 367 | // big block here to ensure the cleanup gets run when we extract out our argv. | |
| 368 | { | |
| 369 | { | |
| 370 | cli_modules.clearRetainingCapacity(); | |
| 371 | const root_mod = try addCliModule(gpa, arena, &cli_modules, configure_runner_module); | |
| 372 | const build_mod = try addCliModule(gpa, arena, &cli_modules, build_zig_module); | |
| 373 | ||
| 374 | const fetch_prog_node = main_progress_node.start("Fetch Packages", 0); | |
| 375 | defer fetch_prog_node.end(); | |
| 376 | ||
| 377 | const work_around_btrfs_bug = native_os == .linux and | |
| 378 | std.zig.EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); | |
| 379 | ||
| 380 | var job_queue: Package.Fetch.JobQueue = .{ | |
| 381 | .http_client = &http_client, | |
| 382 | .thread_pool = &thread_pool, | |
| 383 | .global_cache = global_cache_directory, | |
| 384 | .read_only = false, | |
| 385 | .recursive = true, | |
| 386 | .debug_hash = false, | |
| 387 | .work_around_btrfs_bug = work_around_btrfs_bug, | |
| 388 | .unlazy_set = unlazy_set, | |
| 389 | }; | |
| 390 | defer job_queue.deinit(); | |
| 391 | ||
| 392 | if (graph.system_package_mode) |p| { | |
| 393 | job_queue.global_cache = p; | |
| 394 | job_queue.read_only = true; | |
| 395 | } else { | |
| 396 | try http_client.initDefaultProxies(arena); | |
| 397 | } | |
| 398 | ||
| 399 | try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); | |
| 400 | try job_queue.table.ensureUnusedCapacity(gpa, 1); | |
| 401 | ||
| 402 | var fetch: Package.Fetch = .{ | |
| 403 | .arena = std.heap.ArenaAllocator.init(gpa), | |
| 404 | .location = .{ .relative_path = build_root.directory }, | |
| 405 | .location_tok = 0, | |
| 406 | .hash_tok = .none, | |
| 407 | .name_tok = 0, | |
| 408 | .lazy_status = .eager, | |
| 409 | .parent_package_root = build_root.directory, | |
| 410 | .parent_manifest_ast = null, | |
| 411 | .prog_node = fetch_prog_node, | |
| 412 | .job_queue = &job_queue, | |
| 413 | .omit_missing_hash_error = true, | |
| 414 | .allow_missing_paths_field = false, | |
| 415 | .allow_missing_fingerprint = false, | |
| 416 | .allow_name_string = false, | |
| 417 | .use_latest_commit = false, | |
| 418 | ||
| 419 | .package_root = undefined, | |
| 420 | .error_bundle = undefined, | |
| 421 | .manifest = null, | |
| 422 | .manifest_ast = undefined, | |
| 423 | .computed_hash = undefined, | |
| 424 | .has_build_zig = true, | |
| 425 | .oom_flag = false, | |
| 426 | .latest_commit = null, | |
| 427 | ||
| 428 | .userdata = build_mod, | |
| 429 | }; | |
| 430 | job_queue.all_fetches.appendAssumeCapacity(&fetch); | |
| 431 | ||
| 432 | job_queue.table.putAssumeCapacityNoClobber( | |
| 433 | Package.Fetch.relativePathDigest(build_root.directory, global_cache_directory), | |
| 434 | &fetch, | |
| 435 | ); | |
| 436 | ||
| 437 | job_queue.thread_pool.spawnWg(&job_queue.wait_group, Package.Fetch.workerRun, .{ | |
| 438 | &fetch, "root", | |
| 439 | }); | |
| 440 | job_queue.wait_group.wait(); | |
| 441 | ||
| 442 | try job_queue.consolidateErrors(); | |
| 443 | ||
| 444 | if (fetch.error_bundle.root_list.items.len > 0) { | |
| 445 | var errors = try fetch.error_bundle.toOwnedBundle(""); | |
| 446 | errors.renderToStdErr(color.renderOptions()); | |
| 447 | process.exit(1); | |
| 448 | } | |
| 449 | ||
| 450 | if (fetch_only) return std.process.cleanExit(); | |
| 451 | ||
| 452 | var source_buf = std.ArrayList(u8).init(gpa); | |
| 453 | defer source_buf.deinit(); | |
| 454 | try job_queue.createDependenciesSource(&source_buf); | |
| 455 | const deps_mod = try createDependenciesModule( | |
| 456 | arena, | |
| 457 | source_buf.items, | |
| 458 | root_mod, | |
| 459 | global_cache_directory, | |
| 460 | local_cache_directory, | |
| 461 | builtin_mod, | |
| 462 | config, | |
| 463 | ); | |
| 464 | ||
| 465 | { | |
| 466 | // We need a Module for each package's build.zig. | |
| 467 | const hashes = job_queue.table.keys(); | |
| 468 | const fetches = job_queue.table.values(); | |
| 469 | try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); | |
| 470 | for (hashes, fetches) |*hash, f| { | |
| 471 | if (f == &fetch) { | |
| 472 | // The first one is a dummy package for the current project. | |
| 473 | continue; | |
| 474 | } | |
| 475 | if (!f.has_build_zig) | |
| 476 | continue; | |
| 477 | const hash_slice = hash.toSlice(); | |
| 478 | const m = try Package.Module.create(arena, .{ | |
| 479 | .global_cache_directory = global_cache_directory, | |
| 480 | .paths = .{ | |
| 481 | .root = try f.package_root.clone(arena), | |
| 482 | .root_src_path = Package.build_zig_basename, | |
| 483 | }, | |
| 484 | .fully_qualified_name = try std.fmt.allocPrint( | |
| 485 | arena, | |
| 486 | "root.@dependencies.{s}", | |
| 487 | .{hash_slice}, | |
| 488 | ), | |
| 489 | .cc_argv = &.{}, | |
| 490 | .inherited = .{}, | |
| 491 | .global = config, | |
| 492 | .parent = root_mod, | |
| 493 | .builtin_mod = builtin_mod, | |
| 494 | .builtin_modules = null, // `builtin_mod` is specified | |
| 495 | }); | |
| 496 | const hash_cloned = try arena.dupe(u8, hash_slice); | |
| 497 | deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); | |
| 498 | f.module = m; | |
| 499 | } | |
| 500 | ||
| 501 | // Each build.zig module needs access to each of its | |
| 502 | // dependencies' build.zig modules by name. | |
| 503 | for (fetches) |f| { | |
| 504 | const mod = f.module orelse continue; | |
| 505 | const man = f.manifest orelse continue; | |
| 506 | const dep_names = man.dependencies.keys(); | |
| 507 | try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); | |
| 508 | for (dep_names, man.dependencies.values()) |name, dep| { | |
| 509 | const dep_digest = Package.Fetch.depDigest( | |
| 510 | f.package_root, | |
| 511 | global_cache_directory, | |
| 512 | dep, | |
| 513 | ) orelse continue; | |
| 514 | const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; | |
| 515 | const name_cloned = try arena.dupe(u8, name); | |
| 516 | mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); | |
| 517 | } | |
| 518 | } | |
| 519 | } | |
| 520 | } | |
| 521 | ||
| 522 | try root_mod.deps.put(arena, "@build", Package.build_zig_basename); | |
| 523 | ||
| 524 | const keep_alive = false; | |
| 525 | var prog_node = main_progress_node.start("Compile Build Script", 0); | |
| 526 | defer prog_node.end(); | |
| 527 | ||
| 528 | try child_argv.appendSlice(gpa, &.{ | |
| 529 | zig_exe, "build-exe", build_zig_module, | |
| 530 | "--dep", "@build", configure_runner_module, | |
| 531 | "--listen=-", | |
| 532 | }); | |
| 533 | const maybe_output_dir = try evalZigProcess(step, child_argv.items, prog_node, keep_alive); | |
| 534 | const configure_exe_path = try maybe_output_dir.?.joinString(arena, exe_basename); | |
| 535 | ||
| 536 | prog_node.end(); | |
| 537 | prog_node = main_progress_node.start("Run Build Script", 0); | |
| 538 | ||
| 539 | child_argv.clearRetainingCapacity(); | |
| 540 | child_argv.appendSliceAssumeCapacity(&.{ | |
| 541 | configure_exe_path, | |
| 542 | zig_exe, | |
| 543 | zig_lib_dir, | |
| 544 | cache_root, | |
| 545 | global_cache_root, | |
| 546 | build_root, | |
| 547 | }); | |
| 548 | ||
| 549 | ||
| 550 | child_argv.items[argv_index_exe] = | |
| 551 | try local_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?}); | |
| 552 | } | |
| 553 | ||
| 554 | if (process.can_spawn) { | |
| 555 | var child = std.process.Child.init(child_argv.items, gpa); | |
| 556 | child.stdin_behavior = .Inherit; | |
| 557 | child.stdout_behavior = .Inherit; | |
| 558 | child.stderr_behavior = .Inherit; | |
| 559 | ||
| 560 | const term = t: { | |
| 561 | std.debug.lockStdErr(); | |
| 562 | defer std.debug.unlockStdErr(); | |
| 563 | break :t child.spawnAndWait() catch |err| { | |
| 564 | fatal("failed to spawn build runner {s}: {s}", .{ child_argv.items[0], @errorName(err) }); | |
| 565 | }; | |
| 566 | }; | |
| 567 | ||
| 568 | switch (term) { | |
| 569 | .Exited => |code| { | |
| 570 | if (code == 0) return cleanExit(); | |
| 571 | // Indicates that the build runner has reported compile errors | |
| 572 | // and this parent process does not need to report any further | |
| 573 | // diagnostics. | |
| 574 | if (code == 2) process.exit(2); | |
| 575 | ||
| 576 | if (code == 3) { | |
| 577 | if (!dev.env.supports(.fetch_command)) process.exit(3); | |
| 578 | // Indicates the configure phase failed due to missing lazy | |
| 579 | // dependencies and stdout contains the hashes of the ones | |
| 580 | // that are missing. | |
| 581 | const s = fs.path.sep_str; | |
| 582 | const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce; | |
| 583 | const stdout = local_cache_directory.handle.readFileAlloc(arena, tmp_sub_path, 50 * 1024 * 1024) catch |err| { | |
| 584 | fatal("unable to read results of configure phase from '{}{s}': {s}", .{ | |
| 585 | local_cache_directory, tmp_sub_path, @errorName(err), | |
| 586 | }); | |
| 587 | }; | |
| 588 | local_cache_directory.handle.deleteFile(tmp_sub_path) catch {}; | |
| 589 | ||
| 590 | var it = mem.splitScalar(u8, stdout, '\n'); | |
| 591 | var any_errors = false; | |
| 592 | while (it.next()) |hash| { | |
| 593 | if (hash.len == 0) continue; | |
| 594 | if (hash.len > Package.Hash.max_len) { | |
| 595 | std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ | |
| 596 | hash.len, hash, | |
| 597 | }); | |
| 598 | any_errors = true; | |
| 599 | continue; | |
| 600 | } | |
| 601 | try unlazy_set.put(arena, .fromSlice(hash), {}); | |
| 602 | } | |
| 603 | if (any_errors) process.exit(3); | |
| 604 | if (graph.system_package_mode) |p| { | |
| 605 | // In this mode, the system needs to provide these packages; they | |
| 606 | // cannot be fetched by Zig. | |
| 607 | for (unlazy_set.keys()) |*hash| { | |
| 608 | std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ | |
| 609 | p, hash.toSlice(), | |
| 610 | }); | |
| 611 | } | |
| 612 | std.log.info("remote package fetching disabled due to --system mode", .{}); | |
| 613 | std.log.info("dependencies might be avoidable depending on build configuration", .{}); | |
| 614 | process.exit(3); | |
| 615 | } | |
| 616 | continue; | |
| 617 | } | |
| 618 | ||
| 619 | const cmd = try std.mem.join(arena, " ", child_argv.items); | |
| 620 | fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); | |
| 621 | }, | |
| 622 | else => { | |
| 623 | const cmd = try std.mem.join(arena, " ", child_argv.items); | |
| 624 | fatal("the following build command crashed:\n{s}", .{cmd}); | |
| 625 | }, | |
| 626 | } | |
| 627 | } else { | |
| 628 | const cmd = try std.mem.join(arena, " ", child_argv.items); | |
| 629 | fatal("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(native_os), cmd }); | |
| 630 | } | |
| 631 | } | |
| 632 | } | |
| 633 | ||
| 634 | if (graph.needed_lazy_dependencies.entries.len != 0) { | |
| 635 | var buffer: std.ArrayListUnmanaged(u8) = .empty; | |
| 636 | for (graph.needed_lazy_dependencies.keys()) |k| { | |
| 637 | try buffer.appendSlice(arena, k); | |
| 638 | try buffer.append(arena, '\n'); | |
| 639 | } | |
| 640 | const s = std.fs.path.sep_str; | |
| 641 | const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); | |
| 642 | local_cache_directory.handle.writeFile(.{ | |
| 643 | .sub_path = tmp_sub_path, | |
| 644 | .data = buffer.items, | |
| 645 | .flags = .{ .exclusive = true }, | |
| 646 | }) catch |err| { | |
| 647 | fatal("unable to write configuration results to '{}{s}': {s}", .{ | |
| 648 | local_cache_directory, tmp_sub_path, @errorName(err), | |
| 649 | }); | |
| 650 | }; | |
| 651 | process.exit(3); // Indicate configure phase failed with meaningful stdout. | |
| 652 | } | |
| 653 | ||
| 654 | if (builder.validateUserInputDidItFail()) { | |
| 655 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 656 | } | |
| 657 | ||
| 658 | validateSystemLibraryOptions(builder); | |
| 659 | ||
| 660 | const stdout_writer = io.getStdOut().writer(); | |
| 661 | ||
| 662 | if (help_menu) | |
| 663 | return usage(builder, stdout_writer); | |
| 664 | ||
| 665 | if (steps_menu) | |
| 666 | return steps(builder, stdout_writer); | |
| 667 | ||
| 668 | var run: Run = .{ | |
| 669 | .max_rss = max_rss, | |
| 670 | .max_rss_is_default = false, | |
| 671 | .max_rss_mutex = .{}, | |
| 672 | .skip_oom_steps = skip_oom_steps, | |
| 673 | .watch = watch, | |
| 674 | .fuzz = fuzz, | |
| 675 | .memory_blocked_steps = std.ArrayList(*Step).init(arena), | |
| 676 | .step_stack = .{}, | |
| 677 | .prominent_compile_errors = prominent_compile_errors, | |
| 678 | ||
| 679 | .claimed_rss = 0, | |
| 680 | .summary = summary orelse if (watch) .new else .failures, | |
| 681 | .ttyconf = ttyconf, | |
| 682 | .stderr = stderr, | |
| 683 | .thread_pool = thread_pool, | |
| 684 | }; | |
| 685 | ||
| 686 | if (run.max_rss == 0) { | |
| 687 | run.max_rss = process.totalSystemMemory() catch std.math.maxInt(u64); | |
| 688 | run.max_rss_is_default = true; | |
| 689 | } | |
| 690 | ||
| 691 | const gpa = arena; | |
| 692 | prepare(gpa, arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { | |
| 693 | error.UncleanExit => process.exit(1), | |
| 694 | else => return err, | |
| 695 | }; | |
| 696 | ||
| 697 | var w = if (watch) try Watch.init() else undefined; | |
| 698 | ||
| 699 | rebuild: while (true) { | |
| 700 | runStepNames( | |
| 701 | gpa, | |
| 702 | builder, | |
| 703 | targets.items, | |
| 704 | main_progress_node, | |
| 705 | &run, | |
| 706 | ) catch |err| switch (err) { | |
| 707 | error.UncleanExit => { | |
| 708 | assert(!run.watch); | |
| 709 | process.exit(1); | |
| 710 | }, | |
| 711 | else => return err, | |
| 712 | }; | |
| 713 | if (fuzz) { | |
| 714 | switch (builtin.os.tag) { | |
| 715 | // Current implementation depends on two things that need to be ported to Windows: | |
| 716 | // * Memory-mapping to share data between the fuzzer and build runner. | |
| 717 | // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving | |
| 718 | // many addresses to source locations). | |
| 719 | .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), | |
| 720 | else => {}, | |
| 721 | } | |
| 722 | if (@bitSizeOf(usize) != 64) { | |
| 723 | // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, | |
| 724 | // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case | |
| 725 | // on 32-bit platforms. | |
| 726 | // Affects or affected by issues #5185, #22523, and #22464. | |
| 727 | fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); | |
| 728 | } | |
| 729 | const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable; | |
| 730 | try Fuzz.start( | |
| 731 | gpa, | |
| 732 | arena, | |
| 733 | global_cache_directory, | |
| 734 | zig_lib_directory, | |
| 735 | zig_exe, | |
| 736 | &run.thread_pool, | |
| 737 | run.step_stack.keys(), | |
| 738 | run.ttyconf, | |
| 739 | listen_address, | |
| 740 | main_progress_node, | |
| 741 | ); | |
| 742 | } | |
| 743 | ||
| 744 | if (!watch) return cleanExit(); | |
| 745 | ||
| 746 | if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); | |
| 747 | ||
| 748 | try w.update(gpa, run.step_stack.keys()); | |
| 749 | ||
| 750 | // Wait until a file system notification arrives. Read all such events | |
| 751 | // until the buffer is empty. Then wait for a debounce interval, resetting | |
| 752 | // if any more events come in. After the debounce interval has passed, | |
| 753 | // trigger a rebuild on all steps with modified inputs, as well as their | |
| 754 | // recursive dependants. | |
| 755 | var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; | |
| 756 | const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ | |
| 757 | w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), | |
| 758 | }) catch &caption_buf; | |
| 759 | var debouncing_node = main_progress_node.start(caption, 0); | |
| 760 | var debounce_timeout: Watch.Timeout = .none; | |
| 761 | while (true) switch (try w.wait(gpa, debounce_timeout)) { | |
| 762 | .timeout => { | |
| 763 | debouncing_node.end(); | |
| 764 | markFailedStepsDirty(gpa, run.step_stack.keys()); | |
| 765 | continue :rebuild; | |
| 766 | }, | |
| 767 | .dirty => if (debounce_timeout == .none) { | |
| 768 | debounce_timeout = .{ .ms = debounce_interval_ms }; | |
| 769 | debouncing_node.end(); | |
| 770 | debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); | |
| 771 | }, | |
| 772 | .clean => {}, | |
| 773 | }; | |
| 774 | } | |
| 775 | } | |
| 776 | ||
| 777 | const CliModule = struct { | |
| 778 | deps: std.StringArrayHashMapUnmanaged(*CliModule), | |
| 779 | }; | |
| 780 | ||
| 781 | fn addCliModule(gpa: Allocator, arena: Allocator, aoeu | |
| 782 | const build_mod = try addCliModule(gpa, arena, &cli_modules, build_zig_module); | |
| 783 | ||
| 784 | ||
| 785 | fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { | |
| 786 | for (all_steps) |step| switch (step.state) { | |
| 787 | .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), | |
| 788 | else => continue, | |
| 789 | }; | |
| 790 | // Now that all dirty steps have been found, the remaining steps that | |
| 791 | // succeeded from last run shall be marked "cached". | |
| 792 | for (all_steps) |step| switch (step.state) { | |
| 793 | .success => step.result_cached = true, | |
| 794 | else => continue, | |
| 795 | }; | |
| 796 | } | |
| 797 | ||
| 798 | fn countSubProcesses(all_steps: []const *Step) usize { | |
| 799 | var count: usize = 0; | |
| 800 | for (all_steps) |s| { | |
| 801 | count += @intFromBool(s.getZigProcess() != null); | |
| 802 | } | |
| 803 | return count; | |
| 804 | } | |
| 805 | ||
| 806 | const Run = struct { | |
| 807 | max_rss: u64, | |
| 808 | max_rss_is_default: bool, | |
| 809 | max_rss_mutex: std.Thread.Mutex, | |
| 810 | skip_oom_steps: bool, | |
| 811 | watch: bool, | |
| 812 | fuzz: bool, | |
| 813 | memory_blocked_steps: std.ArrayList(*Step), | |
| 814 | step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 815 | prominent_compile_errors: bool, | |
| 816 | thread_pool: *std.Thread.Pool, | |
| 817 | ||
| 818 | claimed_rss: usize, | |
| 819 | summary: Summary, | |
| 820 | ttyconf: std.io.tty.Config, | |
| 821 | stderr: File, | |
| 822 | ||
| 823 | fn cleanExit(run: Run) void { | |
| 824 | if (run.watch or run.fuzz) return; | |
| 825 | return std.process.cleanExit(); | |
| 826 | } | |
| 827 | }; | |
| 828 | ||
| 829 | fn prepare( | |
| 830 | gpa: Allocator, | |
| 831 | arena: Allocator, | |
| 832 | b: *std.Build, | |
| 833 | step_names: []const []const u8, | |
| 834 | run: *Run, | |
| 835 | seed: u32, | |
| 836 | ) !void { | |
| 837 | const step_stack = &run.step_stack; | |
| 838 | ||
| 839 | if (step_names.len == 0) { | |
| 840 | try step_stack.put(gpa, b.default_step, {}); | |
| 841 | } else { | |
| 842 | try step_stack.ensureUnusedCapacity(gpa, step_names.len); | |
| 843 | for (0..step_names.len) |i| { | |
| 844 | const step_name = step_names[step_names.len - i - 1]; | |
| 845 | const s = b.top_level_steps.get(step_name) orelse { | |
| 846 | std.debug.print("no step named '{s}'\n access the help menu with 'zig build -h'\n", .{step_name}); | |
| 847 | process.exit(1); | |
| 848 | }; | |
| 849 | step_stack.putAssumeCapacity(&s.step, {}); | |
| 850 | } | |
| 851 | } | |
| 852 | ||
| 853 | const starting_steps = try arena.dupe(*Step, step_stack.keys()); | |
| 854 | ||
| 855 | var rng = std.Random.DefaultPrng.init(seed); | |
| 856 | const rand = rng.random(); | |
| 857 | rand.shuffle(*Step, starting_steps); | |
| 858 | ||
| 859 | for (starting_steps) |s| { | |
| 860 | constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) { | |
| 861 | error.DependencyLoopDetected => return uncleanExit(), | |
| 862 | else => |e| return e, | |
| 863 | }; | |
| 864 | } | |
| 865 | ||
| 866 | { | |
| 867 | // Check that we have enough memory to complete the build. | |
| 868 | var any_problems = false; | |
| 869 | for (step_stack.keys()) |s| { | |
| 870 | if (s.max_rss == 0) continue; | |
| 871 | if (s.max_rss > run.max_rss) { | |
| 872 | if (run.skip_oom_steps) { | |
| 873 | s.state = .skipped_oom; | |
| 874 | } else { | |
| 875 | std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{ | |
| 876 | s.owner.dep_prefix, s.name, s.max_rss, run.max_rss, | |
| 877 | }); | |
| 878 | any_problems = true; | |
| 879 | } | |
| 880 | } | |
| 881 | } | |
| 882 | if (any_problems) { | |
| 883 | if (run.max_rss_is_default) { | |
| 884 | std.debug.print("note: use --maxrss to override the default", .{}); | |
| 885 | } | |
| 886 | return uncleanExit(); | |
| 887 | } | |
| 888 | } | |
| 889 | } | |
| 890 | ||
| 891 | fn runStepNames( | |
| 892 | gpa: Allocator, | |
| 893 | b: *std.Build, | |
| 894 | step_names: []const []const u8, | |
| 895 | parent_prog_node: std.Progress.Node, | |
| 896 | run: *Run, | |
| 897 | ) !void { | |
| 898 | const step_stack = &run.step_stack; | |
| 899 | const thread_pool = &run.thread_pool; | |
| 900 | ||
| 901 | { | |
| 902 | const step_prog = parent_prog_node.start("steps", step_stack.count()); | |
| 903 | defer step_prog.end(); | |
| 904 | ||
| 905 | var wait_group: std.Thread.WaitGroup = .{}; | |
| 906 | defer wait_group.wait(); | |
| 907 | ||
| 908 | // Here we spawn the initial set of tasks with a nice heuristic - | |
| 909 | // dependency order. Each worker when it finishes a step will then | |
| 910 | // check whether it should run any dependants. | |
| 911 | const steps_slice = step_stack.keys(); | |
| 912 | for (0..steps_slice.len) |i| { | |
| 913 | const step = steps_slice[steps_slice.len - i - 1]; | |
| 914 | if (step.state == .skipped_oom) continue; | |
| 915 | ||
| 916 | thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{ | |
| 917 | &wait_group, b, step, step_prog, run, | |
| 918 | }); | |
| 919 | } | |
| 920 | } | |
| 921 | assert(run.memory_blocked_steps.items.len == 0); | |
| 922 | ||
| 923 | var test_skip_count: usize = 0; | |
| 924 | var test_fail_count: usize = 0; | |
| 925 | var test_pass_count: usize = 0; | |
| 926 | var test_leak_count: usize = 0; | |
| 927 | var test_count: usize = 0; | |
| 928 | ||
| 929 | var success_count: usize = 0; | |
| 930 | var skipped_count: usize = 0; | |
| 931 | var failure_count: usize = 0; | |
| 932 | var pending_count: usize = 0; | |
| 933 | var total_compile_errors: usize = 0; | |
| 934 | ||
| 935 | for (step_stack.keys()) |s| { | |
| 936 | test_fail_count += s.test_results.fail_count; | |
| 937 | test_skip_count += s.test_results.skip_count; | |
| 938 | test_leak_count += s.test_results.leak_count; | |
| 939 | test_pass_count += s.test_results.passCount(); | |
| 940 | test_count += s.test_results.test_count; | |
| 941 | ||
| 942 | switch (s.state) { | |
| 943 | .precheck_unstarted => unreachable, | |
| 944 | .precheck_started => unreachable, | |
| 945 | .running => unreachable, | |
| 946 | .precheck_done => { | |
| 947 | // precheck_done is equivalent to dependency_failure in the case of | |
| 948 | // transitive dependencies. For example: | |
| 949 | // A -> B -> C (failure) | |
| 950 | // B will be marked as dependency_failure, while A may never be queued, and thus | |
| 951 | // remain in the initial state of precheck_done. | |
| 952 | s.state = .dependency_failure; | |
| 953 | pending_count += 1; | |
| 954 | }, | |
| 955 | .dependency_failure => pending_count += 1, | |
| 956 | .success => success_count += 1, | |
| 957 | .skipped, .skipped_oom => skipped_count += 1, | |
| 958 | .failure => { | |
| 959 | failure_count += 1; | |
| 960 | const compile_errors_len = s.result_error_bundle.errorMessageCount(); | |
| 961 | if (compile_errors_len > 0) { | |
| 962 | total_compile_errors += compile_errors_len; | |
| 963 | } | |
| 964 | }, | |
| 965 | } | |
| 966 | } | |
| 967 | ||
| 968 | // A proper command line application defaults to silently succeeding. | |
| 969 | // The user may request verbose mode if they have a different preference. | |
| 970 | const failures_only = switch (run.summary) { | |
| 971 | .failures, .none => true, | |
| 972 | else => false, | |
| 973 | }; | |
| 974 | if (failure_count == 0 and failures_only) { | |
| 975 | return run.cleanExit(); | |
| 976 | } | |
| 977 | ||
| 978 | const ttyconf = run.ttyconf; | |
| 979 | ||
| 980 | if (run.summary != .none) { | |
| 981 | std.debug.lockStdErr(); | |
| 982 | defer std.debug.unlockStdErr(); | |
| 983 | const stderr = run.stderr; | |
| 984 | ||
| 985 | const total_count = success_count + failure_count + pending_count + skipped_count; | |
| 986 | ttyconf.setColor(stderr, .cyan) catch {}; | |
| 987 | stderr.writeAll("Build Summary:") catch {}; | |
| 988 | ttyconf.setColor(stderr, .reset) catch {}; | |
| 989 | stderr.writer().print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; | |
| 990 | if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {}; | |
| 991 | if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {}; | |
| 992 | ||
| 993 | if (test_count > 0) stderr.writer().print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; | |
| 994 | if (test_skip_count > 0) stderr.writer().print("; {d} skipped", .{test_skip_count}) catch {}; | |
| 995 | if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; | |
| 996 | if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; | |
| 997 | ||
| 998 | stderr.writeAll("\n") catch {}; | |
| 999 | ||
| 1000 | // Print a fancy tree with build results. | |
| 1001 | var step_stack_copy = try step_stack.clone(gpa); | |
| 1002 | defer step_stack_copy.deinit(gpa); | |
| 1003 | ||
| 1004 | var print_node: PrintNode = .{ .parent = null }; | |
| 1005 | if (step_names.len == 0) { | |
| 1006 | print_node.last = true; | |
| 1007 | printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; | |
| 1008 | } else { | |
| 1009 | const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { | |
| 1010 | var i: usize = step_names.len; | |
| 1011 | while (i > 0) { | |
| 1012 | i -= 1; | |
| 1013 | const step = b.top_level_steps.get(step_names[i]).?.step; | |
| 1014 | const found = switch (run.summary) { | |
| 1015 | .all, .none => unreachable, | |
| 1016 | .failures => step.state != .success, | |
| 1017 | .new => !step.result_cached, | |
| 1018 | }; | |
| 1019 | if (found) break :blk i; | |
| 1020 | } | |
| 1021 | break :blk b.top_level_steps.count(); | |
| 1022 | }; | |
| 1023 | for (step_names, 0..) |step_name, i| { | |
| 1024 | const tls = b.top_level_steps.get(step_name).?; | |
| 1025 | print_node.last = i + 1 == last_index; | |
| 1026 | printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; | |
| 1027 | } | |
| 1028 | } | |
| 1029 | } | |
| 1030 | ||
| 1031 | if (failure_count == 0) { | |
| 1032 | return run.cleanExit(); | |
| 1033 | } | |
| 1034 | ||
| 1035 | // Finally, render compile errors at the bottom of the terminal. | |
| 1036 | if (run.prominent_compile_errors and total_compile_errors > 0) { | |
| 1037 | for (step_stack.keys()) |s| { | |
| 1038 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 1039 | s.result_error_bundle.renderToStdErr(.{ .ttyconf = ttyconf, .include_reference_trace = (b.reference_trace orelse 0) > 0 }); | |
| 1040 | } | |
| 1041 | } | |
| 1042 | ||
| 1043 | if (!run.watch) { | |
| 1044 | // Signal to parent process that we have printed compile errors. The | |
| 1045 | // parent process may choose to omit the "following command failed" | |
| 1046 | // line in this case. | |
| 1047 | std.debug.lockStdErr(); | |
| 1048 | process.exit(2); | |
| 1049 | } | |
| 1050 | } | |
| 1051 | ||
| 1052 | if (!run.watch) return uncleanExit(); | |
| 1053 | } | |
| 1054 | ||
| 1055 | const PrintNode = struct { | |
| 1056 | parent: ?*PrintNode, | |
| 1057 | last: bool = false, | |
| 1058 | }; | |
| 1059 | ||
| 1060 | fn printPrefix(node: *PrintNode, stderr: File, ttyconf: std.io.tty.Config) !void { | |
| 1061 | const parent = node.parent orelse return; | |
| 1062 | if (parent.parent == null) return; | |
| 1063 | try printPrefix(parent, stderr, ttyconf); | |
| 1064 | if (parent.last) { | |
| 1065 | try stderr.writeAll(" "); | |
| 1066 | } else { | |
| 1067 | try stderr.writeAll(switch (ttyconf) { | |
| 1068 | .no_color, .windows_api => "| ", | |
| 1069 | .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ | |
| 1070 | }); | |
| 1071 | } | |
| 1072 | } | |
| 1073 | ||
| 1074 | fn printChildNodePrefix(stderr: File, ttyconf: std.io.tty.Config) !void { | |
| 1075 | try stderr.writeAll(switch (ttyconf) { | |
| 1076 | .no_color, .windows_api => "+- ", | |
| 1077 | .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ | |
| 1078 | }); | |
| 1079 | } | |
| 1080 | ||
| 1081 | fn printStepStatus( | |
| 1082 | s: *Step, | |
| 1083 | stderr: File, | |
| 1084 | ttyconf: std.io.tty.Config, | |
| 1085 | run: *const Run, | |
| 1086 | ) !void { | |
| 1087 | switch (s.state) { | |
| 1088 | .precheck_unstarted => unreachable, | |
| 1089 | .precheck_started => unreachable, | |
| 1090 | .precheck_done => unreachable, | |
| 1091 | .running => unreachable, | |
| 1092 | ||
| 1093 | .dependency_failure => { | |
| 1094 | try ttyconf.setColor(stderr, .dim); | |
| 1095 | try stderr.writeAll(" transitive failure\n"); | |
| 1096 | try ttyconf.setColor(stderr, .reset); | |
| 1097 | }, | |
| 1098 | ||
| 1099 | .success => { | |
| 1100 | try ttyconf.setColor(stderr, .green); | |
| 1101 | if (s.result_cached) { | |
| 1102 | try stderr.writeAll(" cached"); | |
| 1103 | } else if (s.test_results.test_count > 0) { | |
| 1104 | const pass_count = s.test_results.passCount(); | |
| 1105 | try stderr.writer().print(" {d} passed", .{pass_count}); | |
| 1106 | if (s.test_results.skip_count > 0) { | |
| 1107 | try ttyconf.setColor(stderr, .yellow); | |
| 1108 | try stderr.writer().print(" {d} skipped", .{s.test_results.skip_count}); | |
| 1109 | } | |
| 1110 | } else { | |
| 1111 | try stderr.writeAll(" success"); | |
| 1112 | } | |
| 1113 | try ttyconf.setColor(stderr, .reset); | |
| 1114 | if (s.result_duration_ns) |ns| { | |
| 1115 | try ttyconf.setColor(stderr, .dim); | |
| 1116 | if (ns >= std.time.ns_per_min) { | |
| 1117 | try stderr.writer().print(" {d}m", .{ns / std.time.ns_per_min}); | |
| 1118 | } else if (ns >= std.time.ns_per_s) { | |
| 1119 | try stderr.writer().print(" {d}s", .{ns / std.time.ns_per_s}); | |
| 1120 | } else if (ns >= std.time.ns_per_ms) { | |
| 1121 | try stderr.writer().print(" {d}ms", .{ns / std.time.ns_per_ms}); | |
| 1122 | } else if (ns >= std.time.ns_per_us) { | |
| 1123 | try stderr.writer().print(" {d}us", .{ns / std.time.ns_per_us}); | |
| 1124 | } else { | |
| 1125 | try stderr.writer().print(" {d}ns", .{ns}); | |
| 1126 | } | |
| 1127 | try ttyconf.setColor(stderr, .reset); | |
| 1128 | } | |
| 1129 | if (s.result_peak_rss != 0) { | |
| 1130 | const rss = s.result_peak_rss; | |
| 1131 | try ttyconf.setColor(stderr, .dim); | |
| 1132 | if (rss >= 1000_000_000) { | |
| 1133 | try stderr.writer().print(" MaxRSS:{d}G", .{rss / 1000_000_000}); | |
| 1134 | } else if (rss >= 1000_000) { | |
| 1135 | try stderr.writer().print(" MaxRSS:{d}M", .{rss / 1000_000}); | |
| 1136 | } else if (rss >= 1000) { | |
| 1137 | try stderr.writer().print(" MaxRSS:{d}K", .{rss / 1000}); | |
| 1138 | } else { | |
| 1139 | try stderr.writer().print(" MaxRSS:{d}B", .{rss}); | |
| 1140 | } | |
| 1141 | try ttyconf.setColor(stderr, .reset); | |
| 1142 | } | |
| 1143 | try stderr.writeAll("\n"); | |
| 1144 | }, | |
| 1145 | .skipped, .skipped_oom => |skip| { | |
| 1146 | try ttyconf.setColor(stderr, .yellow); | |
| 1147 | try stderr.writeAll(" skipped"); | |
| 1148 | if (skip == .skipped_oom) { | |
| 1149 | try stderr.writeAll(" (not enough memory)"); | |
| 1150 | try ttyconf.setColor(stderr, .dim); | |
| 1151 | try stderr.writer().print(" upper bound of {d} exceeded runner limit ({d})", .{ s.max_rss, run.max_rss }); | |
| 1152 | try ttyconf.setColor(stderr, .yellow); | |
| 1153 | } | |
| 1154 | try stderr.writeAll("\n"); | |
| 1155 | try ttyconf.setColor(stderr, .reset); | |
| 1156 | }, | |
| 1157 | .failure => try printStepFailure(s, stderr, ttyconf), | |
| 1158 | } | |
| 1159 | } | |
| 1160 | ||
| 1161 | fn printStepFailure( | |
| 1162 | s: *Step, | |
| 1163 | stderr: File, | |
| 1164 | ttyconf: std.io.tty.Config, | |
| 1165 | ) !void { | |
| 1166 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 1167 | try ttyconf.setColor(stderr, .red); | |
| 1168 | try stderr.writer().print(" {d} errors\n", .{ | |
| 1169 | s.result_error_bundle.errorMessageCount(), | |
| 1170 | }); | |
| 1171 | try ttyconf.setColor(stderr, .reset); | |
| 1172 | } else if (!s.test_results.isSuccess()) { | |
| 1173 | try stderr.writer().print(" {d}/{d} passed", .{ | |
| 1174 | s.test_results.passCount(), s.test_results.test_count, | |
| 1175 | }); | |
| 1176 | if (s.test_results.fail_count > 0) { | |
| 1177 | try stderr.writeAll(", "); | |
| 1178 | try ttyconf.setColor(stderr, .red); | |
| 1179 | try stderr.writer().print("{d} failed", .{ | |
| 1180 | s.test_results.fail_count, | |
| 1181 | }); | |
| 1182 | try ttyconf.setColor(stderr, .reset); | |
| 1183 | } | |
| 1184 | if (s.test_results.skip_count > 0) { | |
| 1185 | try stderr.writeAll(", "); | |
| 1186 | try ttyconf.setColor(stderr, .yellow); | |
| 1187 | try stderr.writer().print("{d} skipped", .{ | |
| 1188 | s.test_results.skip_count, | |
| 1189 | }); | |
| 1190 | try ttyconf.setColor(stderr, .reset); | |
| 1191 | } | |
| 1192 | if (s.test_results.leak_count > 0) { | |
| 1193 | try stderr.writeAll(", "); | |
| 1194 | try ttyconf.setColor(stderr, .red); | |
| 1195 | try stderr.writer().print("{d} leaked", .{ | |
| 1196 | s.test_results.leak_count, | |
| 1197 | }); | |
| 1198 | try ttyconf.setColor(stderr, .reset); | |
| 1199 | } | |
| 1200 | try stderr.writeAll("\n"); | |
| 1201 | } else if (s.result_error_msgs.items.len > 0) { | |
| 1202 | try ttyconf.setColor(stderr, .red); | |
| 1203 | try stderr.writeAll(" failure\n"); | |
| 1204 | try ttyconf.setColor(stderr, .reset); | |
| 1205 | } else { | |
| 1206 | assert(s.result_stderr.len > 0); | |
| 1207 | try ttyconf.setColor(stderr, .red); | |
| 1208 | try stderr.writeAll(" stderr\n"); | |
| 1209 | try ttyconf.setColor(stderr, .reset); | |
| 1210 | } | |
| 1211 | } | |
| 1212 | ||
| 1213 | fn printTreeStep( | |
| 1214 | b: *std.Build, | |
| 1215 | s: *Step, | |
| 1216 | run: *const Run, | |
| 1217 | stderr: File, | |
| 1218 | ttyconf: std.io.tty.Config, | |
| 1219 | parent_node: *PrintNode, | |
| 1220 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1221 | ) !void { | |
| 1222 | const first = step_stack.swapRemove(s); | |
| 1223 | const summary = run.summary; | |
| 1224 | const skip = switch (summary) { | |
| 1225 | .none => unreachable, | |
| 1226 | .all => false, | |
| 1227 | .new => s.result_cached, | |
| 1228 | .failures => s.state == .success, | |
| 1229 | }; | |
| 1230 | if (skip) return; | |
| 1231 | try printPrefix(parent_node, stderr, ttyconf); | |
| 1232 | ||
| 1233 | if (!first) try ttyconf.setColor(stderr, .dim); | |
| 1234 | if (parent_node.parent != null) { | |
| 1235 | if (parent_node.last) { | |
| 1236 | try printChildNodePrefix(stderr, ttyconf); | |
| 1237 | } else { | |
| 1238 | try stderr.writeAll(switch (ttyconf) { | |
| 1239 | .no_color, .windows_api => "+- ", | |
| 1240 | .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ | |
| 1241 | }); | |
| 1242 | } | |
| 1243 | } | |
| 1244 | ||
| 1245 | // dep_prefix omitted here because it is redundant with the tree. | |
| 1246 | try stderr.writeAll(s.name); | |
| 1247 | ||
| 1248 | if (first) { | |
| 1249 | try printStepStatus(s, stderr, ttyconf, run); | |
| 1250 | ||
| 1251 | const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { | |
| 1252 | var i: usize = s.dependencies.items.len; | |
| 1253 | while (i > 0) { | |
| 1254 | i -= 1; | |
| 1255 | ||
| 1256 | const step = s.dependencies.items[i]; | |
| 1257 | const found = switch (summary) { | |
| 1258 | .all, .none => unreachable, | |
| 1259 | .failures => step.state != .success, | |
| 1260 | .new => !step.result_cached, | |
| 1261 | }; | |
| 1262 | if (found) break :blk i; | |
| 1263 | } | |
| 1264 | break :blk s.dependencies.items.len -| 1; | |
| 1265 | }; | |
| 1266 | for (s.dependencies.items, 0..) |dep, i| { | |
| 1267 | var print_node: PrintNode = .{ | |
| 1268 | .parent = parent_node, | |
| 1269 | .last = i == last_index, | |
| 1270 | }; | |
| 1271 | try printTreeStep(b, dep, run, stderr, ttyconf, &print_node, step_stack); | |
| 1272 | } | |
| 1273 | } else { | |
| 1274 | if (s.dependencies.items.len == 0) { | |
| 1275 | try stderr.writeAll(" (reused)\n"); | |
| 1276 | } else { | |
| 1277 | try stderr.writer().print(" (+{d} more reused dependencies)\n", .{ | |
| 1278 | s.dependencies.items.len, | |
| 1279 | }); | |
| 1280 | } | |
| 1281 | try ttyconf.setColor(stderr, .reset); | |
| 1282 | } | |
| 1283 | } | |
| 1284 | ||
| 1285 | /// Traverse the dependency graph depth-first and make it undirected by having | |
| 1286 | /// steps know their dependants (they only know dependencies at start). | |
| 1287 | /// Along the way, check that there is no dependency loop, and record the steps | |
| 1288 | /// in traversal order in `step_stack`. | |
| 1289 | /// Each step has its dependencies traversed in random order, this accomplishes | |
| 1290 | /// two things: | |
| 1291 | /// - `step_stack` will be in randomized-depth-first order, so the build runner | |
| 1292 | /// spawns steps in a random (but optimized) order | |
| 1293 | /// - each step's `dependants` list is also filled in a random order, so that | |
| 1294 | /// when it finishes executing in `workerMakeOneStep`, it spawns next steps | |
| 1295 | /// to run in random order | |
| 1296 | fn constructGraphAndCheckForDependencyLoop( | |
| 1297 | b: *std.Build, | |
| 1298 | s: *Step, | |
| 1299 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1300 | rand: std.Random, | |
| 1301 | ) !void { | |
| 1302 | switch (s.state) { | |
| 1303 | .precheck_started => { | |
| 1304 | std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); | |
| 1305 | return error.DependencyLoopDetected; | |
| 1306 | }, | |
| 1307 | .precheck_unstarted => { | |
| 1308 | s.state = .precheck_started; | |
| 1309 | ||
| 1310 | try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len); | |
| 1311 | ||
| 1312 | // We dupe to avoid shuffling the steps in the summary, it depends | |
| 1313 | // on s.dependencies' order. | |
| 1314 | const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM"); | |
| 1315 | rand.shuffle(*Step, deps); | |
| 1316 | ||
| 1317 | for (deps) |dep| { | |
| 1318 | try step_stack.put(b.allocator, dep, {}); | |
| 1319 | try dep.dependants.append(b.allocator, s); | |
| 1320 | constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| { | |
| 1321 | if (err == error.DependencyLoopDetected) { | |
| 1322 | std.debug.print(" {s}\n", .{s.name}); | |
| 1323 | } | |
| 1324 | return err; | |
| 1325 | }; | |
| 1326 | } | |
| 1327 | ||
| 1328 | s.state = .precheck_done; | |
| 1329 | }, | |
| 1330 | .precheck_done => {}, | |
| 1331 | ||
| 1332 | // These don't happen until we actually run the step graph. | |
| 1333 | .dependency_failure => unreachable, | |
| 1334 | .running => unreachable, | |
| 1335 | .success => unreachable, | |
| 1336 | .failure => unreachable, | |
| 1337 | .skipped => unreachable, | |
| 1338 | .skipped_oom => unreachable, | |
| 1339 | } | |
| 1340 | } | |
| 1341 | ||
| 1342 | fn workerMakeOneStep( | |
| 1343 | wg: *std.Thread.WaitGroup, | |
| 1344 | b: *std.Build, | |
| 1345 | s: *Step, | |
| 1346 | prog_node: std.Progress.Node, | |
| 1347 | run: *Run, | |
| 1348 | ) void { | |
| 1349 | const thread_pool = &run.thread_pool; | |
| 1350 | ||
| 1351 | // First, check the conditions for running this step. If they are not met, | |
| 1352 | // then we return without doing the step, relying on another worker to | |
| 1353 | // queue this step up again when dependencies are met. | |
| 1354 | for (s.dependencies.items) |dep| { | |
| 1355 | switch (@atomicLoad(Step.State, &dep.state, .seq_cst)) { | |
| 1356 | .success, .skipped => continue, | |
| 1357 | .failure, .dependency_failure, .skipped_oom => { | |
| 1358 | @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst); | |
| 1359 | return; | |
| 1360 | }, | |
| 1361 | .precheck_done, .running => { | |
| 1362 | // dependency is not finished yet. | |
| 1363 | return; | |
| 1364 | }, | |
| 1365 | .precheck_unstarted => unreachable, | |
| 1366 | .precheck_started => unreachable, | |
| 1367 | } | |
| 1368 | } | |
| 1369 | ||
| 1370 | if (s.max_rss != 0) { | |
| 1371 | run.max_rss_mutex.lock(); | |
| 1372 | defer run.max_rss_mutex.unlock(); | |
| 1373 | ||
| 1374 | // Avoid running steps twice. | |
| 1375 | if (s.state != .precheck_done) { | |
| 1376 | // Another worker got the job. | |
| 1377 | return; | |
| 1378 | } | |
| 1379 | ||
| 1380 | const new_claimed_rss = run.claimed_rss + s.max_rss; | |
| 1381 | if (new_claimed_rss > run.max_rss) { | |
| 1382 | // Running this step right now could possibly exceed the allotted RSS. | |
| 1383 | // Add this step to the queue of memory-blocked steps. | |
| 1384 | run.memory_blocked_steps.append(s) catch @panic("OOM"); | |
| 1385 | return; | |
| 1386 | } | |
| 1387 | ||
| 1388 | run.claimed_rss = new_claimed_rss; | |
| 1389 | s.state = .running; | |
| 1390 | } else { | |
| 1391 | // Avoid running steps twice. | |
| 1392 | if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .seq_cst, .seq_cst) != null) { | |
| 1393 | // Another worker got the job. | |
| 1394 | return; | |
| 1395 | } | |
| 1396 | } | |
| 1397 | ||
| 1398 | const sub_prog_node = prog_node.start(s.name, 0); | |
| 1399 | defer sub_prog_node.end(); | |
| 1400 | ||
| 1401 | const make_result = s.make(.{ | |
| 1402 | .progress_node = sub_prog_node, | |
| 1403 | .thread_pool = thread_pool, | |
| 1404 | .watch = run.watch, | |
| 1405 | }); | |
| 1406 | ||
| 1407 | // No matter the result, we want to display error/warning messages. | |
| 1408 | const show_compile_errors = !run.prominent_compile_errors and | |
| 1409 | s.result_error_bundle.errorMessageCount() > 0; | |
| 1410 | const show_error_msgs = s.result_error_msgs.items.len > 0; | |
| 1411 | const show_stderr = s.result_stderr.len > 0; | |
| 1412 | ||
| 1413 | if (show_error_msgs or show_compile_errors or show_stderr) { | |
| 1414 | std.debug.lockStdErr(); | |
| 1415 | defer std.debug.unlockStdErr(); | |
| 1416 | ||
| 1417 | const gpa = b.allocator; | |
| 1418 | const options: std.zig.ErrorBundle.RenderOptions = .{ | |
| 1419 | .ttyconf = run.ttyconf, | |
| 1420 | .include_reference_trace = (b.reference_trace orelse 0) > 0, | |
| 1421 | }; | |
| 1422 | printErrorMessages(gpa, s, options, run.stderr, run.prominent_compile_errors) catch {}; | |
| 1423 | } | |
| 1424 | ||
| 1425 | handle_result: { | |
| 1426 | if (make_result) |_| { | |
| 1427 | @atomicStore(Step.State, &s.state, .success, .seq_cst); | |
| 1428 | } else |err| switch (err) { | |
| 1429 | error.MakeFailed => { | |
| 1430 | @atomicStore(Step.State, &s.state, .failure, .seq_cst); | |
| 1431 | break :handle_result; | |
| 1432 | }, | |
| 1433 | error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .seq_cst), | |
| 1434 | } | |
| 1435 | ||
| 1436 | // Successful completion of a step, so we queue up its dependants as well. | |
| 1437 | for (s.dependants.items) |dep| { | |
| 1438 | thread_pool.spawnWg(wg, workerMakeOneStep, .{ | |
| 1439 | wg, b, dep, prog_node, run, | |
| 1440 | }); | |
| 1441 | } | |
| 1442 | } | |
| 1443 | ||
| 1444 | // If this is a step that claims resources, we must now queue up other | |
| 1445 | // steps that are waiting for resources. | |
| 1446 | if (s.max_rss != 0) { | |
| 1447 | run.max_rss_mutex.lock(); | |
| 1448 | defer run.max_rss_mutex.unlock(); | |
| 1449 | ||
| 1450 | // Give the memory back to the scheduler. | |
| 1451 | run.claimed_rss -= s.max_rss; | |
| 1452 | // Avoid kicking off too many tasks that we already know will not have | |
| 1453 | // enough resources. | |
| 1454 | var remaining = run.max_rss - run.claimed_rss; | |
| 1455 | var i: usize = 0; | |
| 1456 | var j: usize = 0; | |
| 1457 | while (j < run.memory_blocked_steps.items.len) : (j += 1) { | |
| 1458 | const dep = run.memory_blocked_steps.items[j]; | |
| 1459 | assert(dep.max_rss != 0); | |
| 1460 | if (dep.max_rss <= remaining) { | |
| 1461 | remaining -= dep.max_rss; | |
| 1462 | ||
| 1463 | thread_pool.spawnWg(wg, workerMakeOneStep, .{ | |
| 1464 | wg, b, dep, prog_node, run, | |
| 1465 | }); | |
| 1466 | } else { | |
| 1467 | run.memory_blocked_steps.items[i] = dep; | |
| 1468 | i += 1; | |
| 1469 | } | |
| 1470 | } | |
| 1471 | run.memory_blocked_steps.shrinkRetainingCapacity(i); | |
| 1472 | } | |
| 1473 | } | |
| 1474 | ||
| 1475 | pub fn printErrorMessages( | |
| 1476 | gpa: Allocator, | |
| 1477 | failing_step: *Step, | |
| 1478 | options: std.zig.ErrorBundle.RenderOptions, | |
| 1479 | stderr: File, | |
| 1480 | prominent_compile_errors: bool, | |
| 1481 | ) !void { | |
| 1482 | // Provide context for where these error messages are coming from by | |
| 1483 | // printing the corresponding Step subtree. | |
| 1484 | ||
| 1485 | var step_stack: std.ArrayListUnmanaged(*Step) = .empty; | |
| 1486 | defer step_stack.deinit(gpa); | |
| 1487 | try step_stack.append(gpa, failing_step); | |
| 1488 | while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { | |
| 1489 | try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); | |
| 1490 | } | |
| 1491 | ||
| 1492 | // Now, `step_stack` has the subtree that we want to print, in reverse order. | |
| 1493 | const ttyconf = options.ttyconf; | |
| 1494 | try ttyconf.setColor(stderr, .dim); | |
| 1495 | var indent: usize = 0; | |
| 1496 | while (step_stack.pop()) |s| : (indent += 1) { | |
| 1497 | if (indent > 0) { | |
| 1498 | try stderr.writer().writeByteNTimes(' ', (indent - 1) * 3); | |
| 1499 | try printChildNodePrefix(stderr, ttyconf); | |
| 1500 | } | |
| 1501 | ||
| 1502 | try stderr.writeAll(s.name); | |
| 1503 | ||
| 1504 | if (s == failing_step) { | |
| 1505 | try printStepFailure(s, stderr, ttyconf); | |
| 1506 | } else { | |
| 1507 | try stderr.writeAll("\n"); | |
| 1508 | } | |
| 1509 | } | |
| 1510 | try ttyconf.setColor(stderr, .reset); | |
| 1511 | ||
| 1512 | if (failing_step.result_stderr.len > 0) { | |
| 1513 | try stderr.writeAll(failing_step.result_stderr); | |
| 1514 | if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { | |
| 1515 | try stderr.writeAll("\n"); | |
| 1516 | } | |
| 1517 | } | |
| 1518 | ||
| 1519 | if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) { | |
| 1520 | try failing_step.result_error_bundle.renderToWriter(options, stderr.writer()); | |
| 1521 | } | |
| 1522 | ||
| 1523 | for (failing_step.result_error_msgs.items) |msg| { | |
| 1524 | try ttyconf.setColor(stderr, .red); | |
| 1525 | try stderr.writeAll("error: "); | |
| 1526 | try ttyconf.setColor(stderr, .reset); | |
| 1527 | try stderr.writeAll(msg); | |
| 1528 | try stderr.writeAll("\n"); | |
| 1529 | } | |
| 1530 | } | |
| 1531 | ||
| 1532 | fn steps(builder: *std.Build, out_stream: anytype) !void { | |
| 1533 | const allocator = builder.allocator; | |
| 1534 | for (builder.top_level_steps.values()) |top_level_step| { | |
| 1535 | const name = if (&top_level_step.step == builder.default_step) | |
| 1536 | try fmt.allocPrint(allocator, "{s} (default)", .{top_level_step.step.name}) | |
| 1537 | else | |
| 1538 | top_level_step.step.name; | |
| 1539 | try out_stream.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); | |
| 1540 | } | |
| 1541 | } | |
| 1542 | ||
| 1543 | fn usage(b: *std.Build, out_stream: anytype) !void { | |
| 1544 | try out_stream.print( | |
| 1545 | \\Usage: {s} build [steps] [options] | |
| 1546 | \\ | |
| 1547 | \\Steps: | |
| 1548 | \\ | |
| 1549 | , .{b.graph.zig_exe}); | |
| 1550 | try steps(b, out_stream); | |
| 1551 | ||
| 1552 | try out_stream.writeAll( | |
| 1553 | \\ | |
| 1554 | \\General Options: | |
| 1555 | \\ -p, --prefix [path] Where to install files (default: zig-out) | |
| 1556 | \\ --prefix-lib-dir [path] Where to install libraries | |
| 1557 | \\ --prefix-exe-dir [path] Where to install executables | |
| 1558 | \\ --prefix-include-dir [path] Where to install C header files | |
| 1559 | \\ | |
| 1560 | \\ --release[=mode] Request release mode, optionally specifying a | |
| 1561 | \\ preferred optimization mode: fast, safe, small | |
| 1562 | \\ | |
| 1563 | \\ -fdarling, -fno-darling Integration with system-installed Darling to | |
| 1564 | \\ execute macOS programs on Linux hosts | |
| 1565 | \\ (default: no) | |
| 1566 | \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute | |
| 1567 | \\ foreign-architecture programs on Linux hosts | |
| 1568 | \\ (default: no) | |
| 1569 | \\ --glibc-runtimes [path] Enhances QEMU integration by providing glibc built | |
| 1570 | \\ for multiple foreign architectures, allowing | |
| 1571 | \\ execution of non-native programs that link with glibc. | |
| 1572 | \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on | |
| 1573 | \\ ARM64 macOS hosts. (default: no) | |
| 1574 | \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to | |
| 1575 | \\ execute WASI binaries. (default: no) | |
| 1576 | \\ -fwine, -fno-wine Integration with system-installed Wine to execute | |
| 1577 | \\ Windows programs on Linux hosts. (default: no) | |
| 1578 | \\ | |
| 1579 | \\ -h, --help Print this help and exit | |
| 1580 | \\ -l, --list-steps Print available steps | |
| 1581 | \\ --verbose Print commands before executing them | |
| 1582 | \\ --color [auto|off|on] Enable or disable colored error messages | |
| 1583 | \\ --prominent-compile-errors Buffer compile errors and display at end | |
| 1584 | \\ --summary [mode] Control the printing of the build summary | |
| 1585 | \\ all Print the build summary in its entirety | |
| 1586 | \\ new Omit cached steps | |
| 1587 | \\ failures (Default) Only print failed steps | |
| 1588 | \\ none Do not print the build summary | |
| 1589 | \\ -j<N> Limit concurrent jobs (default is to use all CPU cores) | |
| 1590 | \\ --maxrss <bytes> Limit memory usage (default is to use available memory) | |
| 1591 | \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss | |
| 1592 | \\ --fetch Exit after fetching dependency tree | |
| 1593 | \\ --watch Continuously rebuild when source files are modified | |
| 1594 | \\ --fuzz Continuously search for unit test failures | |
| 1595 | \\ --debounce <ms> Delay before rebuilding after changed file detected | |
| 1596 | \\ -fincremental Enable incremental compilation | |
| 1597 | \\ -fno-incremental Disable incremental compilation | |
| 1598 | \\ | |
| 1599 | \\Project-Specific Options: | |
| 1600 | \\ | |
| 1601 | ); | |
| 1602 | ||
| 1603 | const arena = b.allocator; | |
| 1604 | if (b.available_options_list.items.len == 0) { | |
| 1605 | try out_stream.print(" (none)\n", .{}); | |
| 1606 | } else { | |
| 1607 | for (b.available_options_list.items) |option| { | |
| 1608 | const name = try fmt.allocPrint(arena, " -D{s}=[{s}]", .{ | |
| 1609 | option.name, | |
| 1610 | @tagName(option.type_id), | |
| 1611 | }); | |
| 1612 | try out_stream.print("{s:<30} {s}\n", .{ name, option.description }); | |
| 1613 | if (option.enum_options) |enum_options| { | |
| 1614 | const padding = " " ** 33; | |
| 1615 | try out_stream.writeAll(padding ++ "Supported Values:\n"); | |
| 1616 | for (enum_options) |enum_option| { | |
| 1617 | try out_stream.print(padding ++ " {s}\n", .{enum_option}); | |
| 1618 | } | |
| 1619 | } | |
| 1620 | } | |
| 1621 | } | |
| 1622 | ||
| 1623 | try out_stream.writeAll( | |
| 1624 | \\ | |
| 1625 | \\System Integration Options: | |
| 1626 | \\ --search-prefix [path] Add a path to look for binaries, libraries, headers | |
| 1627 | \\ --sysroot [path] Set the system root directory (usually /) | |
| 1628 | \\ --libc [file] Provide a file which specifies libc paths | |
| 1629 | \\ | |
| 1630 | \\ --system [pkgdir] Disable package fetching; enable all integrations | |
| 1631 | \\ -fsys=[name] Enable a system integration | |
| 1632 | \\ -fno-sys=[name] Disable a system integration | |
| 1633 | \\ | |
| 1634 | \\ Available System Integrations: Enabled: | |
| 1635 | \\ | |
| 1636 | ); | |
| 1637 | if (b.graph.system_library_options.entries.len == 0) { | |
| 1638 | try out_stream.writeAll(" (none) -\n"); | |
| 1639 | } else { | |
| 1640 | for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { | |
| 1641 | const status = switch (v) { | |
| 1642 | .declared_enabled => "yes", | |
| 1643 | .declared_disabled => "no", | |
| 1644 | .user_enabled, .user_disabled => unreachable, // already emitted error | |
| 1645 | }; | |
| 1646 | try out_stream.print(" {s:<43} {s}\n", .{ k, status }); | |
| 1647 | } | |
| 1648 | } | |
| 1649 | ||
| 1650 | try out_stream.writeAll( | |
| 1651 | \\ | |
| 1652 | \\Advanced Options: | |
| 1653 | \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error | |
| 1654 | \\ -fno-reference-trace Disable reference trace | |
| 1655 | \\ -fallow-so-scripts Allows .so files to be GNU ld scripts | |
| 1656 | \\ -fno-allow-so-scripts (default) .so files must be ELF files | |
| 1657 | \\ --build-file [file] Override path to build.zig | |
| 1658 | \\ --cache-dir [path] Override path to local Zig cache directory | |
| 1659 | \\ --global-cache-dir [path] Override path to global Zig cache directory | |
| 1660 | \\ --zig-lib-dir [arg] Override path to Zig lib directory | |
| 1661 | \\ --build-runner [file] Override path to build runner | |
| 1662 | \\ --seed [integer] For shuffling dependency traversal order (default: random) | |
| 1663 | \\ --debug-log [scope] Enable debugging the compiler | |
| 1664 | \\ --debug-pkg-config Fail if unknown pkg-config flags encountered | |
| 1665 | \\ --debug-rt Debug compiler runtime libraries | |
| 1666 | \\ --verbose-link Enable compiler debug output for linking | |
| 1667 | \\ --verbose-air Enable compiler debug output for Zig AIR | |
| 1668 | \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR | |
| 1669 | \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC | |
| 1670 | \\ --verbose-cimport Enable compiler debug output for C imports | |
| 1671 | \\ --verbose-cc Enable compiler debug output for C compilation | |
| 1672 | \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features | |
| 1673 | \\ | |
| 1674 | ); | |
| 1675 | } | |
| 1676 | ||
| 1677 | fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { | |
| 1678 | if (idx.* >= args.len) return null; | |
| 1679 | defer idx.* += 1; | |
| 1680 | return args[idx.*]; | |
| 1681 | } | |
| 1682 | ||
| 1683 | fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { | |
| 1684 | return nextArg(args, idx) orelse { | |
| 1685 | std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); | |
| 1686 | process.exit(1); | |
| 1687 | }; | |
| 1688 | } | |
| 1689 | ||
| 1690 | fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { | |
| 1691 | if (idx >= args.len) return null; | |
| 1692 | return args[idx..]; | |
| 1693 | } | |
| 1694 | ||
| 1695 | /// Perhaps in the future there could be an Advanced Options flag such as | |
| 1696 | /// --debug-build-runner-leaks which would make this function return instead of | |
| 1697 | /// calling exit. | |
| 1698 | fn uncleanExit() error{UncleanExit} { | |
| 1699 | std.debug.lockStdErr(); | |
| 1700 | process.exit(1); | |
| 1701 | } | |
| 1702 | ||
| 1703 | const Color = std.zig.Color; | |
| 1704 | const Summary = enum { all, new, failures, none }; | |
| 1705 | ||
| 1706 | fn get_tty_conf(color: Color, stderr: File) std.io.tty.Config { | |
| 1707 | return switch (color) { | |
| 1708 | .auto => std.io.tty.detectConfig(stderr), | |
| 1709 | .on => .escape_codes, | |
| 1710 | .off => .no_color, | |
| 1711 | }; | |
| 1712 | } | |
| 1713 | ||
| 1714 | fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { | |
| 1715 | std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); | |
| 1716 | process.exit(1); | |
| 1717 | } | |
| 1718 | ||
| 1719 | fn validateSystemLibraryOptions(b: *std.Build) void { | |
| 1720 | var bad = false; | |
| 1721 | for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { | |
| 1722 | switch (v) { | |
| 1723 | .user_disabled, .user_enabled => { | |
| 1724 | // The user tried to enable or disable a system library integration, but | |
| 1725 | // the build script did not recognize that option. | |
| 1726 | std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); | |
| 1727 | bad = true; | |
| 1728 | }, | |
| 1729 | .declared_disabled, .declared_enabled => {}, | |
| 1730 | } | |
| 1731 | } | |
| 1732 | if (bad) { | |
| 1733 | std.debug.print(" access the help menu with 'zig build -h'\n", .{}); | |
| 1734 | process.exit(1); | |
| 1735 | } | |
| 1736 | } | |
| 1737 | ||
| 1738 | /// Creates the dependencies.zig file and corresponding `Module` for the | |
| 1739 | /// build runner to obtain via `@import("@dependencies")`. | |
| 1740 | fn createDependenciesModule( | |
| 1741 | arena: Allocator, | |
| 1742 | source: []const u8, | |
| 1743 | main_mod: *CliModule, | |
| 1744 | global_cache_directory: Directory, | |
| 1745 | local_cache_directory: Directory, | |
| 1746 | builtin_mod: *Module, | |
| 1747 | global_options: Compilation.Config, | |
| 1748 | ) !*CliModule { | |
| 1749 | // Atomically create the file in a directory named after the hash of its contents. | |
| 1750 | const basename = "dependencies.zig"; | |
| 1751 | const rand_int = std.crypto.random.int(u64); | |
| 1752 | const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); | |
| 1753 | { | |
| 1754 | var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); | |
| 1755 | defer tmp_dir.close(); | |
| 1756 | try tmp_dir.writeFile(.{ .sub_path = basename, .data = source }); | |
| 1757 | } | |
| 1758 | ||
| 1759 | var hh: Cache.HashHelper = .{}; | |
| 1760 | hh.addBytes(build_options.version); | |
| 1761 | hh.addBytes(source); | |
| 1762 | const hex_digest = hh.final(); | |
| 1763 | ||
| 1764 | const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); | |
| 1765 | try Package.Fetch.renameTmpIntoCache( | |
| 1766 | local_cache_directory.handle, | |
| 1767 | tmp_dir_sub_path, | |
| 1768 | o_dir_sub_path, | |
| 1769 | ); | |
| 1770 | ||
| 1771 | const deps_mod = try Module.create(arena, .{ | |
| 1772 | .global_cache_directory = global_cache_directory, | |
| 1773 | .paths = .{ | |
| 1774 | .root = .{ | |
| 1775 | .root_dir = local_cache_directory, | |
| 1776 | .sub_path = o_dir_sub_path, | |
| 1777 | }, | |
| 1778 | .root_src_path = basename, | |
| 1779 | }, | |
| 1780 | .fully_qualified_name = "root.@dependencies", | |
| 1781 | .parent = main_mod, | |
| 1782 | .cc_argv = &.{}, | |
| 1783 | .inherited = .{}, | |
| 1784 | .global = global_options, | |
| 1785 | .builtin_mod = builtin_mod, | |
| 1786 | .builtin_modules = null, // `builtin_mod` is specified | |
| 1787 | }); | |
| 1788 | try main_mod.deps.put(arena, "@dependencies", deps_mod); | |
| 1789 | return deps_mod; | |
| 1790 | } |
lib/compiler/build_runner.zig deleted-1525
| ... | ... | @@ -1,1525 +0,0 @@ |
| 1 | const std = @import("std"); | |
| 2 | const builtin = @import("builtin"); | |
| 3 | const assert = std.debug.assert; | |
| 4 | const io = std.io; | |
| 5 | const fmt = std.fmt; | |
| 6 | const mem = std.mem; | |
| 7 | const process = std.process; | |
| 8 | const ArrayList = std.ArrayList; | |
| 9 | const File = std.fs.File; | |
| 10 | const Step = std.Build.Step; | |
| 11 | const Watch = std.Build.Watch; | |
| 12 | const Fuzz = std.Build.Fuzz; | |
| 13 | const Allocator = std.mem.Allocator; | |
| 14 | const fatal = std.process.fatal; | |
| 15 | const runner = @This(); | |
| 16 | ||
| 17 | pub const root = @import("@build"); | |
| 18 | pub const dependencies = @import("@dependencies"); | |
| 19 | ||
| 20 | pub const std_options: std.Options = .{ | |
| 21 | .side_channels_mitigations = .none, | |
| 22 | .http_disable_tls = true, | |
| 23 | .crypto_fork_safety = false, | |
| 24 | }; | |
| 25 | ||
| 26 | pub fn main() !void { | |
| 27 | // Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived, | |
| 28 | // one shot program. We don't need to waste time freeing memory and finding places to squish | |
| 29 | // bytes into. So we free everything all at once at the very end. | |
| 30 | var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | |
| 31 | defer single_threaded_arena.deinit(); | |
| 32 | ||
| 33 | var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ | |
| 34 | .child_allocator = single_threaded_arena.allocator(), | |
| 35 | }; | |
| 36 | const arena = thread_safe_arena.allocator(); | |
| 37 | ||
| 38 | const args = try process.argsAlloc(arena); | |
| 39 | ||
| 40 | // skip my own exe name | |
| 41 | var arg_idx: usize = 1; | |
| 42 | ||
| 43 | const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); | |
| 44 | const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); | |
| 45 | const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); | |
| 46 | const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); | |
| 47 | const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); | |
| 48 | ||
| 49 | const zig_lib_directory: std.Build.Cache.Directory = .{ | |
| 50 | .path = zig_lib_dir, | |
| 51 | .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), | |
| 52 | }; | |
| 53 | ||
| 54 | const build_root_directory: std.Build.Cache.Directory = .{ | |
| 55 | .path = build_root, | |
| 56 | .handle = try std.fs.cwd().openDir(build_root, .{}), | |
| 57 | }; | |
| 58 | ||
| 59 | const local_cache_directory: std.Build.Cache.Directory = .{ | |
| 60 | .path = cache_root, | |
| 61 | .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), | |
| 62 | }; | |
| 63 | ||
| 64 | const global_cache_directory: std.Build.Cache.Directory = .{ | |
| 65 | .path = global_cache_root, | |
| 66 | .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), | |
| 67 | }; | |
| 68 | ||
| 69 | var graph: std.Build.Graph = .{ | |
| 70 | .arena = arena, | |
| 71 | .cache = .{ | |
| 72 | .gpa = arena, | |
| 73 | .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), | |
| 74 | }, | |
| 75 | .zig_exe = zig_exe, | |
| 76 | .env_map = try process.getEnvMap(arena), | |
| 77 | .global_cache_root = global_cache_directory, | |
| 78 | .zig_lib_directory = zig_lib_directory, | |
| 79 | .host = .{ | |
| 80 | .query = .{}, | |
| 81 | .result = try std.zig.system.resolveTargetQuery(.{}), | |
| 82 | }, | |
| 83 | }; | |
| 84 | ||
| 85 | graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); | |
| 86 | graph.cache.addPrefix(build_root_directory); | |
| 87 | graph.cache.addPrefix(local_cache_directory); | |
| 88 | graph.cache.addPrefix(global_cache_directory); | |
| 89 | graph.cache.hash.addBytes(builtin.zig_version_string); | |
| 90 | ||
| 91 | const builder = try std.Build.create( | |
| 92 | &graph, | |
| 93 | build_root_directory, | |
| 94 | local_cache_directory, | |
| 95 | dependencies.root_deps, | |
| 96 | ); | |
| 97 | ||
| 98 | var targets = ArrayList([]const u8).init(arena); | |
| 99 | var debug_log_scopes = ArrayList([]const u8).init(arena); | |
| 100 | var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena }; | |
| 101 | ||
| 102 | var install_prefix: ?[]const u8 = null; | |
| 103 | var dir_list = std.Build.DirList{}; | |
| 104 | var summary: ?Summary = null; | |
| 105 | var max_rss: u64 = 0; | |
| 106 | var skip_oom_steps = false; | |
| 107 | var color: Color = .auto; | |
| 108 | var prominent_compile_errors = false; | |
| 109 | var help_menu = false; | |
| 110 | var steps_menu = false; | |
| 111 | var output_tmp_nonce: ?[16]u8 = null; | |
| 112 | var watch = false; | |
| 113 | var fuzz = false; | |
| 114 | var debounce_interval_ms: u16 = 50; | |
| 115 | var listen_port: u16 = 0; | |
| 116 | ||
| 117 | while (nextArg(args, &arg_idx)) |arg| { | |
| 118 | if (mem.startsWith(u8, arg, "-Z")) { | |
| 119 | if (arg.len != 18) fatalWithHint("bad argument: '{s}'", .{arg}); | |
| 120 | output_tmp_nonce = arg[2..18].*; | |
| 121 | } else if (mem.startsWith(u8, arg, "-D")) { | |
| 122 | const option_contents = arg[2..]; | |
| 123 | if (option_contents.len == 0) | |
| 124 | fatalWithHint("expected option name after '-D'", .{}); | |
| 125 | if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { | |
| 126 | const option_name = option_contents[0..name_end]; | |
| 127 | const option_value = option_contents[name_end + 1 ..]; | |
| 128 | if (try builder.addUserInputOption(option_name, option_value)) | |
| 129 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 130 | } else { | |
| 131 | if (try builder.addUserInputFlag(option_contents)) | |
| 132 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 133 | } | |
| 134 | } else if (mem.startsWith(u8, arg, "-")) { | |
| 135 | if (mem.eql(u8, arg, "--verbose")) { | |
| 136 | builder.verbose = true; | |
| 137 | } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { | |
| 138 | help_menu = true; | |
| 139 | } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { | |
| 140 | install_prefix = nextArgOrFatal(args, &arg_idx); | |
| 141 | } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { | |
| 142 | steps_menu = true; | |
| 143 | } else if (mem.startsWith(u8, arg, "-fsys=")) { | |
| 144 | const name = arg["-fsys=".len..]; | |
| 145 | graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); | |
| 146 | } else if (mem.startsWith(u8, arg, "-fno-sys=")) { | |
| 147 | const name = arg["-fno-sys=".len..]; | |
| 148 | graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); | |
| 149 | } else if (mem.eql(u8, arg, "--release")) { | |
| 150 | builder.release_mode = .any; | |
| 151 | } else if (mem.startsWith(u8, arg, "--release=")) { | |
| 152 | const text = arg["--release=".len..]; | |
| 153 | builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { | |
| 154 | fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ | |
| 155 | arg, text, | |
| 156 | }); | |
| 157 | }; | |
| 158 | } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { | |
| 159 | dir_list.lib_dir = nextArgOrFatal(args, &arg_idx); | |
| 160 | } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { | |
| 161 | dir_list.exe_dir = nextArgOrFatal(args, &arg_idx); | |
| 162 | } else if (mem.eql(u8, arg, "--prefix-include-dir")) { | |
| 163 | dir_list.include_dir = nextArgOrFatal(args, &arg_idx); | |
| 164 | } else if (mem.eql(u8, arg, "--sysroot")) { | |
| 165 | builder.sysroot = nextArgOrFatal(args, &arg_idx); | |
| 166 | } else if (mem.eql(u8, arg, "--maxrss")) { | |
| 167 | const max_rss_text = nextArgOrFatal(args, &arg_idx); | |
| 168 | max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| { | |
| 169 | std.debug.print("invalid byte size: '{s}': {s}\n", .{ | |
| 170 | max_rss_text, @errorName(err), | |
| 171 | }); | |
| 172 | process.exit(1); | |
| 173 | }; | |
| 174 | } else if (mem.eql(u8, arg, "--skip-oom-steps")) { | |
| 175 | skip_oom_steps = true; | |
| 176 | } else if (mem.eql(u8, arg, "--search-prefix")) { | |
| 177 | const search_prefix = nextArgOrFatal(args, &arg_idx); | |
| 178 | builder.addSearchPrefix(search_prefix); | |
| 179 | } else if (mem.eql(u8, arg, "--libc")) { | |
| 180 | builder.libc_file = nextArgOrFatal(args, &arg_idx); | |
| 181 | } else if (mem.eql(u8, arg, "--color")) { | |
| 182 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 183 | fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); | |
| 184 | color = std.meta.stringToEnum(Color, next_arg) orelse { | |
| 185 | fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ | |
| 186 | arg, next_arg, | |
| 187 | }); | |
| 188 | }; | |
| 189 | } else if (mem.eql(u8, arg, "--summary")) { | |
| 190 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 191 | fatalWithHint("expected [all|new|failures|none] after '{s}'", .{arg}); | |
| 192 | summary = std.meta.stringToEnum(Summary, next_arg) orelse { | |
| 193 | fatalWithHint("expected [all|new|failures|none] after '{s}', found '{s}'", .{ | |
| 194 | arg, next_arg, | |
| 195 | }); | |
| 196 | }; | |
| 197 | } else if (mem.eql(u8, arg, "--seed")) { | |
| 198 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 199 | fatalWithHint("expected u32 after '{s}'", .{arg}); | |
| 200 | graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { | |
| 201 | fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ | |
| 202 | next_arg, @errorName(err), | |
| 203 | }); | |
| 204 | }; | |
| 205 | } else if (mem.eql(u8, arg, "--debounce")) { | |
| 206 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 207 | fatalWithHint("expected u16 after '{s}'", .{arg}); | |
| 208 | debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { | |
| 209 | fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {s}\n", .{ | |
| 210 | next_arg, @errorName(err), | |
| 211 | }); | |
| 212 | }; | |
| 213 | } else if (mem.eql(u8, arg, "--port")) { | |
| 214 | const next_arg = nextArg(args, &arg_idx) orelse | |
| 215 | fatalWithHint("expected u16 after '{s}'", .{arg}); | |
| 216 | listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| { | |
| 217 | fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{ | |
| 218 | next_arg, @errorName(err), | |
| 219 | }); | |
| 220 | }; | |
| 221 | } else if (mem.eql(u8, arg, "--debug-log")) { | |
| 222 | const next_arg = nextArgOrFatal(args, &arg_idx); | |
| 223 | try debug_log_scopes.append(next_arg); | |
| 224 | } else if (mem.eql(u8, arg, "--debug-pkg-config")) { | |
| 225 | builder.debug_pkg_config = true; | |
| 226 | } else if (mem.eql(u8, arg, "--debug-rt")) { | |
| 227 | graph.debug_compiler_runtime_libs = true; | |
| 228 | } else if (mem.eql(u8, arg, "--debug-compile-errors")) { | |
| 229 | builder.debug_compile_errors = true; | |
| 230 | } else if (mem.eql(u8, arg, "--system")) { | |
| 231 | // The usage text shows another argument after this parameter | |
| 232 | // but it is handled by the parent process. The build runner | |
| 233 | // only sees this flag. | |
| 234 | graph.system_package_mode = true; | |
| 235 | } else if (mem.eql(u8, arg, "--glibc-runtimes")) { | |
| 236 | builder.glibc_runtimes_dir = nextArgOrFatal(args, &arg_idx); | |
| 237 | } else if (mem.eql(u8, arg, "--verbose-link")) { | |
| 238 | builder.verbose_link = true; | |
| 239 | } else if (mem.eql(u8, arg, "--verbose-air")) { | |
| 240 | builder.verbose_air = true; | |
| 241 | } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { | |
| 242 | builder.verbose_llvm_ir = "-"; | |
| 243 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { | |
| 244 | builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; | |
| 245 | } else if (mem.eql(u8, arg, "--verbose-llvm-bc=")) { | |
| 246 | builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; | |
| 247 | } else if (mem.eql(u8, arg, "--verbose-cimport")) { | |
| 248 | builder.verbose_cimport = true; | |
| 249 | } else if (mem.eql(u8, arg, "--verbose-cc")) { | |
| 250 | builder.verbose_cc = true; | |
| 251 | } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { | |
| 252 | builder.verbose_llvm_cpu_features = true; | |
| 253 | } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { | |
| 254 | prominent_compile_errors = true; | |
| 255 | } else if (mem.eql(u8, arg, "--watch")) { | |
| 256 | watch = true; | |
| 257 | } else if (mem.eql(u8, arg, "--fuzz")) { | |
| 258 | fuzz = true; | |
| 259 | } else if (mem.eql(u8, arg, "-fincremental")) { | |
| 260 | graph.incremental = true; | |
| 261 | } else if (mem.eql(u8, arg, "-fno-incremental")) { | |
| 262 | graph.incremental = false; | |
| 263 | } else if (mem.eql(u8, arg, "-fwine")) { | |
| 264 | builder.enable_wine = true; | |
| 265 | } else if (mem.eql(u8, arg, "-fno-wine")) { | |
| 266 | builder.enable_wine = false; | |
| 267 | } else if (mem.eql(u8, arg, "-fqemu")) { | |
| 268 | builder.enable_qemu = true; | |
| 269 | } else if (mem.eql(u8, arg, "-fno-qemu")) { | |
| 270 | builder.enable_qemu = false; | |
| 271 | } else if (mem.eql(u8, arg, "-fwasmtime")) { | |
| 272 | builder.enable_wasmtime = true; | |
| 273 | } else if (mem.eql(u8, arg, "-fno-wasmtime")) { | |
| 274 | builder.enable_wasmtime = false; | |
| 275 | } else if (mem.eql(u8, arg, "-frosetta")) { | |
| 276 | builder.enable_rosetta = true; | |
| 277 | } else if (mem.eql(u8, arg, "-fno-rosetta")) { | |
| 278 | builder.enable_rosetta = false; | |
| 279 | } else if (mem.eql(u8, arg, "-fdarling")) { | |
| 280 | builder.enable_darling = true; | |
| 281 | } else if (mem.eql(u8, arg, "-fno-darling")) { | |
| 282 | builder.enable_darling = false; | |
| 283 | } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { | |
| 284 | graph.allow_so_scripts = true; | |
| 285 | } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { | |
| 286 | graph.allow_so_scripts = false; | |
| 287 | } else if (mem.eql(u8, arg, "-freference-trace")) { | |
| 288 | builder.reference_trace = 256; | |
| 289 | } else if (mem.startsWith(u8, arg, "-freference-trace=")) { | |
| 290 | const num = arg["-freference-trace=".len..]; | |
| 291 | builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 292 | std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); | |
| 293 | process.exit(1); | |
| 294 | }; | |
| 295 | } else if (mem.eql(u8, arg, "-fno-reference-trace")) { | |
| 296 | builder.reference_trace = null; | |
| 297 | } else if (mem.startsWith(u8, arg, "-j")) { | |
| 298 | const num = arg["-j".len..]; | |
| 299 | const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 300 | std.debug.print("unable to parse jobs count '{s}': {s}", .{ | |
| 301 | num, @errorName(err), | |
| 302 | }); | |
| 303 | process.exit(1); | |
| 304 | }; | |
| 305 | if (n_jobs < 1) { | |
| 306 | std.debug.print("number of jobs must be at least 1\n", .{}); | |
| 307 | process.exit(1); | |
| 308 | } | |
| 309 | thread_pool_options.n_jobs = n_jobs; | |
| 310 | } else if (mem.eql(u8, arg, "--")) { | |
| 311 | builder.args = argsRest(args, arg_idx); | |
| 312 | break; | |
| 313 | } else { | |
| 314 | fatalWithHint("unrecognized argument: '{s}'", .{arg}); | |
| 315 | } | |
| 316 | } else { | |
| 317 | try targets.append(arg); | |
| 318 | } | |
| 319 | } | |
| 320 | ||
| 321 | const stderr = std.io.getStdErr(); | |
| 322 | const ttyconf = get_tty_conf(color, stderr); | |
| 323 | switch (ttyconf) { | |
| 324 | .no_color => try graph.env_map.put("NO_COLOR", "1"), | |
| 325 | .escape_codes => try graph.env_map.put("CLICOLOR_FORCE", "1"), | |
| 326 | .windows_api => {}, | |
| 327 | } | |
| 328 | ||
| 329 | const main_progress_node = std.Progress.start(.{ | |
| 330 | .disable_printing = (color == .off), | |
| 331 | }); | |
| 332 | defer main_progress_node.end(); | |
| 333 | ||
| 334 | builder.debug_log_scopes = debug_log_scopes.items; | |
| 335 | builder.resolveInstallPrefix(install_prefix, dir_list); | |
| 336 | { | |
| 337 | var prog_node = main_progress_node.start("Configure", 0); | |
| 338 | defer prog_node.end(); | |
| 339 | try builder.runBuild(root); | |
| 340 | createModuleDependencies(builder) catch @panic("OOM"); | |
| 341 | } | |
| 342 | ||
| 343 | if (graph.needed_lazy_dependencies.entries.len != 0) { | |
| 344 | var buffer: std.ArrayListUnmanaged(u8) = .empty; | |
| 345 | for (graph.needed_lazy_dependencies.keys()) |k| { | |
| 346 | try buffer.appendSlice(arena, k); | |
| 347 | try buffer.append(arena, '\n'); | |
| 348 | } | |
| 349 | const s = std.fs.path.sep_str; | |
| 350 | const tmp_sub_path = "tmp" ++ s ++ (output_tmp_nonce orelse fatal("missing -Z arg", .{})); | |
| 351 | local_cache_directory.handle.writeFile(.{ | |
| 352 | .sub_path = tmp_sub_path, | |
| 353 | .data = buffer.items, | |
| 354 | .flags = .{ .exclusive = true }, | |
| 355 | }) catch |err| { | |
| 356 | fatal("unable to write configuration results to '{}{s}': {s}", .{ | |
| 357 | local_cache_directory, tmp_sub_path, @errorName(err), | |
| 358 | }); | |
| 359 | }; | |
| 360 | process.exit(3); // Indicate configure phase failed with meaningful stdout. | |
| 361 | } | |
| 362 | ||
| 363 | if (builder.validateUserInputDidItFail()) { | |
| 364 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 365 | } | |
| 366 | ||
| 367 | validateSystemLibraryOptions(builder); | |
| 368 | ||
| 369 | const stdout_writer = io.getStdOut().writer(); | |
| 370 | ||
| 371 | if (help_menu) | |
| 372 | return usage(builder, stdout_writer); | |
| 373 | ||
| 374 | if (steps_menu) | |
| 375 | return steps(builder, stdout_writer); | |
| 376 | ||
| 377 | var run: Run = .{ | |
| 378 | .max_rss = max_rss, | |
| 379 | .max_rss_is_default = false, | |
| 380 | .max_rss_mutex = .{}, | |
| 381 | .skip_oom_steps = skip_oom_steps, | |
| 382 | .watch = watch, | |
| 383 | .fuzz = fuzz, | |
| 384 | .memory_blocked_steps = std.ArrayList(*Step).init(arena), | |
| 385 | .step_stack = .{}, | |
| 386 | .prominent_compile_errors = prominent_compile_errors, | |
| 387 | ||
| 388 | .claimed_rss = 0, | |
| 389 | .summary = summary orelse if (watch) .new else .failures, | |
| 390 | .ttyconf = ttyconf, | |
| 391 | .stderr = stderr, | |
| 392 | .thread_pool = undefined, | |
| 393 | }; | |
| 394 | ||
| 395 | if (run.max_rss == 0) { | |
| 396 | run.max_rss = process.totalSystemMemory() catch std.math.maxInt(u64); | |
| 397 | run.max_rss_is_default = true; | |
| 398 | } | |
| 399 | ||
| 400 | const gpa = arena; | |
| 401 | prepare(gpa, arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { | |
| 402 | error.UncleanExit => process.exit(1), | |
| 403 | else => return err, | |
| 404 | }; | |
| 405 | ||
| 406 | var w = if (watch) try Watch.init() else undefined; | |
| 407 | ||
| 408 | try run.thread_pool.init(thread_pool_options); | |
| 409 | defer run.thread_pool.deinit(); | |
| 410 | ||
| 411 | rebuild: while (true) { | |
| 412 | runStepNames( | |
| 413 | gpa, | |
| 414 | builder, | |
| 415 | targets.items, | |
| 416 | main_progress_node, | |
| 417 | &run, | |
| 418 | ) catch |err| switch (err) { | |
| 419 | error.UncleanExit => { | |
| 420 | assert(!run.watch); | |
| 421 | process.exit(1); | |
| 422 | }, | |
| 423 | else => return err, | |
| 424 | }; | |
| 425 | if (fuzz) { | |
| 426 | switch (builtin.os.tag) { | |
| 427 | // Current implementation depends on two things that need to be ported to Windows: | |
| 428 | // * Memory-mapping to share data between the fuzzer and build runner. | |
| 429 | // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving | |
| 430 | // many addresses to source locations). | |
| 431 | .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), | |
| 432 | else => {}, | |
| 433 | } | |
| 434 | if (@bitSizeOf(usize) != 64) { | |
| 435 | // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, | |
| 436 | // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case | |
| 437 | // on 32-bit platforms. | |
| 438 | // Affects or affected by issues #5185, #22523, and #22464. | |
| 439 | fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); | |
| 440 | } | |
| 441 | const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable; | |
| 442 | try Fuzz.start( | |
| 443 | gpa, | |
| 444 | arena, | |
| 445 | global_cache_directory, | |
| 446 | zig_lib_directory, | |
| 447 | zig_exe, | |
| 448 | &run.thread_pool, | |
| 449 | run.step_stack.keys(), | |
| 450 | run.ttyconf, | |
| 451 | listen_address, | |
| 452 | main_progress_node, | |
| 453 | ); | |
| 454 | } | |
| 455 | ||
| 456 | if (!watch) return cleanExit(); | |
| 457 | ||
| 458 | if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); | |
| 459 | ||
| 460 | try w.update(gpa, run.step_stack.keys()); | |
| 461 | ||
| 462 | // Wait until a file system notification arrives. Read all such events | |
| 463 | // until the buffer is empty. Then wait for a debounce interval, resetting | |
| 464 | // if any more events come in. After the debounce interval has passed, | |
| 465 | // trigger a rebuild on all steps with modified inputs, as well as their | |
| 466 | // recursive dependants. | |
| 467 | var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; | |
| 468 | const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ | |
| 469 | w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), | |
| 470 | }) catch &caption_buf; | |
| 471 | var debouncing_node = main_progress_node.start(caption, 0); | |
| 472 | var debounce_timeout: Watch.Timeout = .none; | |
| 473 | while (true) switch (try w.wait(gpa, debounce_timeout)) { | |
| 474 | .timeout => { | |
| 475 | debouncing_node.end(); | |
| 476 | markFailedStepsDirty(gpa, run.step_stack.keys()); | |
| 477 | continue :rebuild; | |
| 478 | }, | |
| 479 | .dirty => if (debounce_timeout == .none) { | |
| 480 | debounce_timeout = .{ .ms = debounce_interval_ms }; | |
| 481 | debouncing_node.end(); | |
| 482 | debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); | |
| 483 | }, | |
| 484 | .clean => {}, | |
| 485 | }; | |
| 486 | } | |
| 487 | } | |
| 488 | ||
| 489 | fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { | |
| 490 | for (all_steps) |step| switch (step.state) { | |
| 491 | .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), | |
| 492 | else => continue, | |
| 493 | }; | |
| 494 | // Now that all dirty steps have been found, the remaining steps that | |
| 495 | // succeeded from last run shall be marked "cached". | |
| 496 | for (all_steps) |step| switch (step.state) { | |
| 497 | .success => step.result_cached = true, | |
| 498 | else => continue, | |
| 499 | }; | |
| 500 | } | |
| 501 | ||
| 502 | fn countSubProcesses(all_steps: []const *Step) usize { | |
| 503 | var count: usize = 0; | |
| 504 | for (all_steps) |s| { | |
| 505 | count += @intFromBool(s.getZigProcess() != null); | |
| 506 | } | |
| 507 | return count; | |
| 508 | } | |
| 509 | ||
| 510 | const Run = struct { | |
| 511 | max_rss: u64, | |
| 512 | max_rss_is_default: bool, | |
| 513 | max_rss_mutex: std.Thread.Mutex, | |
| 514 | skip_oom_steps: bool, | |
| 515 | watch: bool, | |
| 516 | fuzz: bool, | |
| 517 | memory_blocked_steps: std.ArrayList(*Step), | |
| 518 | step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 519 | prominent_compile_errors: bool, | |
| 520 | thread_pool: std.Thread.Pool, | |
| 521 | ||
| 522 | claimed_rss: usize, | |
| 523 | summary: Summary, | |
| 524 | ttyconf: std.io.tty.Config, | |
| 525 | stderr: File, | |
| 526 | ||
| 527 | fn cleanExit(run: Run) void { | |
| 528 | if (run.watch or run.fuzz) return; | |
| 529 | return runner.cleanExit(); | |
| 530 | } | |
| 531 | }; | |
| 532 | ||
| 533 | fn prepare( | |
| 534 | gpa: Allocator, | |
| 535 | arena: Allocator, | |
| 536 | b: *std.Build, | |
| 537 | step_names: []const []const u8, | |
| 538 | run: *Run, | |
| 539 | seed: u32, | |
| 540 | ) !void { | |
| 541 | const step_stack = &run.step_stack; | |
| 542 | ||
| 543 | if (step_names.len == 0) { | |
| 544 | try step_stack.put(gpa, b.default_step, {}); | |
| 545 | } else { | |
| 546 | try step_stack.ensureUnusedCapacity(gpa, step_names.len); | |
| 547 | for (0..step_names.len) |i| { | |
| 548 | const step_name = step_names[step_names.len - i - 1]; | |
| 549 | const s = b.top_level_steps.get(step_name) orelse { | |
| 550 | std.debug.print("no step named '{s}'\n access the help menu with 'zig build -h'\n", .{step_name}); | |
| 551 | process.exit(1); | |
| 552 | }; | |
| 553 | step_stack.putAssumeCapacity(&s.step, {}); | |
| 554 | } | |
| 555 | } | |
| 556 | ||
| 557 | const starting_steps = try arena.dupe(*Step, step_stack.keys()); | |
| 558 | ||
| 559 | var rng = std.Random.DefaultPrng.init(seed); | |
| 560 | const rand = rng.random(); | |
| 561 | rand.shuffle(*Step, starting_steps); | |
| 562 | ||
| 563 | for (starting_steps) |s| { | |
| 564 | constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) { | |
| 565 | error.DependencyLoopDetected => return uncleanExit(), | |
| 566 | else => |e| return e, | |
| 567 | }; | |
| 568 | } | |
| 569 | ||
| 570 | { | |
| 571 | // Check that we have enough memory to complete the build. | |
| 572 | var any_problems = false; | |
| 573 | for (step_stack.keys()) |s| { | |
| 574 | if (s.max_rss == 0) continue; | |
| 575 | if (s.max_rss > run.max_rss) { | |
| 576 | if (run.skip_oom_steps) { | |
| 577 | s.state = .skipped_oom; | |
| 578 | } else { | |
| 579 | std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{ | |
| 580 | s.owner.dep_prefix, s.name, s.max_rss, run.max_rss, | |
| 581 | }); | |
| 582 | any_problems = true; | |
| 583 | } | |
| 584 | } | |
| 585 | } | |
| 586 | if (any_problems) { | |
| 587 | if (run.max_rss_is_default) { | |
| 588 | std.debug.print("note: use --maxrss to override the default", .{}); | |
| 589 | } | |
| 590 | return uncleanExit(); | |
| 591 | } | |
| 592 | } | |
| 593 | } | |
| 594 | ||
| 595 | fn runStepNames( | |
| 596 | gpa: Allocator, | |
| 597 | b: *std.Build, | |
| 598 | step_names: []const []const u8, | |
| 599 | parent_prog_node: std.Progress.Node, | |
| 600 | run: *Run, | |
| 601 | ) !void { | |
| 602 | const step_stack = &run.step_stack; | |
| 603 | const thread_pool = &run.thread_pool; | |
| 604 | ||
| 605 | { | |
| 606 | const step_prog = parent_prog_node.start("steps", step_stack.count()); | |
| 607 | defer step_prog.end(); | |
| 608 | ||
| 609 | var wait_group: std.Thread.WaitGroup = .{}; | |
| 610 | defer wait_group.wait(); | |
| 611 | ||
| 612 | // Here we spawn the initial set of tasks with a nice heuristic - | |
| 613 | // dependency order. Each worker when it finishes a step will then | |
| 614 | // check whether it should run any dependants. | |
| 615 | const steps_slice = step_stack.keys(); | |
| 616 | for (0..steps_slice.len) |i| { | |
| 617 | const step = steps_slice[steps_slice.len - i - 1]; | |
| 618 | if (step.state == .skipped_oom) continue; | |
| 619 | ||
| 620 | thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{ | |
| 621 | &wait_group, b, step, step_prog, run, | |
| 622 | }); | |
| 623 | } | |
| 624 | } | |
| 625 | assert(run.memory_blocked_steps.items.len == 0); | |
| 626 | ||
| 627 | var test_skip_count: usize = 0; | |
| 628 | var test_fail_count: usize = 0; | |
| 629 | var test_pass_count: usize = 0; | |
| 630 | var test_leak_count: usize = 0; | |
| 631 | var test_count: usize = 0; | |
| 632 | ||
| 633 | var success_count: usize = 0; | |
| 634 | var skipped_count: usize = 0; | |
| 635 | var failure_count: usize = 0; | |
| 636 | var pending_count: usize = 0; | |
| 637 | var total_compile_errors: usize = 0; | |
| 638 | ||
| 639 | for (step_stack.keys()) |s| { | |
| 640 | test_fail_count += s.test_results.fail_count; | |
| 641 | test_skip_count += s.test_results.skip_count; | |
| 642 | test_leak_count += s.test_results.leak_count; | |
| 643 | test_pass_count += s.test_results.passCount(); | |
| 644 | test_count += s.test_results.test_count; | |
| 645 | ||
| 646 | switch (s.state) { | |
| 647 | .precheck_unstarted => unreachable, | |
| 648 | .precheck_started => unreachable, | |
| 649 | .running => unreachable, | |
| 650 | .precheck_done => { | |
| 651 | // precheck_done is equivalent to dependency_failure in the case of | |
| 652 | // transitive dependencies. For example: | |
| 653 | // A -> B -> C (failure) | |
| 654 | // B will be marked as dependency_failure, while A may never be queued, and thus | |
| 655 | // remain in the initial state of precheck_done. | |
| 656 | s.state = .dependency_failure; | |
| 657 | pending_count += 1; | |
| 658 | }, | |
| 659 | .dependency_failure => pending_count += 1, | |
| 660 | .success => success_count += 1, | |
| 661 | .skipped, .skipped_oom => skipped_count += 1, | |
| 662 | .failure => { | |
| 663 | failure_count += 1; | |
| 664 | const compile_errors_len = s.result_error_bundle.errorMessageCount(); | |
| 665 | if (compile_errors_len > 0) { | |
| 666 | total_compile_errors += compile_errors_len; | |
| 667 | } | |
| 668 | }, | |
| 669 | } | |
| 670 | } | |
| 671 | ||
| 672 | // A proper command line application defaults to silently succeeding. | |
| 673 | // The user may request verbose mode if they have a different preference. | |
| 674 | const failures_only = switch (run.summary) { | |
| 675 | .failures, .none => true, | |
| 676 | else => false, | |
| 677 | }; | |
| 678 | if (failure_count == 0 and failures_only) { | |
| 679 | return run.cleanExit(); | |
| 680 | } | |
| 681 | ||
| 682 | const ttyconf = run.ttyconf; | |
| 683 | ||
| 684 | if (run.summary != .none) { | |
| 685 | std.debug.lockStdErr(); | |
| 686 | defer std.debug.unlockStdErr(); | |
| 687 | const stderr = run.stderr; | |
| 688 | ||
| 689 | const total_count = success_count + failure_count + pending_count + skipped_count; | |
| 690 | ttyconf.setColor(stderr, .cyan) catch {}; | |
| 691 | stderr.writeAll("Build Summary:") catch {}; | |
| 692 | ttyconf.setColor(stderr, .reset) catch {}; | |
| 693 | stderr.writer().print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; | |
| 694 | if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {}; | |
| 695 | if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {}; | |
| 696 | ||
| 697 | if (test_count > 0) stderr.writer().print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; | |
| 698 | if (test_skip_count > 0) stderr.writer().print("; {d} skipped", .{test_skip_count}) catch {}; | |
| 699 | if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; | |
| 700 | if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; | |
| 701 | ||
| 702 | stderr.writeAll("\n") catch {}; | |
| 703 | ||
| 704 | // Print a fancy tree with build results. | |
| 705 | var step_stack_copy = try step_stack.clone(gpa); | |
| 706 | defer step_stack_copy.deinit(gpa); | |
| 707 | ||
| 708 | var print_node: PrintNode = .{ .parent = null }; | |
| 709 | if (step_names.len == 0) { | |
| 710 | print_node.last = true; | |
| 711 | printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; | |
| 712 | } else { | |
| 713 | const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { | |
| 714 | var i: usize = step_names.len; | |
| 715 | while (i > 0) { | |
| 716 | i -= 1; | |
| 717 | const step = b.top_level_steps.get(step_names[i]).?.step; | |
| 718 | const found = switch (run.summary) { | |
| 719 | .all, .none => unreachable, | |
| 720 | .failures => step.state != .success, | |
| 721 | .new => !step.result_cached, | |
| 722 | }; | |
| 723 | if (found) break :blk i; | |
| 724 | } | |
| 725 | break :blk b.top_level_steps.count(); | |
| 726 | }; | |
| 727 | for (step_names, 0..) |step_name, i| { | |
| 728 | const tls = b.top_level_steps.get(step_name).?; | |
| 729 | print_node.last = i + 1 == last_index; | |
| 730 | printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; | |
| 731 | } | |
| 732 | } | |
| 733 | } | |
| 734 | ||
| 735 | if (failure_count == 0) { | |
| 736 | return run.cleanExit(); | |
| 737 | } | |
| 738 | ||
| 739 | // Finally, render compile errors at the bottom of the terminal. | |
| 740 | if (run.prominent_compile_errors and total_compile_errors > 0) { | |
| 741 | for (step_stack.keys()) |s| { | |
| 742 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 743 | s.result_error_bundle.renderToStdErr(.{ .ttyconf = ttyconf, .include_reference_trace = (b.reference_trace orelse 0) > 0 }); | |
| 744 | } | |
| 745 | } | |
| 746 | ||
| 747 | if (!run.watch) { | |
| 748 | // Signal to parent process that we have printed compile errors. The | |
| 749 | // parent process may choose to omit the "following command failed" | |
| 750 | // line in this case. | |
| 751 | std.debug.lockStdErr(); | |
| 752 | process.exit(2); | |
| 753 | } | |
| 754 | } | |
| 755 | ||
| 756 | if (!run.watch) return uncleanExit(); | |
| 757 | } | |
| 758 | ||
| 759 | const PrintNode = struct { | |
| 760 | parent: ?*PrintNode, | |
| 761 | last: bool = false, | |
| 762 | }; | |
| 763 | ||
| 764 | fn printPrefix(node: *PrintNode, stderr: File, ttyconf: std.io.tty.Config) !void { | |
| 765 | const parent = node.parent orelse return; | |
| 766 | if (parent.parent == null) return; | |
| 767 | try printPrefix(parent, stderr, ttyconf); | |
| 768 | if (parent.last) { | |
| 769 | try stderr.writeAll(" "); | |
| 770 | } else { | |
| 771 | try stderr.writeAll(switch (ttyconf) { | |
| 772 | .no_color, .windows_api => "| ", | |
| 773 | .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ | |
| 774 | }); | |
| 775 | } | |
| 776 | } | |
| 777 | ||
| 778 | fn printChildNodePrefix(stderr: File, ttyconf: std.io.tty.Config) !void { | |
| 779 | try stderr.writeAll(switch (ttyconf) { | |
| 780 | .no_color, .windows_api => "+- ", | |
| 781 | .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ | |
| 782 | }); | |
| 783 | } | |
| 784 | ||
| 785 | fn printStepStatus( | |
| 786 | s: *Step, | |
| 787 | stderr: File, | |
| 788 | ttyconf: std.io.tty.Config, | |
| 789 | run: *const Run, | |
| 790 | ) !void { | |
| 791 | switch (s.state) { | |
| 792 | .precheck_unstarted => unreachable, | |
| 793 | .precheck_started => unreachable, | |
| 794 | .precheck_done => unreachable, | |
| 795 | .running => unreachable, | |
| 796 | ||
| 797 | .dependency_failure => { | |
| 798 | try ttyconf.setColor(stderr, .dim); | |
| 799 | try stderr.writeAll(" transitive failure\n"); | |
| 800 | try ttyconf.setColor(stderr, .reset); | |
| 801 | }, | |
| 802 | ||
| 803 | .success => { | |
| 804 | try ttyconf.setColor(stderr, .green); | |
| 805 | if (s.result_cached) { | |
| 806 | try stderr.writeAll(" cached"); | |
| 807 | } else if (s.test_results.test_count > 0) { | |
| 808 | const pass_count = s.test_results.passCount(); | |
| 809 | try stderr.writer().print(" {d} passed", .{pass_count}); | |
| 810 | if (s.test_results.skip_count > 0) { | |
| 811 | try ttyconf.setColor(stderr, .yellow); | |
| 812 | try stderr.writer().print(" {d} skipped", .{s.test_results.skip_count}); | |
| 813 | } | |
| 814 | } else { | |
| 815 | try stderr.writeAll(" success"); | |
| 816 | } | |
| 817 | try ttyconf.setColor(stderr, .reset); | |
| 818 | if (s.result_duration_ns) |ns| { | |
| 819 | try ttyconf.setColor(stderr, .dim); | |
| 820 | if (ns >= std.time.ns_per_min) { | |
| 821 | try stderr.writer().print(" {d}m", .{ns / std.time.ns_per_min}); | |
| 822 | } else if (ns >= std.time.ns_per_s) { | |
| 823 | try stderr.writer().print(" {d}s", .{ns / std.time.ns_per_s}); | |
| 824 | } else if (ns >= std.time.ns_per_ms) { | |
| 825 | try stderr.writer().print(" {d}ms", .{ns / std.time.ns_per_ms}); | |
| 826 | } else if (ns >= std.time.ns_per_us) { | |
| 827 | try stderr.writer().print(" {d}us", .{ns / std.time.ns_per_us}); | |
| 828 | } else { | |
| 829 | try stderr.writer().print(" {d}ns", .{ns}); | |
| 830 | } | |
| 831 | try ttyconf.setColor(stderr, .reset); | |
| 832 | } | |
| 833 | if (s.result_peak_rss != 0) { | |
| 834 | const rss = s.result_peak_rss; | |
| 835 | try ttyconf.setColor(stderr, .dim); | |
| 836 | if (rss >= 1000_000_000) { | |
| 837 | try stderr.writer().print(" MaxRSS:{d}G", .{rss / 1000_000_000}); | |
| 838 | } else if (rss >= 1000_000) { | |
| 839 | try stderr.writer().print(" MaxRSS:{d}M", .{rss / 1000_000}); | |
| 840 | } else if (rss >= 1000) { | |
| 841 | try stderr.writer().print(" MaxRSS:{d}K", .{rss / 1000}); | |
| 842 | } else { | |
| 843 | try stderr.writer().print(" MaxRSS:{d}B", .{rss}); | |
| 844 | } | |
| 845 | try ttyconf.setColor(stderr, .reset); | |
| 846 | } | |
| 847 | try stderr.writeAll("\n"); | |
| 848 | }, | |
| 849 | .skipped, .skipped_oom => |skip| { | |
| 850 | try ttyconf.setColor(stderr, .yellow); | |
| 851 | try stderr.writeAll(" skipped"); | |
| 852 | if (skip == .skipped_oom) { | |
| 853 | try stderr.writeAll(" (not enough memory)"); | |
| 854 | try ttyconf.setColor(stderr, .dim); | |
| 855 | try stderr.writer().print(" upper bound of {d} exceeded runner limit ({d})", .{ s.max_rss, run.max_rss }); | |
| 856 | try ttyconf.setColor(stderr, .yellow); | |
| 857 | } | |
| 858 | try stderr.writeAll("\n"); | |
| 859 | try ttyconf.setColor(stderr, .reset); | |
| 860 | }, | |
| 861 | .failure => try printStepFailure(s, stderr, ttyconf), | |
| 862 | } | |
| 863 | } | |
| 864 | ||
| 865 | fn printStepFailure( | |
| 866 | s: *Step, | |
| 867 | stderr: File, | |
| 868 | ttyconf: std.io.tty.Config, | |
| 869 | ) !void { | |
| 870 | if (s.result_error_bundle.errorMessageCount() > 0) { | |
| 871 | try ttyconf.setColor(stderr, .red); | |
| 872 | try stderr.writer().print(" {d} errors\n", .{ | |
| 873 | s.result_error_bundle.errorMessageCount(), | |
| 874 | }); | |
| 875 | try ttyconf.setColor(stderr, .reset); | |
| 876 | } else if (!s.test_results.isSuccess()) { | |
| 877 | try stderr.writer().print(" {d}/{d} passed", .{ | |
| 878 | s.test_results.passCount(), s.test_results.test_count, | |
| 879 | }); | |
| 880 | if (s.test_results.fail_count > 0) { | |
| 881 | try stderr.writeAll(", "); | |
| 882 | try ttyconf.setColor(stderr, .red); | |
| 883 | try stderr.writer().print("{d} failed", .{ | |
| 884 | s.test_results.fail_count, | |
| 885 | }); | |
| 886 | try ttyconf.setColor(stderr, .reset); | |
| 887 | } | |
| 888 | if (s.test_results.skip_count > 0) { | |
| 889 | try stderr.writeAll(", "); | |
| 890 | try ttyconf.setColor(stderr, .yellow); | |
| 891 | try stderr.writer().print("{d} skipped", .{ | |
| 892 | s.test_results.skip_count, | |
| 893 | }); | |
| 894 | try ttyconf.setColor(stderr, .reset); | |
| 895 | } | |
| 896 | if (s.test_results.leak_count > 0) { | |
| 897 | try stderr.writeAll(", "); | |
| 898 | try ttyconf.setColor(stderr, .red); | |
| 899 | try stderr.writer().print("{d} leaked", .{ | |
| 900 | s.test_results.leak_count, | |
| 901 | }); | |
| 902 | try ttyconf.setColor(stderr, .reset); | |
| 903 | } | |
| 904 | try stderr.writeAll("\n"); | |
| 905 | } else if (s.result_error_msgs.items.len > 0) { | |
| 906 | try ttyconf.setColor(stderr, .red); | |
| 907 | try stderr.writeAll(" failure\n"); | |
| 908 | try ttyconf.setColor(stderr, .reset); | |
| 909 | } else { | |
| 910 | assert(s.result_stderr.len > 0); | |
| 911 | try ttyconf.setColor(stderr, .red); | |
| 912 | try stderr.writeAll(" stderr\n"); | |
| 913 | try ttyconf.setColor(stderr, .reset); | |
| 914 | } | |
| 915 | } | |
| 916 | ||
| 917 | fn printTreeStep( | |
| 918 | b: *std.Build, | |
| 919 | s: *Step, | |
| 920 | run: *const Run, | |
| 921 | stderr: File, | |
| 922 | ttyconf: std.io.tty.Config, | |
| 923 | parent_node: *PrintNode, | |
| 924 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 925 | ) !void { | |
| 926 | const first = step_stack.swapRemove(s); | |
| 927 | const summary = run.summary; | |
| 928 | const skip = switch (summary) { | |
| 929 | .none => unreachable, | |
| 930 | .all => false, | |
| 931 | .new => s.result_cached, | |
| 932 | .failures => s.state == .success, | |
| 933 | }; | |
| 934 | if (skip) return; | |
| 935 | try printPrefix(parent_node, stderr, ttyconf); | |
| 936 | ||
| 937 | if (!first) try ttyconf.setColor(stderr, .dim); | |
| 938 | if (parent_node.parent != null) { | |
| 939 | if (parent_node.last) { | |
| 940 | try printChildNodePrefix(stderr, ttyconf); | |
| 941 | } else { | |
| 942 | try stderr.writeAll(switch (ttyconf) { | |
| 943 | .no_color, .windows_api => "+- ", | |
| 944 | .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ | |
| 945 | }); | |
| 946 | } | |
| 947 | } | |
| 948 | ||
| 949 | // dep_prefix omitted here because it is redundant with the tree. | |
| 950 | try stderr.writeAll(s.name); | |
| 951 | ||
| 952 | if (first) { | |
| 953 | try printStepStatus(s, stderr, ttyconf, run); | |
| 954 | ||
| 955 | const last_index = if (summary == .all) s.dependencies.items.len -| 1 else blk: { | |
| 956 | var i: usize = s.dependencies.items.len; | |
| 957 | while (i > 0) { | |
| 958 | i -= 1; | |
| 959 | ||
| 960 | const step = s.dependencies.items[i]; | |
| 961 | const found = switch (summary) { | |
| 962 | .all, .none => unreachable, | |
| 963 | .failures => step.state != .success, | |
| 964 | .new => !step.result_cached, | |
| 965 | }; | |
| 966 | if (found) break :blk i; | |
| 967 | } | |
| 968 | break :blk s.dependencies.items.len -| 1; | |
| 969 | }; | |
| 970 | for (s.dependencies.items, 0..) |dep, i| { | |
| 971 | var print_node: PrintNode = .{ | |
| 972 | .parent = parent_node, | |
| 973 | .last = i == last_index, | |
| 974 | }; | |
| 975 | try printTreeStep(b, dep, run, stderr, ttyconf, &print_node, step_stack); | |
| 976 | } | |
| 977 | } else { | |
| 978 | if (s.dependencies.items.len == 0) { | |
| 979 | try stderr.writeAll(" (reused)\n"); | |
| 980 | } else { | |
| 981 | try stderr.writer().print(" (+{d} more reused dependencies)\n", .{ | |
| 982 | s.dependencies.items.len, | |
| 983 | }); | |
| 984 | } | |
| 985 | try ttyconf.setColor(stderr, .reset); | |
| 986 | } | |
| 987 | } | |
| 988 | ||
| 989 | /// Traverse the dependency graph depth-first and make it undirected by having | |
| 990 | /// steps know their dependants (they only know dependencies at start). | |
| 991 | /// Along the way, check that there is no dependency loop, and record the steps | |
| 992 | /// in traversal order in `step_stack`. | |
| 993 | /// Each step has its dependencies traversed in random order, this accomplishes | |
| 994 | /// two things: | |
| 995 | /// - `step_stack` will be in randomized-depth-first order, so the build runner | |
| 996 | /// spawns steps in a random (but optimized) order | |
| 997 | /// - each step's `dependants` list is also filled in a random order, so that | |
| 998 | /// when it finishes executing in `workerMakeOneStep`, it spawns next steps | |
| 999 | /// to run in random order | |
| 1000 | fn constructGraphAndCheckForDependencyLoop( | |
| 1001 | b: *std.Build, | |
| 1002 | s: *Step, | |
| 1003 | step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), | |
| 1004 | rand: std.Random, | |
| 1005 | ) !void { | |
| 1006 | switch (s.state) { | |
| 1007 | .precheck_started => { | |
| 1008 | std.debug.print("dependency loop detected:\n {s}\n", .{s.name}); | |
| 1009 | return error.DependencyLoopDetected; | |
| 1010 | }, | |
| 1011 | .precheck_unstarted => { | |
| 1012 | s.state = .precheck_started; | |
| 1013 | ||
| 1014 | try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len); | |
| 1015 | ||
| 1016 | // We dupe to avoid shuffling the steps in the summary, it depends | |
| 1017 | // on s.dependencies' order. | |
| 1018 | const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM"); | |
| 1019 | rand.shuffle(*Step, deps); | |
| 1020 | ||
| 1021 | for (deps) |dep| { | |
| 1022 | try step_stack.put(b.allocator, dep, {}); | |
| 1023 | try dep.dependants.append(b.allocator, s); | |
| 1024 | constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| { | |
| 1025 | if (err == error.DependencyLoopDetected) { | |
| 1026 | std.debug.print(" {s}\n", .{s.name}); | |
| 1027 | } | |
| 1028 | return err; | |
| 1029 | }; | |
| 1030 | } | |
| 1031 | ||
| 1032 | s.state = .precheck_done; | |
| 1033 | }, | |
| 1034 | .precheck_done => {}, | |
| 1035 | ||
| 1036 | // These don't happen until we actually run the step graph. | |
| 1037 | .dependency_failure => unreachable, | |
| 1038 | .running => unreachable, | |
| 1039 | .success => unreachable, | |
| 1040 | .failure => unreachable, | |
| 1041 | .skipped => unreachable, | |
| 1042 | .skipped_oom => unreachable, | |
| 1043 | } | |
| 1044 | } | |
| 1045 | ||
| 1046 | fn workerMakeOneStep( | |
| 1047 | wg: *std.Thread.WaitGroup, | |
| 1048 | b: *std.Build, | |
| 1049 | s: *Step, | |
| 1050 | prog_node: std.Progress.Node, | |
| 1051 | run: *Run, | |
| 1052 | ) void { | |
| 1053 | const thread_pool = &run.thread_pool; | |
| 1054 | ||
| 1055 | // First, check the conditions for running this step. If they are not met, | |
| 1056 | // then we return without doing the step, relying on another worker to | |
| 1057 | // queue this step up again when dependencies are met. | |
| 1058 | for (s.dependencies.items) |dep| { | |
| 1059 | switch (@atomicLoad(Step.State, &dep.state, .seq_cst)) { | |
| 1060 | .success, .skipped => continue, | |
| 1061 | .failure, .dependency_failure, .skipped_oom => { | |
| 1062 | @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst); | |
| 1063 | return; | |
| 1064 | }, | |
| 1065 | .precheck_done, .running => { | |
| 1066 | // dependency is not finished yet. | |
| 1067 | return; | |
| 1068 | }, | |
| 1069 | .precheck_unstarted => unreachable, | |
| 1070 | .precheck_started => unreachable, | |
| 1071 | } | |
| 1072 | } | |
| 1073 | ||
| 1074 | if (s.max_rss != 0) { | |
| 1075 | run.max_rss_mutex.lock(); | |
| 1076 | defer run.max_rss_mutex.unlock(); | |
| 1077 | ||
| 1078 | // Avoid running steps twice. | |
| 1079 | if (s.state != .precheck_done) { | |
| 1080 | // Another worker got the job. | |
| 1081 | return; | |
| 1082 | } | |
| 1083 | ||
| 1084 | const new_claimed_rss = run.claimed_rss + s.max_rss; | |
| 1085 | if (new_claimed_rss > run.max_rss) { | |
| 1086 | // Running this step right now could possibly exceed the allotted RSS. | |
| 1087 | // Add this step to the queue of memory-blocked steps. | |
| 1088 | run.memory_blocked_steps.append(s) catch @panic("OOM"); | |
| 1089 | return; | |
| 1090 | } | |
| 1091 | ||
| 1092 | run.claimed_rss = new_claimed_rss; | |
| 1093 | s.state = .running; | |
| 1094 | } else { | |
| 1095 | // Avoid running steps twice. | |
| 1096 | if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .seq_cst, .seq_cst) != null) { | |
| 1097 | // Another worker got the job. | |
| 1098 | return; | |
| 1099 | } | |
| 1100 | } | |
| 1101 | ||
| 1102 | const sub_prog_node = prog_node.start(s.name, 0); | |
| 1103 | defer sub_prog_node.end(); | |
| 1104 | ||
| 1105 | const make_result = s.make(.{ | |
| 1106 | .progress_node = sub_prog_node, | |
| 1107 | .thread_pool = thread_pool, | |
| 1108 | .watch = run.watch, | |
| 1109 | }); | |
| 1110 | ||
| 1111 | // No matter the result, we want to display error/warning messages. | |
| 1112 | const show_compile_errors = !run.prominent_compile_errors and | |
| 1113 | s.result_error_bundle.errorMessageCount() > 0; | |
| 1114 | const show_error_msgs = s.result_error_msgs.items.len > 0; | |
| 1115 | const show_stderr = s.result_stderr.len > 0; | |
| 1116 | ||
| 1117 | if (show_error_msgs or show_compile_errors or show_stderr) { | |
| 1118 | std.debug.lockStdErr(); | |
| 1119 | defer std.debug.unlockStdErr(); | |
| 1120 | ||
| 1121 | const gpa = b.allocator; | |
| 1122 | const options: std.zig.ErrorBundle.RenderOptions = .{ | |
| 1123 | .ttyconf = run.ttyconf, | |
| 1124 | .include_reference_trace = (b.reference_trace orelse 0) > 0, | |
| 1125 | }; | |
| 1126 | printErrorMessages(gpa, s, options, run.stderr, run.prominent_compile_errors) catch {}; | |
| 1127 | } | |
| 1128 | ||
| 1129 | handle_result: { | |
| 1130 | if (make_result) |_| { | |
| 1131 | @atomicStore(Step.State, &s.state, .success, .seq_cst); | |
| 1132 | } else |err| switch (err) { | |
| 1133 | error.MakeFailed => { | |
| 1134 | @atomicStore(Step.State, &s.state, .failure, .seq_cst); | |
| 1135 | break :handle_result; | |
| 1136 | }, | |
| 1137 | error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .seq_cst), | |
| 1138 | } | |
| 1139 | ||
| 1140 | // Successful completion of a step, so we queue up its dependants as well. | |
| 1141 | for (s.dependants.items) |dep| { | |
| 1142 | thread_pool.spawnWg(wg, workerMakeOneStep, .{ | |
| 1143 | wg, b, dep, prog_node, run, | |
| 1144 | }); | |
| 1145 | } | |
| 1146 | } | |
| 1147 | ||
| 1148 | // If this is a step that claims resources, we must now queue up other | |
| 1149 | // steps that are waiting for resources. | |
| 1150 | if (s.max_rss != 0) { | |
| 1151 | run.max_rss_mutex.lock(); | |
| 1152 | defer run.max_rss_mutex.unlock(); | |
| 1153 | ||
| 1154 | // Give the memory back to the scheduler. | |
| 1155 | run.claimed_rss -= s.max_rss; | |
| 1156 | // Avoid kicking off too many tasks that we already know will not have | |
| 1157 | // enough resources. | |
| 1158 | var remaining = run.max_rss - run.claimed_rss; | |
| 1159 | var i: usize = 0; | |
| 1160 | var j: usize = 0; | |
| 1161 | while (j < run.memory_blocked_steps.items.len) : (j += 1) { | |
| 1162 | const dep = run.memory_blocked_steps.items[j]; | |
| 1163 | assert(dep.max_rss != 0); | |
| 1164 | if (dep.max_rss <= remaining) { | |
| 1165 | remaining -= dep.max_rss; | |
| 1166 | ||
| 1167 | thread_pool.spawnWg(wg, workerMakeOneStep, .{ | |
| 1168 | wg, b, dep, prog_node, run, | |
| 1169 | }); | |
| 1170 | } else { | |
| 1171 | run.memory_blocked_steps.items[i] = dep; | |
| 1172 | i += 1; | |
| 1173 | } | |
| 1174 | } | |
| 1175 | run.memory_blocked_steps.shrinkRetainingCapacity(i); | |
| 1176 | } | |
| 1177 | } | |
| 1178 | ||
| 1179 | pub fn printErrorMessages( | |
| 1180 | gpa: Allocator, | |
| 1181 | failing_step: *Step, | |
| 1182 | options: std.zig.ErrorBundle.RenderOptions, | |
| 1183 | stderr: File, | |
| 1184 | prominent_compile_errors: bool, | |
| 1185 | ) !void { | |
| 1186 | // Provide context for where these error messages are coming from by | |
| 1187 | // printing the corresponding Step subtree. | |
| 1188 | ||
| 1189 | var step_stack: std.ArrayListUnmanaged(*Step) = .empty; | |
| 1190 | defer step_stack.deinit(gpa); | |
| 1191 | try step_stack.append(gpa, failing_step); | |
| 1192 | while (step_stack.items[step_stack.items.len - 1].dependants.items.len != 0) { | |
| 1193 | try step_stack.append(gpa, step_stack.items[step_stack.items.len - 1].dependants.items[0]); | |
| 1194 | } | |
| 1195 | ||
| 1196 | // Now, `step_stack` has the subtree that we want to print, in reverse order. | |
| 1197 | const ttyconf = options.ttyconf; | |
| 1198 | try ttyconf.setColor(stderr, .dim); | |
| 1199 | var indent: usize = 0; | |
| 1200 | while (step_stack.pop()) |s| : (indent += 1) { | |
| 1201 | if (indent > 0) { | |
| 1202 | try stderr.writer().writeByteNTimes(' ', (indent - 1) * 3); | |
| 1203 | try printChildNodePrefix(stderr, ttyconf); | |
| 1204 | } | |
| 1205 | ||
| 1206 | try stderr.writeAll(s.name); | |
| 1207 | ||
| 1208 | if (s == failing_step) { | |
| 1209 | try printStepFailure(s, stderr, ttyconf); | |
| 1210 | } else { | |
| 1211 | try stderr.writeAll("\n"); | |
| 1212 | } | |
| 1213 | } | |
| 1214 | try ttyconf.setColor(stderr, .reset); | |
| 1215 | ||
| 1216 | if (failing_step.result_stderr.len > 0) { | |
| 1217 | try stderr.writeAll(failing_step.result_stderr); | |
| 1218 | if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { | |
| 1219 | try stderr.writeAll("\n"); | |
| 1220 | } | |
| 1221 | } | |
| 1222 | ||
| 1223 | if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) { | |
| 1224 | try failing_step.result_error_bundle.renderToWriter(options, stderr.writer()); | |
| 1225 | } | |
| 1226 | ||
| 1227 | for (failing_step.result_error_msgs.items) |msg| { | |
| 1228 | try ttyconf.setColor(stderr, .red); | |
| 1229 | try stderr.writeAll("error: "); | |
| 1230 | try ttyconf.setColor(stderr, .reset); | |
| 1231 | try stderr.writeAll(msg); | |
| 1232 | try stderr.writeAll("\n"); | |
| 1233 | } | |
| 1234 | } | |
| 1235 | ||
| 1236 | fn steps(builder: *std.Build, out_stream: anytype) !void { | |
| 1237 | const allocator = builder.allocator; | |
| 1238 | for (builder.top_level_steps.values()) |top_level_step| { | |
| 1239 | const name = if (&top_level_step.step == builder.default_step) | |
| 1240 | try fmt.allocPrint(allocator, "{s} (default)", .{top_level_step.step.name}) | |
| 1241 | else | |
| 1242 | top_level_step.step.name; | |
| 1243 | try out_stream.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); | |
| 1244 | } | |
| 1245 | } | |
| 1246 | ||
| 1247 | fn usage(b: *std.Build, out_stream: anytype) !void { | |
| 1248 | try out_stream.print( | |
| 1249 | \\Usage: {s} build [steps] [options] | |
| 1250 | \\ | |
| 1251 | \\Steps: | |
| 1252 | \\ | |
| 1253 | , .{b.graph.zig_exe}); | |
| 1254 | try steps(b, out_stream); | |
| 1255 | ||
| 1256 | try out_stream.writeAll( | |
| 1257 | \\ | |
| 1258 | \\General Options: | |
| 1259 | \\ -p, --prefix [path] Where to install files (default: zig-out) | |
| 1260 | \\ --prefix-lib-dir [path] Where to install libraries | |
| 1261 | \\ --prefix-exe-dir [path] Where to install executables | |
| 1262 | \\ --prefix-include-dir [path] Where to install C header files | |
| 1263 | \\ | |
| 1264 | \\ --release[=mode] Request release mode, optionally specifying a | |
| 1265 | \\ preferred optimization mode: fast, safe, small | |
| 1266 | \\ | |
| 1267 | \\ -fdarling, -fno-darling Integration with system-installed Darling to | |
| 1268 | \\ execute macOS programs on Linux hosts | |
| 1269 | \\ (default: no) | |
| 1270 | \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute | |
| 1271 | \\ foreign-architecture programs on Linux hosts | |
| 1272 | \\ (default: no) | |
| 1273 | \\ --glibc-runtimes [path] Enhances QEMU integration by providing glibc built | |
| 1274 | \\ for multiple foreign architectures, allowing | |
| 1275 | \\ execution of non-native programs that link with glibc. | |
| 1276 | \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on | |
| 1277 | \\ ARM64 macOS hosts. (default: no) | |
| 1278 | \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to | |
| 1279 | \\ execute WASI binaries. (default: no) | |
| 1280 | \\ -fwine, -fno-wine Integration with system-installed Wine to execute | |
| 1281 | \\ Windows programs on Linux hosts. (default: no) | |
| 1282 | \\ | |
| 1283 | \\ -h, --help Print this help and exit | |
| 1284 | \\ -l, --list-steps Print available steps | |
| 1285 | \\ --verbose Print commands before executing them | |
| 1286 | \\ --color [auto|off|on] Enable or disable colored error messages | |
| 1287 | \\ --prominent-compile-errors Buffer compile errors and display at end | |
| 1288 | \\ --summary [mode] Control the printing of the build summary | |
| 1289 | \\ all Print the build summary in its entirety | |
| 1290 | \\ new Omit cached steps | |
| 1291 | \\ failures (Default) Only print failed steps | |
| 1292 | \\ none Do not print the build summary | |
| 1293 | \\ -j<N> Limit concurrent jobs (default is to use all CPU cores) | |
| 1294 | \\ --maxrss <bytes> Limit memory usage (default is to use available memory) | |
| 1295 | \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss | |
| 1296 | \\ --fetch Exit after fetching dependency tree | |
| 1297 | \\ --watch Continuously rebuild when source files are modified | |
| 1298 | \\ --fuzz Continuously search for unit test failures | |
| 1299 | \\ --debounce <ms> Delay before rebuilding after changed file detected | |
| 1300 | \\ -fincremental Enable incremental compilation | |
| 1301 | \\ -fno-incremental Disable incremental compilation | |
| 1302 | \\ | |
| 1303 | \\Project-Specific Options: | |
| 1304 | \\ | |
| 1305 | ); | |
| 1306 | ||
| 1307 | const arena = b.allocator; | |
| 1308 | if (b.available_options_list.items.len == 0) { | |
| 1309 | try out_stream.print(" (none)\n", .{}); | |
| 1310 | } else { | |
| 1311 | for (b.available_options_list.items) |option| { | |
| 1312 | const name = try fmt.allocPrint(arena, " -D{s}=[{s}]", .{ | |
| 1313 | option.name, | |
| 1314 | @tagName(option.type_id), | |
| 1315 | }); | |
| 1316 | try out_stream.print("{s:<30} {s}\n", .{ name, option.description }); | |
| 1317 | if (option.enum_options) |enum_options| { | |
| 1318 | const padding = " " ** 33; | |
| 1319 | try out_stream.writeAll(padding ++ "Supported Values:\n"); | |
| 1320 | for (enum_options) |enum_option| { | |
| 1321 | try out_stream.print(padding ++ " {s}\n", .{enum_option}); | |
| 1322 | } | |
| 1323 | } | |
| 1324 | } | |
| 1325 | } | |
| 1326 | ||
| 1327 | try out_stream.writeAll( | |
| 1328 | \\ | |
| 1329 | \\System Integration Options: | |
| 1330 | \\ --search-prefix [path] Add a path to look for binaries, libraries, headers | |
| 1331 | \\ --sysroot [path] Set the system root directory (usually /) | |
| 1332 | \\ --libc [file] Provide a file which specifies libc paths | |
| 1333 | \\ | |
| 1334 | \\ --system [pkgdir] Disable package fetching; enable all integrations | |
| 1335 | \\ -fsys=[name] Enable a system integration | |
| 1336 | \\ -fno-sys=[name] Disable a system integration | |
| 1337 | \\ | |
| 1338 | \\ Available System Integrations: Enabled: | |
| 1339 | \\ | |
| 1340 | ); | |
| 1341 | if (b.graph.system_library_options.entries.len == 0) { | |
| 1342 | try out_stream.writeAll(" (none) -\n"); | |
| 1343 | } else { | |
| 1344 | for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { | |
| 1345 | const status = switch (v) { | |
| 1346 | .declared_enabled => "yes", | |
| 1347 | .declared_disabled => "no", | |
| 1348 | .user_enabled, .user_disabled => unreachable, // already emitted error | |
| 1349 | }; | |
| 1350 | try out_stream.print(" {s:<43} {s}\n", .{ k, status }); | |
| 1351 | } | |
| 1352 | } | |
| 1353 | ||
| 1354 | try out_stream.writeAll( | |
| 1355 | \\ | |
| 1356 | \\Advanced Options: | |
| 1357 | \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error | |
| 1358 | \\ -fno-reference-trace Disable reference trace | |
| 1359 | \\ -fallow-so-scripts Allows .so files to be GNU ld scripts | |
| 1360 | \\ -fno-allow-so-scripts (default) .so files must be ELF files | |
| 1361 | \\ --build-file [file] Override path to build.zig | |
| 1362 | \\ --cache-dir [path] Override path to local Zig cache directory | |
| 1363 | \\ --global-cache-dir [path] Override path to global Zig cache directory | |
| 1364 | \\ --zig-lib-dir [arg] Override path to Zig lib directory | |
| 1365 | \\ --build-runner [file] Override path to build runner | |
| 1366 | \\ --seed [integer] For shuffling dependency traversal order (default: random) | |
| 1367 | \\ --debug-log [scope] Enable debugging the compiler | |
| 1368 | \\ --debug-pkg-config Fail if unknown pkg-config flags encountered | |
| 1369 | \\ --debug-rt Debug compiler runtime libraries | |
| 1370 | \\ --verbose-link Enable compiler debug output for linking | |
| 1371 | \\ --verbose-air Enable compiler debug output for Zig AIR | |
| 1372 | \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR | |
| 1373 | \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC | |
| 1374 | \\ --verbose-cimport Enable compiler debug output for C imports | |
| 1375 | \\ --verbose-cc Enable compiler debug output for C compilation | |
| 1376 | \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features | |
| 1377 | \\ | |
| 1378 | ); | |
| 1379 | } | |
| 1380 | ||
| 1381 | fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { | |
| 1382 | if (idx.* >= args.len) return null; | |
| 1383 | defer idx.* += 1; | |
| 1384 | return args[idx.*]; | |
| 1385 | } | |
| 1386 | ||
| 1387 | fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { | |
| 1388 | return nextArg(args, idx) orelse { | |
| 1389 | std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); | |
| 1390 | process.exit(1); | |
| 1391 | }; | |
| 1392 | } | |
| 1393 | ||
| 1394 | fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { | |
| 1395 | if (idx >= args.len) return null; | |
| 1396 | return args[idx..]; | |
| 1397 | } | |
| 1398 | ||
| 1399 | /// Perhaps in the future there could be an Advanced Options flag such as | |
| 1400 | /// --debug-build-runner-leaks which would make this function return instead of | |
| 1401 | /// calling exit. | |
| 1402 | fn cleanExit() void { | |
| 1403 | std.debug.lockStdErr(); | |
| 1404 | process.exit(0); | |
| 1405 | } | |
| 1406 | ||
| 1407 | /// Perhaps in the future there could be an Advanced Options flag such as | |
| 1408 | /// --debug-build-runner-leaks which would make this function return instead of | |
| 1409 | /// calling exit. | |
| 1410 | fn uncleanExit() error{UncleanExit} { | |
| 1411 | std.debug.lockStdErr(); | |
| 1412 | process.exit(1); | |
| 1413 | } | |
| 1414 | ||
| 1415 | const Color = std.zig.Color; | |
| 1416 | const Summary = enum { all, new, failures, none }; | |
| 1417 | ||
| 1418 | fn get_tty_conf(color: Color, stderr: File) std.io.tty.Config { | |
| 1419 | return switch (color) { | |
| 1420 | .auto => std.io.tty.detectConfig(stderr), | |
| 1421 | .on => .escape_codes, | |
| 1422 | .off => .no_color, | |
| 1423 | }; | |
| 1424 | } | |
| 1425 | ||
| 1426 | fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { | |
| 1427 | std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); | |
| 1428 | process.exit(1); | |
| 1429 | } | |
| 1430 | ||
| 1431 | fn validateSystemLibraryOptions(b: *std.Build) void { | |
| 1432 | var bad = false; | |
| 1433 | for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { | |
| 1434 | switch (v) { | |
| 1435 | .user_disabled, .user_enabled => { | |
| 1436 | // The user tried to enable or disable a system library integration, but | |
| 1437 | // the build script did not recognize that option. | |
| 1438 | std.debug.print("system library name not recognized by build script: '{s}'\n", .{k}); | |
| 1439 | bad = true; | |
| 1440 | }, | |
| 1441 | .declared_disabled, .declared_enabled => {}, | |
| 1442 | } | |
| 1443 | } | |
| 1444 | if (bad) { | |
| 1445 | std.debug.print(" access the help menu with 'zig build -h'\n", .{}); | |
| 1446 | process.exit(1); | |
| 1447 | } | |
| 1448 | } | |
| 1449 | ||
| 1450 | /// Starting from all top-level steps in `b`, traverses the entire step graph | |
| 1451 | /// and adds all step dependencies implied by module graphs. | |
| 1452 | fn createModuleDependencies(b: *std.Build) Allocator.Error!void { | |
| 1453 | const arena = b.graph.arena; | |
| 1454 | ||
| 1455 | var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; | |
| 1456 | var next_step_idx: usize = 0; | |
| 1457 | ||
| 1458 | try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); | |
| 1459 | for (b.top_level_steps.values()) |tls| { | |
| 1460 | all_steps.putAssumeCapacityNoClobber(&tls.step, {}); | |
| 1461 | } | |
| 1462 | ||
| 1463 | while (next_step_idx < all_steps.count()) { | |
| 1464 | const step = all_steps.keys()[next_step_idx]; | |
| 1465 | next_step_idx += 1; | |
| 1466 | ||
| 1467 | // Set up any implied dependencies for this step. It's important that we do this first, so | |
| 1468 | // that the loop below discovers steps implied by the module graph. | |
| 1469 | try createModuleDependenciesForStep(step); | |
| 1470 | ||
| 1471 | try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); | |
| 1472 | for (step.dependencies.items) |other_step| { | |
| 1473 | all_steps.putAssumeCapacity(other_step, {}); | |
| 1474 | } | |
| 1475 | } | |
| 1476 | } | |
| 1477 | ||
| 1478 | /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which | |
| 1479 | /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. | |
| 1480 | fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { | |
| 1481 | const root_module = if (step.cast(Step.Compile)) |cs| root: { | |
| 1482 | break :root cs.root_module; | |
| 1483 | } else return; // not a compile step so no module dependencies | |
| 1484 | ||
| 1485 | // Starting from `root_module`, discover all modules in this graph. | |
| 1486 | const modules = root_module.getGraph().modules; | |
| 1487 | ||
| 1488 | // For each of those modules, set up the implied step dependencies. | |
| 1489 | for (modules) |mod| { | |
| 1490 | if (mod.root_source_file) |lp| lp.addStepDependencies(step); | |
| 1491 | for (mod.include_dirs.items) |include_dir| switch (include_dir) { | |
| 1492 | .path, | |
| 1493 | .path_system, | |
| 1494 | .path_after, | |
| 1495 | .framework_path, | |
| 1496 | .framework_path_system, | |
| 1497 | => |lp| lp.addStepDependencies(step), | |
| 1498 | ||
| 1499 | .other_step => |other| { | |
| 1500 | other.getEmittedIncludeTree().addStepDependencies(step); | |
| 1501 | step.dependOn(&other.step); | |
| 1502 | }, | |
| 1503 | ||
| 1504 | .config_header_step => |other| step.dependOn(&other.step), | |
| 1505 | }; | |
| 1506 | for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); | |
| 1507 | for (mod.rpaths.items) |rpath| switch (rpath) { | |
| 1508 | .lazy_path => |lp| lp.addStepDependencies(step), | |
| 1509 | .special => {}, | |
| 1510 | }; | |
| 1511 | for (mod.link_objects.items) |link_object| switch (link_object) { | |
| 1512 | .static_path, | |
| 1513 | .assembly_file, | |
| 1514 | => |lp| lp.addStepDependencies(step), | |
| 1515 | .other_step => |other| step.dependOn(&other.step), | |
| 1516 | .system_lib => {}, | |
| 1517 | .c_source_file => |source| source.file.addStepDependencies(step), | |
| 1518 | .c_source_files => |source_files| source_files.root.addStepDependencies(step), | |
| 1519 | .win32_resource_file => |rc_source| { | |
| 1520 | rc_source.file.addStepDependencies(step); | |
| 1521 | for (rc_source.include_paths) |lp| lp.addStepDependencies(step); | |
| 1522 | }, | |
| 1523 | }; | |
| 1524 | } | |
| 1525 | } |
lib/compiler/configure_runner.zig created+214
| ... | ... | @@ -0,0 +1,214 @@ |
| 1 | const builtin = @import("builtin"); | |
| 2 | ||
| 3 | const std = @import("std"); | |
| 4 | const mem = std.mem; | |
| 5 | const fatal = std.process.fatal; | |
| 6 | const assert = std.debug.assert; | |
| 7 | const Allocator = std.mem.Allocator; | |
| 8 | const Step = std.Build.Step; | |
| 9 | ||
| 10 | pub const root = @import("@build"); | |
| 11 | pub const dependencies = @import("@dependencies"); | |
| 12 | ||
| 13 | pub const std_options: std.Options = .{ | |
| 14 | .side_channels_mitigations = .none, | |
| 15 | .http_disable_tls = true, | |
| 16 | .crypto_fork_safety = false, | |
| 17 | }; | |
| 18 | ||
| 19 | comptime { | |
| 20 | assert(builtin.single_threaded); | |
| 21 | } | |
| 22 | ||
| 23 | pub fn main() !void { | |
| 24 | var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); | |
| 25 | defer single_threaded_arena.deinit(); | |
| 26 | const arena = single_threaded_arena.allocator(); | |
| 27 | ||
| 28 | const args = try std.process.argsAlloc(arena); | |
| 29 | ||
| 30 | // skip my own exe name | |
| 31 | var arg_idx: usize = 1; | |
| 32 | ||
| 33 | const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); | |
| 34 | const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); | |
| 35 | const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); | |
| 36 | const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); | |
| 37 | const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); | |
| 38 | ||
| 39 | const zig_lib_directory: std.Build.Cache.Directory = .{ | |
| 40 | .path = zig_lib_dir, | |
| 41 | .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), | |
| 42 | }; | |
| 43 | ||
| 44 | const build_root_directory: std.Build.Cache.Directory = .{ | |
| 45 | .path = build_root, | |
| 46 | .handle = try std.fs.cwd().openDir(build_root, .{}), | |
| 47 | }; | |
| 48 | ||
| 49 | const local_cache_directory: std.Build.Cache.Directory = .{ | |
| 50 | .path = cache_root, | |
| 51 | .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), | |
| 52 | }; | |
| 53 | ||
| 54 | const global_cache_directory: std.Build.Cache.Directory = .{ | |
| 55 | .path = global_cache_root, | |
| 56 | .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), | |
| 57 | }; | |
| 58 | ||
| 59 | var graph: std.Build.Graph = .{ | |
| 60 | .arena = arena, | |
| 61 | .cache = .{ | |
| 62 | .gpa = arena, | |
| 63 | .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), | |
| 64 | }, | |
| 65 | .zig_exe = zig_exe, | |
| 66 | .env_map = try std.process.getEnvMap(arena), | |
| 67 | .global_cache_root = global_cache_directory, | |
| 68 | .zig_lib_directory = zig_lib_directory, | |
| 69 | .host = .{ | |
| 70 | .query = .{}, | |
| 71 | .result = try std.zig.system.resolveTargetQuery(.{}), | |
| 72 | }, | |
| 73 | }; | |
| 74 | ||
| 75 | graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); | |
| 76 | graph.cache.addPrefix(build_root_directory); | |
| 77 | graph.cache.addPrefix(local_cache_directory); | |
| 78 | graph.cache.addPrefix(global_cache_directory); | |
| 79 | graph.cache.hash.addBytes(builtin.zig_version_string); | |
| 80 | ||
| 81 | const builder = try std.Build.create( | |
| 82 | &graph, | |
| 83 | build_root_directory, | |
| 84 | local_cache_directory, | |
| 85 | dependencies.root_deps, | |
| 86 | ); | |
| 87 | ||
| 88 | var install_prefix: ?std.Build.Cache.Path = null; | |
| 89 | var install_paths: std.Build.InstallPaths = .{}; | |
| 90 | ||
| 91 | while (nextArg(args, &arg_idx)) |arg| { | |
| 92 | if (mem.startsWith(u8, arg, "-D")) { | |
| 93 | const option_contents = arg[2..]; | |
| 94 | if (option_contents.len == 0) | |
| 95 | fatal("expected option name after '-D'", .{}); | |
| 96 | if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { | |
| 97 | const option_name = option_contents[0..name_end]; | |
| 98 | const option_value = option_contents[name_end + 1 ..]; | |
| 99 | if (try builder.addUserInputOption(option_name, option_value)) | |
| 100 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 101 | } else { | |
| 102 | if (try builder.addUserInputFlag(option_contents)) | |
| 103 | fatal(" access the help menu with 'zig build -h'", .{}); | |
| 104 | } | |
| 105 | } else if (mem.startsWith(u8, arg, "-")) { | |
| 106 | if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { | |
| 107 | install_prefix = nextArgOrFatal(args, &arg_idx); | |
| 108 | } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { | |
| 109 | install_paths.lib_dir = nextArgOrFatal(args, &arg_idx); | |
| 110 | } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { | |
| 111 | install_paths.exe_dir = nextArgOrFatal(args, &arg_idx); | |
| 112 | } else if (mem.eql(u8, arg, "--prefix-include-dir")) { | |
| 113 | install_paths.include_dir = nextArgOrFatal(args, &arg_idx); | |
| 114 | } else { | |
| 115 | fatal("unrecognized argument: '{s}'", .{arg}); | |
| 116 | } | |
| 117 | } else { | |
| 118 | fatal("unrecognized argument: '{s}'", .{arg}); | |
| 119 | } | |
| 120 | } | |
| 121 | ||
| 122 | builder.resolveInstallPrefix(install_prefix, install_paths); | |
| 123 | try builder.runBuild(root); | |
| 124 | createModuleDependencies(builder) catch @panic("OOM"); | |
| 125 | ||
| 126 | try std.io.getStdOut().writeAll("TODO\n"); | |
| 127 | } | |
| 128 | ||
| 129 | fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { | |
| 130 | if (idx.* >= args.len) return null; | |
| 131 | defer idx.* += 1; | |
| 132 | return args[idx.*]; | |
| 133 | } | |
| 134 | ||
| 135 | fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { | |
| 136 | return nextArg(args, idx) orelse fatal("expected argument after '{s}'", .{args[idx.* - 1]}); | |
| 137 | } | |
| 138 | ||
| 139 | /// Starting from all top-level steps in `b`, traverses the entire step graph | |
| 140 | /// and adds all step dependencies implied by module graphs. | |
| 141 | fn createModuleDependencies(b: *std.Build) Allocator.Error!void { | |
| 142 | const arena = b.graph.arena; | |
| 143 | ||
| 144 | var all_steps: std.AutoArrayHashMapUnmanaged(*Step, void) = .empty; | |
| 145 | var next_step_idx: usize = 0; | |
| 146 | ||
| 147 | try all_steps.ensureUnusedCapacity(arena, b.top_level_steps.count()); | |
| 148 | for (b.top_level_steps.values()) |tls| { | |
| 149 | all_steps.putAssumeCapacityNoClobber(&tls.step, {}); | |
| 150 | } | |
| 151 | ||
| 152 | while (next_step_idx < all_steps.count()) { | |
| 153 | const step = all_steps.keys()[next_step_idx]; | |
| 154 | next_step_idx += 1; | |
| 155 | ||
| 156 | // Set up any implied dependencies for this step. It's important that we do this first, so | |
| 157 | // that the loop below discovers steps implied by the module graph. | |
| 158 | try createModuleDependenciesForStep(step); | |
| 159 | ||
| 160 | try all_steps.ensureUnusedCapacity(arena, step.dependencies.items.len); | |
| 161 | for (step.dependencies.items) |other_step| { | |
| 162 | all_steps.putAssumeCapacity(other_step, {}); | |
| 163 | } | |
| 164 | } | |
| 165 | } | |
| 166 | ||
| 167 | /// If the given `Step` is a `Step.Compile`, adds any dependencies for that step which | |
| 168 | /// are implied by the module graph rooted at `step.cast(Step.Compile).?.root_module`. | |
| 169 | fn createModuleDependenciesForStep(step: *Step) Allocator.Error!void { | |
| 170 | const root_module = if (step.cast(Step.Compile)) |cs| root: { | |
| 171 | break :root cs.root_module; | |
| 172 | } else return; // not a compile step so no module dependencies | |
| 173 | ||
| 174 | // Starting from `root_module`, discover all modules in this graph. | |
| 175 | const modules = root_module.getGraph().modules; | |
| 176 | ||
| 177 | // For each of those modules, set up the implied step dependencies. | |
| 178 | for (modules) |mod| { | |
| 179 | if (mod.root_source_file) |lp| lp.addStepDependencies(step); | |
| 180 | for (mod.include_dirs.items) |include_dir| switch (include_dir) { | |
| 181 | .path, | |
| 182 | .path_system, | |
| 183 | .path_after, | |
| 184 | .framework_path, | |
| 185 | .framework_path_system, | |
| 186 | => |lp| lp.addStepDependencies(step), | |
| 187 | ||
| 188 | .other_step => |other| { | |
| 189 | other.getEmittedIncludeTree().addStepDependencies(step); | |
| 190 | step.dependOn(&other.step); | |
| 191 | }, | |
| 192 | ||
| 193 | .config_header_step => |other| step.dependOn(&other.step), | |
| 194 | }; | |
| 195 | for (mod.lib_paths.items) |lp| lp.addStepDependencies(step); | |
| 196 | for (mod.rpaths.items) |rpath| switch (rpath) { | |
| 197 | .lazy_path => |lp| lp.addStepDependencies(step), | |
| 198 | .special => {}, | |
| 199 | }; | |
| 200 | for (mod.link_objects.items) |link_object| switch (link_object) { | |
| 201 | .static_path, | |
| 202 | .assembly_file, | |
| 203 | => |lp| lp.addStepDependencies(step), | |
| 204 | .other_step => |other| step.dependOn(&other.step), | |
| 205 | .system_lib => {}, | |
| 206 | .c_source_file => |source| source.file.addStepDependencies(step), | |
| 207 | .c_source_files => |source_files| source_files.root.addStepDependencies(step), | |
| 208 | .win32_resource_file => |rc_source| { | |
| 209 | rc_source.file.addStepDependencies(step); | |
| 210 | for (rc_source.include_paths) |lp| lp.addStepDependencies(step); | |
| 211 | }, | |
| 212 | }; | |
| 213 | } | |
| 214 | } |
lib/compiler/fetch.zig created+382
| ... | ... | @@ -0,0 +1,382 @@ |
| 1 | const builtin = @import("builtin"); | |
| 2 | const native_os = builtin.os.tag; | |
| 3 | ||
| 4 | const std = @import("std"); | |
| 5 | const mem = std.mem; | |
| 6 | const fs = std.fs; | |
| 7 | const process = std.process; | |
| 8 | const fatal = std.process.fatal; | |
| 9 | const Path = std.Build.Cache.Path; | |
| 10 | const Directory = std.Build.Cache.Directory; | |
| 11 | const Package = std.zig.Package; | |
| 12 | const Allocator = std.mem.Allocator; | |
| 13 | ||
| 14 | const usage = | |
| 15 | \\Usage: zig fetch [options] <url> | |
| 16 | \\Usage: zig fetch [options] <path> | |
| 17 | \\ | |
| 18 | \\ Copy a package into the global cache and print its hash. | |
| 19 | \\ <url> must point to one of the following: | |
| 20 | \\ - A git+http / git+https server for the package | |
| 21 | \\ - A tarball file (with or without compression) containing | |
| 22 | \\ package source | |
| 23 | \\ - A git bundle file containing package source | |
| 24 | \\ | |
| 25 | \\Examples: | |
| 26 | \\ | |
| 27 | \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git | |
| 28 | \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz | |
| 29 | \\ | |
| 30 | \\Options: | |
| 31 | \\ -h, --help Print this help and exit | |
| 32 | \\ --global-cache-dir [path] Override path to global Zig cache directory | |
| 33 | \\ --debug-hash Print verbose hash information to stdout | |
| 34 | \\ --save Add the fetched package to build.zig.zon | |
| 35 | \\ --save=[name] Add the fetched package to build.zig.zon as name | |
| 36 | \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim | |
| 37 | \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim | |
| 38 | \\ | |
| 39 | ; | |
| 40 | ||
| 41 | pub const std_options: std.Options = .{ | |
| 42 | .side_channels_mitigations = .none, | |
| 43 | .crypto_fork_safety = false, | |
| 44 | }; | |
| 45 | ||
| 46 | pub fn main() !void { | |
| 47 | var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); | |
| 48 | defer arena_instance.deinit(); | |
| 49 | const arena = arena_instance.allocator(); | |
| 50 | ||
| 51 | const gpa = arena; | |
| 52 | ||
| 53 | const args = try process.argsAlloc(arena); | |
| 54 | ||
| 55 | var zig_lib_directory: Directory = .{ | |
| 56 | .handle = try std.fs.cwd().openDir(args[1], .{}), | |
| 57 | }; | |
| 58 | defer zig_lib_directory.handle.close(); | |
| 59 | ||
| 60 | var global_cache_directory: Directory = .{ | |
| 61 | .handle = try std.fs.cwd().openDir(args[2], .{}), | |
| 62 | }; | |
| 63 | defer global_cache_directory.handle.close(); | |
| 64 | ||
| 65 | const color: std.zig.Color = .auto; | |
| 66 | const work_around_btrfs_bug = native_os == .linux and std.zig.EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); | |
| 67 | var opt_path_or_url: ?[]const u8 = null; | |
| 68 | var debug_hash: bool = false; | |
| 69 | var save: union(enum) { | |
| 70 | no, | |
| 71 | yes: ?[]const u8, | |
| 72 | exact: ?[]const u8, | |
| 73 | } = .no; | |
| 74 | ||
| 75 | { | |
| 76 | var i: usize = 3; | |
| 77 | while (i < args.len) : (i += 1) { | |
| 78 | const arg = args[i]; | |
| 79 | if (mem.startsWith(u8, arg, "-")) { | |
| 80 | if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { | |
| 81 | const stdout = std.io.getStdOut().writer(); | |
| 82 | try stdout.writeAll(usage); | |
| 83 | return process.cleanExit(); | |
| 84 | } else if (mem.eql(u8, arg, "--debug-hash")) { | |
| 85 | debug_hash = true; | |
| 86 | } else if (mem.eql(u8, arg, "--save")) { | |
| 87 | save = .{ .yes = null }; | |
| 88 | } else if (mem.startsWith(u8, arg, "--save=")) { | |
| 89 | save = .{ .yes = arg["--save=".len..] }; | |
| 90 | } else if (mem.eql(u8, arg, "--save-exact")) { | |
| 91 | save = .{ .exact = null }; | |
| 92 | } else if (mem.startsWith(u8, arg, "--save-exact=")) { | |
| 93 | save = .{ .exact = arg["--save-exact=".len..] }; | |
| 94 | } else { | |
| 95 | fatal("unrecognized parameter: '{s}'", .{arg}); | |
| 96 | } | |
| 97 | } else if (opt_path_or_url != null) { | |
| 98 | fatal("unexpected extra parameter: '{s}'", .{arg}); | |
| 99 | } else { | |
| 100 | opt_path_or_url = arg; | |
| 101 | } | |
| 102 | } | |
| 103 | } | |
| 104 | ||
| 105 | const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); | |
| 106 | ||
| 107 | var thread_pool: std.Thread.Pool = undefined; | |
| 108 | try thread_pool.init(.{ .allocator = gpa }); | |
| 109 | defer thread_pool.deinit(); | |
| 110 | ||
| 111 | var http_client: std.http.Client = .{ .allocator = gpa }; | |
| 112 | defer http_client.deinit(); | |
| 113 | ||
| 114 | try http_client.initDefaultProxies(arena); | |
| 115 | ||
| 116 | var root_prog_node = std.Progress.start(.{ | |
| 117 | .root_name = "Fetch", | |
| 118 | }); | |
| 119 | defer root_prog_node.end(); | |
| 120 | ||
| 121 | var job_queue: Package.Fetch.JobQueue = .{ | |
| 122 | .http_client = &http_client, | |
| 123 | .thread_pool = &thread_pool, | |
| 124 | .global_cache = global_cache_directory, | |
| 125 | .recursive = false, | |
| 126 | .read_only = false, | |
| 127 | .debug_hash = debug_hash, | |
| 128 | .work_around_btrfs_bug = work_around_btrfs_bug, | |
| 129 | }; | |
| 130 | defer job_queue.deinit(); | |
| 131 | ||
| 132 | var fetch: Package.Fetch = .{ | |
| 133 | .arena = std.heap.ArenaAllocator.init(gpa), | |
| 134 | .location = .{ .path_or_url = path_or_url }, | |
| 135 | .location_tok = 0, | |
| 136 | .hash_tok = .none, | |
| 137 | .name_tok = 0, | |
| 138 | .lazy_status = .eager, | |
| 139 | .parent_package_root = undefined, | |
| 140 | .parent_manifest_ast = null, | |
| 141 | .prog_node = root_prog_node, | |
| 142 | .job_queue = &job_queue, | |
| 143 | .omit_missing_hash_error = true, | |
| 144 | .allow_missing_paths_field = false, | |
| 145 | .allow_missing_fingerprint = true, | |
| 146 | .allow_name_string = true, | |
| 147 | .use_latest_commit = true, | |
| 148 | ||
| 149 | .package_root = undefined, | |
| 150 | .error_bundle = undefined, | |
| 151 | .manifest = null, | |
| 152 | .manifest_ast = undefined, | |
| 153 | .computed_hash = undefined, | |
| 154 | .has_build_zig = false, | |
| 155 | .oom_flag = false, | |
| 156 | .latest_commit = null, | |
| 157 | }; | |
| 158 | defer fetch.deinit(); | |
| 159 | ||
| 160 | fetch.run() catch |err| switch (err) { | |
| 161 | error.OutOfMemory => fatal("out of memory", .{}), | |
| 162 | error.FetchFailed => {}, // error bundle checked below | |
| 163 | }; | |
| 164 | ||
| 165 | if (fetch.error_bundle.root_list.items.len > 0) { | |
| 166 | var errors = try fetch.error_bundle.toOwnedBundle(""); | |
| 167 | errors.renderToStdErr(color.renderOptions()); | |
| 168 | process.exit(1); | |
| 169 | } | |
| 170 | ||
| 171 | const package_hash = fetch.computedPackageHash(); | |
| 172 | const package_hash_slice = package_hash.toSlice(); | |
| 173 | ||
| 174 | root_prog_node.end(); | |
| 175 | root_prog_node = .{ .index = .none }; | |
| 176 | ||
| 177 | const name = switch (save) { | |
| 178 | .no => { | |
| 179 | try std.io.getStdOut().writer().print("{s}\n", .{package_hash_slice}); | |
| 180 | return process.cleanExit(); | |
| 181 | }, | |
| 182 | .yes, .exact => |name| name: { | |
| 183 | if (name) |n| break :name n; | |
| 184 | const fetched_manifest = fetch.manifest orelse | |
| 185 | fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); | |
| 186 | break :name fetched_manifest.name; | |
| 187 | }, | |
| 188 | }; | |
| 189 | ||
| 190 | const cwd_path = try process.getCwdAlloc(arena); | |
| 191 | ||
| 192 | var build_root = try Package.findBuildRoot(arena, .{ | |
| 193 | .cwd_path = cwd_path, | |
| 194 | }); | |
| 195 | defer build_root.deinit(); | |
| 196 | ||
| 197 | // The name to use in case the manifest file needs to be created now. | |
| 198 | const init_root_name = std.fs.path.basename(build_root.directory.path orelse cwd_path); | |
| 199 | var manifest, var ast = try loadManifest(gpa, arena, zig_lib_directory, .{ | |
| 200 | .root_name = try Package.sanitizeExampleName(arena, init_root_name), | |
| 201 | .dir = build_root.directory.handle, | |
| 202 | .color = color, | |
| 203 | }); | |
| 204 | defer { | |
| 205 | manifest.deinit(gpa); | |
| 206 | ast.deinit(gpa); | |
| 207 | } | |
| 208 | ||
| 209 | var fixups: std.zig.Ast.Fixups = .{}; | |
| 210 | defer fixups.deinit(gpa); | |
| 211 | ||
| 212 | var saved_path_or_url = path_or_url; | |
| 213 | ||
| 214 | if (fetch.latest_commit) |latest_commit| resolved: { | |
| 215 | const latest_commit_hex = try std.fmt.allocPrint(arena, "{}", .{latest_commit}); | |
| 216 | ||
| 217 | var uri = try std.Uri.parse(path_or_url); | |
| 218 | ||
| 219 | if (uri.fragment) |fragment| { | |
| 220 | const target_ref = try fragment.toRawMaybeAlloc(arena); | |
| 221 | ||
| 222 | // the refspec may already be fully resolved | |
| 223 | if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; | |
| 224 | ||
| 225 | std.log.info("resolved ref '{s}' to commit {s}", .{ target_ref, latest_commit_hex }); | |
| 226 | ||
| 227 | // include the original refspec in a query parameter, could be used to check for updates | |
| 228 | uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={%}", .{fragment}) }; | |
| 229 | } else { | |
| 230 | std.log.info("resolved to commit {s}", .{latest_commit_hex}); | |
| 231 | } | |
| 232 | ||
| 233 | // replace the refspec with the resolved commit SHA | |
| 234 | uri.fragment = .{ .raw = latest_commit_hex }; | |
| 235 | ||
| 236 | switch (save) { | |
| 237 | .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{}", .{uri}), | |
| 238 | .no, .exact => {}, // keep the original URL | |
| 239 | } | |
| 240 | } | |
| 241 | ||
| 242 | const new_node_init = try std.fmt.allocPrint(arena, | |
| 243 | \\.{{ | |
| 244 | \\ .url = "{}", | |
| 245 | \\ .hash = "{}", | |
| 246 | \\ }} | |
| 247 | , .{ | |
| 248 | std.zig.fmtEscapes(saved_path_or_url), | |
| 249 | std.zig.fmtEscapes(package_hash_slice), | |
| 250 | }); | |
| 251 | ||
| 252 | const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{ | |
| 253 | std.zig.fmtId(name), new_node_init, | |
| 254 | }); | |
| 255 | ||
| 256 | const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ | |
| 257 | new_node_text, | |
| 258 | }); | |
| 259 | ||
| 260 | const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ | |
| 261 | dependencies_init, | |
| 262 | }); | |
| 263 | ||
| 264 | if (manifest.dependencies.get(name)) |dep| { | |
| 265 | if (dep.hash) |h| { | |
| 266 | switch (dep.location) { | |
| 267 | .url => |u| { | |
| 268 | if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { | |
| 269 | std.log.info("existing dependency named '{s}' is up-to-date", .{name}); | |
| 270 | process.exit(0); | |
| 271 | } | |
| 272 | }, | |
| 273 | .path => {}, | |
| 274 | } | |
| 275 | } | |
| 276 | ||
| 277 | const location_replace = try std.fmt.allocPrint( | |
| 278 | arena, | |
| 279 | "\"{}\"", | |
| 280 | .{std.zig.fmtEscapes(saved_path_or_url)}, | |
| 281 | ); | |
| 282 | const hash_replace = try std.fmt.allocPrint( | |
| 283 | arena, | |
| 284 | "\"{}\"", | |
| 285 | .{std.zig.fmtEscapes(package_hash_slice)}, | |
| 286 | ); | |
| 287 | ||
| 288 | std.log.warn("overwriting existing dependency named '{s}'", .{name}); | |
| 289 | try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); | |
| 290 | if (dep.hash_node.unwrap()) |hash_node| { | |
| 291 | try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); | |
| 292 | } else { | |
| 293 | // https://github.com/ziglang/zig/issues/21690 | |
| 294 | } | |
| 295 | } else if (manifest.dependencies.count() > 0) { | |
| 296 | // Add fixup for adding another dependency. | |
| 297 | const deps = manifest.dependencies.values(); | |
| 298 | const last_dep_node = deps[deps.len - 1].node; | |
| 299 | try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); | |
| 300 | } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { | |
| 301 | // Add fixup for replacing the entire dependencies struct. | |
| 302 | try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); | |
| 303 | } else { | |
| 304 | // Add fixup for adding dependencies struct. | |
| 305 | try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); | |
| 306 | } | |
| 307 | ||
| 308 | var rendered = std.ArrayList(u8).init(gpa); | |
| 309 | defer rendered.deinit(); | |
| 310 | try ast.renderToArrayList(&rendered, fixups); | |
| 311 | ||
| 312 | build_root.directory.handle.writeFile(.{ .sub_path = Package.Manifest.basename, .data = rendered.items }) catch |err| { | |
| 313 | fatal("unable to write {s} file: {s}", .{ Package.Manifest.basename, @errorName(err) }); | |
| 314 | }; | |
| 315 | ||
| 316 | return process.cleanExit(); | |
| 317 | } | |
| 318 | ||
| 319 | const LoadManifestOptions = struct { | |
| 320 | root_name: []const u8, | |
| 321 | dir: fs.Dir, | |
| 322 | color: std.zig.Color, | |
| 323 | }; | |
| 324 | ||
| 325 | fn loadManifest( | |
| 326 | gpa: Allocator, | |
| 327 | arena: Allocator, | |
| 328 | zig_lib_directory: Directory, | |
| 329 | options: LoadManifestOptions, | |
| 330 | ) !struct { Package.Manifest, std.zig.Ast } { | |
| 331 | const manifest_bytes = while (true) { | |
| 332 | break options.dir.readFileAllocOptions( | |
| 333 | arena, | |
| 334 | Package.Manifest.basename, | |
| 335 | Package.Manifest.max_bytes, | |
| 336 | null, | |
| 337 | 1, | |
| 338 | 0, | |
| 339 | ) catch |err| switch (err) { | |
| 340 | error.FileNotFound => { | |
| 341 | const fingerprint: Package.Fingerprint = .generate(options.root_name); | |
| 342 | var templates = Package.Templates.find(gpa, zig_lib_directory); | |
| 343 | defer templates.deinit(gpa); | |
| 344 | templates.write(arena, options.dir, options.root_name, Package.Manifest.basename, fingerprint) catch |e| { | |
| 345 | fatal("unable to write {s}: {s}", .{ | |
| 346 | Package.Manifest.basename, @errorName(e), | |
| 347 | }); | |
| 348 | }; | |
| 349 | continue; | |
| 350 | }, | |
| 351 | else => |e| fatal("unable to load {s}: {s}", .{ | |
| 352 | Package.Manifest.basename, @errorName(e), | |
| 353 | }), | |
| 354 | }; | |
| 355 | }; | |
| 356 | var ast = try std.zig.Ast.parse(gpa, manifest_bytes, .zon); | |
| 357 | errdefer ast.deinit(gpa); | |
| 358 | ||
| 359 | if (ast.errors.len > 0) { | |
| 360 | try std.zig.printAstErrorsToStderr(gpa, ast, Package.Manifest.basename, options.color); | |
| 361 | process.exit(2); | |
| 362 | } | |
| 363 | ||
| 364 | var manifest = try Package.Manifest.parse(gpa, ast, .{}); | |
| 365 | errdefer manifest.deinit(gpa); | |
| 366 | ||
| 367 | if (manifest.errors.len > 0) { | |
| 368 | var wip_errors: std.zig.ErrorBundle.Wip = undefined; | |
| 369 | try wip_errors.init(gpa); | |
| 370 | defer wip_errors.deinit(); | |
| 371 | ||
| 372 | const src_path = try wip_errors.addString(Package.Manifest.basename); | |
| 373 | try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); | |
| 374 | ||
| 375 | var error_bundle = try wip_errors.toOwnedBundle(""); | |
| 376 | defer error_bundle.deinit(gpa); | |
| 377 | error_bundle.renderToStdErr(options.color.renderOptions()); | |
| 378 | ||
| 379 | process.exit(2); | |
| 380 | } | |
| 381 | return .{ manifest, ast }; | |
| 382 | } |
lib/std/Build.zig+41-128
| ... | ... | @@ -31,35 +31,17 @@ allocator: Allocator, |
| 31 | 31 | user_input_options: UserInputOptionsMap, |
| 32 | 32 | available_options_map: AvailableOptionsMap, |
| 33 | 33 | available_options_list: ArrayList(AvailableOption), |
| 34 | verbose: bool, | |
| 35 | verbose_link: bool, | |
| 36 | verbose_cc: bool, | |
| 37 | verbose_air: bool, | |
| 38 | verbose_llvm_ir: ?[]const u8, | |
| 39 | verbose_llvm_bc: ?[]const u8, | |
| 40 | verbose_cimport: bool, | |
| 41 | verbose_llvm_cpu_features: bool, | |
| 42 | reference_trace: ?u32 = null, | |
| 43 | 34 | invalid_user_input: bool, |
| 44 | 35 | default_step: *Step, |
| 45 | 36 | top_level_steps: std.StringArrayHashMapUnmanaged(*TopLevelStep), |
| 46 | install_prefix: []const u8, | |
| 47 | dest_dir: ?[]const u8, | |
| 48 | lib_dir: []const u8, | |
| 49 | exe_dir: []const u8, | |
| 50 | h_dir: []const u8, | |
| 51 | install_path: []const u8, | |
| 52 | sysroot: ?[]const u8 = null, | |
| 53 | search_prefixes: std.ArrayListUnmanaged([]const u8), | |
| 54 | libc_file: ?[]const u8 = null, | |
| 37 | install_prefix: Cache.Path, | |
| 38 | install_lib_path: Cache.Path, | |
| 39 | install_exe_path: Cache.Path, | |
| 40 | install_include_path: Cache.Path, | |
| 55 | 41 | /// Path to the directory containing build.zig. |
| 56 | 42 | build_root: Cache.Directory, |
| 57 | 43 | cache_root: Cache.Directory, |
| 58 | 44 | pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, |
| 59 | args: ?[]const []const u8 = null, | |
| 60 | debug_log_scopes: []const []const u8 = &.{}, | |
| 61 | debug_compile_errors: bool = false, | |
| 62 | debug_pkg_config: bool = false, | |
| 63 | 45 | /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, |
| 64 | 46 | /// in particular at `Step` creation. |
| 65 | 47 | /// Set to 0 to disable stack collection. |
| ... | ... | @@ -75,11 +57,6 @@ enable_rosetta: bool = false, |
| 75 | 57 | enable_wasmtime: bool = false, |
| 76 | 58 | /// Use system Wine installation to run cross compiled Windows build artifacts. |
| 77 | 59 | enable_wine: bool = false, |
| 78 | /// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc, | |
| 79 | /// this will be the directory $glibc-build-dir/install/glibcs | |
| 80 | /// Given the example of the aarch64 target, this is the directory | |
| 81 | /// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. | |
| 82 | glibc_runtimes_dir: ?[]const u8 = null, | |
| 83 | 60 | |
| 84 | 61 | dep_prefix: []const u8 = "", |
| 85 | 62 | |
| ... | ... | @@ -92,8 +69,6 @@ pkg_hash: []const u8, |
| 92 | 69 | /// A mapping from dependency names to package hashes. |
| 93 | 70 | available_deps: AvailableDeps, |
| 94 | 71 | |
| 95 | release_mode: ReleaseMode, | |
| 96 | ||
| 97 | 72 | pub const ReleaseMode = enum { |
| 98 | 73 | off, |
| 99 | 74 | any, |
| ... | ... | @@ -107,7 +82,7 @@ pub const ReleaseMode = enum { |
| 107 | 82 | pub const Graph = struct { |
| 108 | 83 | arena: Allocator, |
| 109 | 84 | system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .empty, |
| 110 | system_package_mode: bool = false, | |
| 85 | system_package_mode: ?Cache.Directory = null, | |
| 111 | 86 | debug_compiler_runtime_libs: bool = false, |
| 112 | 87 | cache: Cache, |
| 113 | 88 | zig_exe: [:0]const u8, |
| ... | ... | @@ -121,6 +96,31 @@ pub const Graph = struct { |
| 121 | 96 | random_seed: u32 = 0, |
| 122 | 97 | dependency_cache: InitializedDepMap = .empty, |
| 123 | 98 | allow_so_scripts: ?bool = null, |
| 99 | ||
| 100 | release_mode: ReleaseMode, | |
| 101 | sysroot: ?[]const u8 = null, | |
| 102 | search_prefixes: std.ArrayListUnmanaged([]const u8), | |
| 103 | libc_file: ?[]const u8 = null, | |
| 104 | debug_compile_errors: bool = false, | |
| 105 | /// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc, | |
| 106 | /// this will be the directory $glibc-build-dir/install/glibcs | |
| 107 | /// Given the example of the aarch64 target, this is the directory | |
| 108 | /// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. | |
| 109 | glibc_runtimes_dir: ?[]const u8 = null, | |
| 110 | verbose: bool, | |
| 111 | verbose_link: bool, | |
| 112 | verbose_cc: bool, | |
| 113 | verbose_air: bool, | |
| 114 | verbose_llvm_ir: ?[]const u8, | |
| 115 | verbose_llvm_bc: ?[]const u8, | |
| 116 | verbose_cimport: bool, | |
| 117 | verbose_llvm_cpu_features: bool, | |
| 118 | reference_trace: ?u32 = null, | |
| 119 | debug_log_scopes: []const []const u8 = &.{}, | |
| 120 | ||
| 121 | pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { | |
| 122 | b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); | |
| 123 | } | |
| 124 | 124 | }; |
| 125 | 125 | |
| 126 | 126 | const AvailableDeps = []const struct { []const u8, []const u8 }; |
| ... | ... | @@ -239,10 +239,10 @@ const TopLevelStep = struct { |
| 239 | 239 | description: []const u8, |
| 240 | 240 | }; |
| 241 | 241 | |
| 242 | pub const DirList = struct { | |
| 243 | lib_dir: ?[]const u8 = null, | |
| 244 | exe_dir: ?[]const u8 = null, | |
| 245 | include_dir: ?[]const u8 = null, | |
| 242 | pub const InstallPaths = struct { | |
| 243 | lib_path: ?Cache.Path = null, | |
| 244 | exe_path: ?Cache.Path = null, | |
| 245 | include_path: ?Cache.Path = null, | |
| 246 | 246 | }; |
| 247 | 247 | |
| 248 | 248 | pub fn create( |
| ... | ... | @@ -259,13 +259,6 @@ pub fn create( |
| 259 | 259 | .build_root = build_root, |
| 260 | 260 | .cache_root = cache_root, |
| 261 | 261 | .verbose = false, |
| 262 | .verbose_link = false, | |
| 263 | .verbose_cc = false, | |
| 264 | .verbose_air = false, | |
| 265 | .verbose_llvm_ir = null, | |
| 266 | .verbose_llvm_bc = null, | |
| 267 | .verbose_cimport = false, | |
| 268 | .verbose_llvm_cpu_features = false, | |
| 269 | 262 | .invalid_user_input = false, |
| 270 | 263 | .allocator = arena, |
| 271 | 264 | .user_input_options = UserInputOptionsMap.init(arena), |
| ... | ... | @@ -273,12 +266,10 @@ pub fn create( |
| 273 | 266 | .available_options_list = ArrayList(AvailableOption).init(arena), |
| 274 | 267 | .top_level_steps = .{}, |
| 275 | 268 | .default_step = undefined, |
| 276 | .search_prefixes = .{}, | |
| 277 | 269 | .install_prefix = undefined, |
| 278 | 270 | .lib_dir = undefined, |
| 279 | 271 | .exe_dir = undefined, |
| 280 | 272 | .h_dir = undefined, |
| 281 | .dest_dir = graph.env_map.get("DESTDIR"), | |
| 282 | 273 | .install_tls = .{ |
| 283 | 274 | .step = Step.init(.{ |
| 284 | 275 | .id = TopLevelStep.base_id, |
| ... | ... | @@ -297,7 +288,6 @@ pub fn create( |
| 297 | 288 | .description = "Remove build artifacts from prefix path", |
| 298 | 289 | }, |
| 299 | 290 | .install_path = undefined, |
| 300 | .args = null, | |
| 301 | 291 | .modules = .init(arena), |
| 302 | 292 | .named_writefiles = .init(arena), |
| 303 | 293 | .named_lazy_paths = .init(arena), |
| ... | ... | @@ -358,37 +348,22 @@ fn createChildOnly( |
| 358 | 348 | .available_options_map = AvailableOptionsMap.init(allocator), |
| 359 | 349 | .available_options_list = ArrayList(AvailableOption).init(allocator), |
| 360 | 350 | .verbose = parent.verbose, |
| 361 | .verbose_link = parent.verbose_link, | |
| 362 | .verbose_cc = parent.verbose_cc, | |
| 363 | .verbose_air = parent.verbose_air, | |
| 364 | .verbose_llvm_ir = parent.verbose_llvm_ir, | |
| 365 | .verbose_llvm_bc = parent.verbose_llvm_bc, | |
| 366 | .verbose_cimport = parent.verbose_cimport, | |
| 367 | .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, | |
| 368 | .reference_trace = parent.reference_trace, | |
| 369 | 351 | .invalid_user_input = false, |
| 370 | 352 | .default_step = undefined, |
| 371 | 353 | .top_level_steps = .{}, |
| 372 | 354 | .install_prefix = undefined, |
| 373 | .dest_dir = parent.dest_dir, | |
| 374 | 355 | .lib_dir = parent.lib_dir, |
| 375 | 356 | .exe_dir = parent.exe_dir, |
| 376 | 357 | .h_dir = parent.h_dir, |
| 377 | 358 | .install_path = parent.install_path, |
| 378 | 359 | .sysroot = parent.sysroot, |
| 379 | .search_prefixes = parent.search_prefixes, | |
| 380 | .libc_file = parent.libc_file, | |
| 381 | 360 | .build_root = build_root, |
| 382 | 361 | .cache_root = parent.cache_root, |
| 383 | .debug_log_scopes = parent.debug_log_scopes, | |
| 384 | .debug_compile_errors = parent.debug_compile_errors, | |
| 385 | .debug_pkg_config = parent.debug_pkg_config, | |
| 386 | 362 | .enable_darling = parent.enable_darling, |
| 387 | 363 | .enable_qemu = parent.enable_qemu, |
| 388 | 364 | .enable_rosetta = parent.enable_rosetta, |
| 389 | 365 | .enable_wasmtime = parent.enable_wasmtime, |
| 390 | 366 | .enable_wine = parent.enable_wine, |
| 391 | .glibc_runtimes_dir = parent.glibc_runtimes_dir, | |
| 392 | 367 | .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), |
| 393 | 368 | .modules = .init(allocator), |
| 394 | 369 | .named_writefiles = .init(allocator), |
| ... | ... | @@ -638,42 +613,16 @@ fn determineAndApplyInstallPrefix(b: *Build) error{OutOfMemory}!void { |
| 638 | 613 | |
| 639 | 614 | const digest = hash.final(); |
| 640 | 615 | const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); |
| 641 | b.resolveInstallPrefix(install_prefix, .{}); | |
| 616 | try b.resolveInstallPrefix(install_prefix, .{}); | |
| 642 | 617 | } |
| 643 | 618 | |
| 644 | /// This function is intended to be called by lib/build_runner.zig, not a build.zig file. | |
| 645 | pub fn resolveInstallPrefix(b: *Build, install_prefix: ?[]const u8, dir_list: DirList) void { | |
| 646 | if (b.dest_dir) |dest_dir| { | |
| 647 | b.install_prefix = install_prefix orelse "/usr"; | |
| 648 | b.install_path = b.pathJoin(&.{ dest_dir, b.install_prefix }); | |
| 649 | } else { | |
| 650 | b.install_prefix = install_prefix orelse | |
| 651 | (b.build_root.join(b.allocator, &.{"zig-out"}) catch @panic("unhandled error")); | |
| 652 | b.install_path = b.install_prefix; | |
| 653 | } | |
| 654 | ||
| 655 | var lib_list = [_][]const u8{ b.install_path, "lib" }; | |
| 656 | var exe_list = [_][]const u8{ b.install_path, "bin" }; | |
| 657 | var h_list = [_][]const u8{ b.install_path, "include" }; | |
| 619 | fn resolveInstallPrefix(b: *Build, install_prefix: Cache.Path, paths: InstallPaths) !void { | |
| 620 | const arena = b.allocator; | |
| 658 | 621 | |
| 659 | if (dir_list.lib_dir) |dir| { | |
| 660 | if (fs.path.isAbsolute(dir)) lib_list[0] = b.dest_dir orelse ""; | |
| 661 | lib_list[1] = dir; | |
| 662 | } | |
| 663 | ||
| 664 | if (dir_list.exe_dir) |dir| { | |
| 665 | if (fs.path.isAbsolute(dir)) exe_list[0] = b.dest_dir orelse ""; | |
| 666 | exe_list[1] = dir; | |
| 667 | } | |
| 668 | ||
| 669 | if (dir_list.include_dir) |dir| { | |
| 670 | if (fs.path.isAbsolute(dir)) h_list[0] = b.dest_dir orelse ""; | |
| 671 | h_list[1] = dir; | |
| 672 | } | |
| 673 | ||
| 674 | b.lib_dir = b.pathJoin(&lib_list); | |
| 675 | b.exe_dir = b.pathJoin(&exe_list); | |
| 676 | b.h_dir = b.pathJoin(&h_list); | |
| 622 | b.install_prefix = install_prefix; | |
| 623 | b.install_lib_path = paths.lib_path orelse try install_prefix.join(arena, "lib"); | |
| 624 | b.install_exe_path = paths.exe_path orelse try install_prefix.join(arena, "bin"); | |
| 625 | b.install_include_path = paths.include_path orelse try install_prefix.join(arena, "include"); | |
| 677 | 626 | } |
| 678 | 627 | |
| 679 | 628 | /// Create a set of key-value pairs that can be converted into a Zig source |
| ... | ... | @@ -1990,38 +1939,6 @@ fn tryFindProgram(b: *Build, full_path: []const u8) ?[]const u8 { |
| 1990 | 1939 | return null; |
| 1991 | 1940 | } |
| 1992 | 1941 | |
| 1993 | pub fn findProgram(b: *Build, names: []const []const u8, paths: []const []const u8) error{FileNotFound}![]const u8 { | |
| 1994 | // TODO report error for ambiguous situations | |
| 1995 | for (b.search_prefixes.items) |search_prefix| { | |
| 1996 | for (names) |name| { | |
| 1997 | if (fs.path.isAbsolute(name)) { | |
| 1998 | return name; | |
| 1999 | } | |
| 2000 | return tryFindProgram(b, b.pathJoin(&.{ search_prefix, "bin", name })) orelse continue; | |
| 2001 | } | |
| 2002 | } | |
| 2003 | if (b.graph.env_map.get("PATH")) |PATH| { | |
| 2004 | for (names) |name| { | |
| 2005 | if (fs.path.isAbsolute(name)) { | |
| 2006 | return name; | |
| 2007 | } | |
| 2008 | var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter); | |
| 2009 | while (it.next()) |p| { | |
| 2010 | return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; | |
| 2011 | } | |
| 2012 | } | |
| 2013 | } | |
| 2014 | for (names) |name| { | |
| 2015 | if (fs.path.isAbsolute(name)) { | |
| 2016 | return name; | |
| 2017 | } | |
| 2018 | for (paths) |p| { | |
| 2019 | return tryFindProgram(b, b.pathJoin(&.{ p, name })) orelse continue; | |
| 2020 | } | |
| 2021 | } | |
| 2022 | return error.FileNotFound; | |
| 2023 | } | |
| 2024 | ||
| 2025 | 1942 | pub fn runAllowFail( |
| 2026 | 1943 | b: *Build, |
| 2027 | 1944 | argv: []const []const u8, |
| ... | ... | @@ -2085,10 +2002,6 @@ pub fn run(b: *Build, argv: []const []const u8) []u8 { |
| 2085 | 2002 | }; |
| 2086 | 2003 | } |
| 2087 | 2004 | |
| 2088 | pub fn addSearchPrefix(b: *Build, search_prefix: []const u8) void { | |
| 2089 | b.search_prefixes.append(b.allocator, b.dupePath(search_prefix)) catch @panic("OOM"); | |
| 2090 | } | |
| 2091 | ||
| 2092 | 2005 | pub fn getInstallPath(b: *Build, dir: InstallDir, dest_rel_path: []const u8) []const u8 { |
| 2093 | 2006 | assert(!fs.path.isAbsolute(dest_rel_path)); // Install paths must be relative to the prefix |
| 2094 | 2007 | const base_dir = switch (dir) { |
lib/std/zig.zig+1
| ... | ... | @@ -25,6 +25,7 @@ pub const WindowsSdk = @import("zig/WindowsSdk.zig"); |
| 25 | 25 | pub const LibCDirs = @import("zig/LibCDirs.zig"); |
| 26 | 26 | pub const target = @import("zig/target.zig"); |
| 27 | 27 | pub const llvm = @import("zig/llvm.zig"); |
| 28 | pub const Package = @import("zig/Package.zig"); | |
| 28 | 29 | |
| 29 | 30 | // Character literal parsing |
| 30 | 31 | pub const ParsedCharLiteral = string_literal.ParsedCharLiteral; |
lib/std/zig/Package.zig created+307
| ... | ... | @@ -0,0 +1,307 @@ |
| 1 | const std = @import("std"); | |
| 2 | const assert = std.debug.assert; | |
| 3 | const Allocator = std.mem.Allocator; | |
| 4 | ||
| 5 | pub const Fetch = @import("Package/Fetch.zig"); | |
| 6 | pub const build_zig_basename = "build.zig"; | |
| 7 | pub const Manifest = @import("Package/Manifest.zig"); | |
| 8 | ||
| 9 | pub const multihash_len = 1 + 1 + Hash.Algo.digest_length; | |
| 10 | pub const multihash_hex_digest_len = 2 * multihash_len; | |
| 11 | pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; | |
| 12 | ||
| 13 | pub const Fingerprint = packed struct(u64) { | |
| 14 | id: u32, | |
| 15 | checksum: u32, | |
| 16 | ||
| 17 | pub fn generate(name: []const u8) Fingerprint { | |
| 18 | return .{ | |
| 19 | .id = std.crypto.random.intRangeLessThan(u32, 1, 0xffffffff), | |
| 20 | .checksum = std.hash.Crc32.hash(name), | |
| 21 | }; | |
| 22 | } | |
| 23 | ||
| 24 | pub fn validate(n: Fingerprint, name: []const u8) bool { | |
| 25 | switch (n.id) { | |
| 26 | 0x00000000, 0xffffffff => return false, | |
| 27 | else => return std.hash.Crc32.hash(name) == n.checksum, | |
| 28 | } | |
| 29 | } | |
| 30 | ||
| 31 | pub fn int(n: Fingerprint) u64 { | |
| 32 | return @bitCast(n); | |
| 33 | } | |
| 34 | }; | |
| 35 | ||
| 36 | /// A user-readable, file system safe hash that identifies an exact package | |
| 37 | /// snapshot, including file contents. | |
| 38 | /// | |
| 39 | /// The hash is not only to prevent collisions but must resist attacks where | |
| 40 | /// the adversary fully controls the contents being hashed. Thus, it contains | |
| 41 | /// a full SHA-256 digest. | |
| 42 | /// | |
| 43 | /// This data structure can be used to store the legacy hash format too. Legacy | |
| 44 | /// hash format is scheduled to be removed after 0.14.0 is tagged. | |
| 45 | /// | |
| 46 | /// There's also a third way this structure is used. When using path rather than | |
| 47 | /// hash, a unique hash is still needed, so one is computed based on the path. | |
| 48 | pub const Hash = struct { | |
| 49 | /// Maximum size of a package hash. Unused bytes at the end are | |
| 50 | /// filled with zeroes. | |
| 51 | bytes: [max_len]u8, | |
| 52 | ||
| 53 | pub const Algo = std.crypto.hash.sha2.Sha256; | |
| 54 | pub const Digest = [Algo.digest_length]u8; | |
| 55 | ||
| 56 | /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" | |
| 57 | pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; | |
| 58 | ||
| 59 | pub fn fromSlice(s: []const u8) Hash { | |
| 60 | assert(s.len <= max_len); | |
| 61 | var result: Hash = undefined; | |
| 62 | @memcpy(result.bytes[0..s.len], s); | |
| 63 | @memset(result.bytes[s.len..], 0); | |
| 64 | return result; | |
| 65 | } | |
| 66 | ||
| 67 | pub fn toSlice(ph: *const Hash) []const u8 { | |
| 68 | var end: usize = ph.bytes.len; | |
| 69 | while (true) { | |
| 70 | end -= 1; | |
| 71 | if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; | |
| 72 | } | |
| 73 | } | |
| 74 | ||
| 75 | pub fn eql(a: *const Hash, b: *const Hash) bool { | |
| 76 | return std.mem.eql(u8, &a.bytes, &b.bytes); | |
| 77 | } | |
| 78 | ||
| 79 | /// Distinguishes whether the legacy multihash format is being stored here. | |
| 80 | pub fn isOld(h: *const Hash) bool { | |
| 81 | if (h.bytes.len < 2) return false; | |
| 82 | const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false; | |
| 83 | if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false; | |
| 84 | if (h.toSlice().len != multihash_hex_digest_len) return false; | |
| 85 | return std.mem.indexOfScalar(u8, &h.bytes, '-') == null; | |
| 86 | } | |
| 87 | ||
| 88 | test isOld { | |
| 89 | const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7"); | |
| 90 | try std.testing.expect(h.isOld()); | |
| 91 | } | |
| 92 | ||
| 93 | /// Produces "$name-$semver-$hashplus". | |
| 94 | /// * name is the name field from build.zig.zon, asserted to be at most 32 | |
| 95 | /// bytes and assumed be a valid zig identifier | |
| 96 | /// * semver is the version field from build.zig.zon, asserted to be at | |
| 97 | /// most 32 bytes | |
| 98 | /// * hashplus is the following 33-byte array, base64 encoded using -_ to make | |
| 99 | /// it filesystem safe: | |
| 100 | /// - (4 bytes) LE u32 Package ID | |
| 101 | /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated | |
| 102 | /// - (25 bytes) truncated SHA-256 digest of hashed files of the package | |
| 103 | pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { | |
| 104 | assert(name.len <= 32); | |
| 105 | assert(ver.len <= 32); | |
| 106 | var result: Hash = undefined; | |
| 107 | var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes); | |
| 108 | buf.appendSliceAssumeCapacity(name); | |
| 109 | buf.appendAssumeCapacity('-'); | |
| 110 | buf.appendSliceAssumeCapacity(ver); | |
| 111 | buf.appendAssumeCapacity('-'); | |
| 112 | var hashplus: [33]u8 = undefined; | |
| 113 | std.mem.writeInt(u32, hashplus[0..4], id, .little); | |
| 114 | std.mem.writeInt(u32, hashplus[4..8], size, .little); | |
| 115 | hashplus[8..].* = digest[0..25].*; | |
| 116 | _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); | |
| 117 | @memset(buf.unusedCapacitySlice(), 0); | |
| 118 | return result; | |
| 119 | } | |
| 120 | ||
| 121 | /// Produces a unique hash based on the path provided. The result should | |
| 122 | /// not be user-visible. | |
| 123 | pub fn initPath(sub_path: []const u8, is_global: bool) Hash { | |
| 124 | var result: Hash = .{ .bytes = @splat(0) }; | |
| 125 | var i: usize = 0; | |
| 126 | if (is_global) { | |
| 127 | result.bytes[0] = '/'; | |
| 128 | i += 1; | |
| 129 | } | |
| 130 | if (i + sub_path.len <= result.bytes.len) { | |
| 131 | @memcpy(result.bytes[i..][0..sub_path.len], sub_path); | |
| 132 | return result; | |
| 133 | } | |
| 134 | var bin_digest: [Algo.digest_length]u8 = undefined; | |
| 135 | Algo.hash(sub_path, &bin_digest, .{}); | |
| 136 | _ = std.fmt.bufPrint(result.bytes[i..], "{}", .{std.fmt.fmtSliceHexLower(&bin_digest)}) catch unreachable; | |
| 137 | return result; | |
| 138 | } | |
| 139 | }; | |
| 140 | ||
| 141 | pub const MultihashFunction = enum(u16) { | |
| 142 | identity = 0x00, | |
| 143 | sha1 = 0x11, | |
| 144 | @"sha2-256" = 0x12, | |
| 145 | @"sha2-512" = 0x13, | |
| 146 | @"sha3-512" = 0x14, | |
| 147 | @"sha3-384" = 0x15, | |
| 148 | @"sha3-256" = 0x16, | |
| 149 | @"sha3-224" = 0x17, | |
| 150 | @"sha2-384" = 0x20, | |
| 151 | @"sha2-256-trunc254-padded" = 0x1012, | |
| 152 | @"sha2-224" = 0x1013, | |
| 153 | @"sha2-512-224" = 0x1014, | |
| 154 | @"sha2-512-256" = 0x1015, | |
| 155 | @"blake2b-256" = 0xb220, | |
| 156 | _, | |
| 157 | }; | |
| 158 | ||
| 159 | pub const multihash_function: MultihashFunction = switch (Hash.Algo) { | |
| 160 | std.crypto.hash.sha2.Sha256 => .@"sha2-256", | |
| 161 | else => unreachable, | |
| 162 | }; | |
| 163 | ||
| 164 | pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest { | |
| 165 | const hex_charset = std.fmt.hex_charset; | |
| 166 | ||
| 167 | var result: MultiHashHexDigest = undefined; | |
| 168 | ||
| 169 | result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; | |
| 170 | result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; | |
| 171 | ||
| 172 | result[2] = hex_charset[Hash.Algo.digest_length >> 4]; | |
| 173 | result[3] = hex_charset[Hash.Algo.digest_length & 15]; | |
| 174 | ||
| 175 | for (digest, 0..) |byte, i| { | |
| 176 | result[4 + i * 2] = hex_charset[byte >> 4]; | |
| 177 | result[5 + i * 2] = hex_charset[byte & 15]; | |
| 178 | } | |
| 179 | return result; | |
| 180 | } | |
| 181 | ||
| 182 | comptime { | |
| 183 | // We avoid unnecessary uleb128 code in hexDigest by asserting here the | |
| 184 | // values are small enough to be contained in the one-byte encoding. | |
| 185 | assert(@intFromEnum(multihash_function) < 127); | |
| 186 | assert(Hash.Algo.digest_length < 127); | |
| 187 | } | |
| 188 | ||
| 189 | test Hash { | |
| 190 | const example_digest: Hash.Digest = .{ | |
| 191 | 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, | |
| 192 | 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, | |
| 193 | }; | |
| 194 | const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); | |
| 195 | try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); | |
| 196 | } | |
| 197 | ||
| 198 | pub fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 { | |
| 199 | var result: std.ArrayListUnmanaged(u8) = .empty; | |
| 200 | for (bytes, 0..) |byte, i| switch (byte) { | |
| 201 | '0'...'9' => { | |
| 202 | if (i == 0) try result.append(arena, '_'); | |
| 203 | try result.append(arena, byte); | |
| 204 | }, | |
| 205 | '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte), | |
| 206 | '-', '.', ' ' => try result.append(arena, '_'), | |
| 207 | else => continue, | |
| 208 | }; | |
| 209 | if (!std.zig.isValidId(result.items)) return "foo"; | |
| 210 | if (result.items.len > Manifest.max_name_len) | |
| 211 | result.shrinkRetainingCapacity(Manifest.max_name_len); | |
| 212 | ||
| 213 | return result.toOwnedSlice(arena); | |
| 214 | } | |
| 215 | ||
| 216 | test sanitizeExampleName { | |
| 217 | var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); | |
| 218 | defer arena_instance.deinit(); | |
| 219 | const arena = arena_instance.allocator(); | |
| 220 | ||
| 221 | try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+")); | |
| 222 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "")); | |
| 223 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!")); | |
| 224 | try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a")); | |
| 225 | try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!")); | |
| 226 | try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234")); | |
| 227 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error")); | |
| 228 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test")); | |
| 229 | try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests")); | |
| 230 | try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); | |
| 231 | } | |
| 232 | ||
| 233 | pub const BuildRoot = struct { | |
| 234 | directory: std.Build.Cache.Directory, | |
| 235 | build_zig_basename: []const u8, | |
| 236 | cleanup_build_dir: ?std.fs.Dir, | |
| 237 | ||
| 238 | fn deinit(br: *BuildRoot) void { | |
| 239 | if (br.cleanup_build_dir) |*dir| dir.close(); | |
| 240 | br.* = undefined; | |
| 241 | } | |
| 242 | }; | |
| 243 | ||
| 244 | pub const FindBuildRootOptions = struct { | |
| 245 | build_file: ?[]const u8 = null, | |
| 246 | cwd_path: ?[]const u8 = null, | |
| 247 | }; | |
| 248 | ||
| 249 | pub fn findBuildRoot(arena: Allocator, options: FindBuildRootOptions) !BuildRoot { | |
| 250 | const cwd_path = options.cwd_path orelse try std.process.getCwdAlloc(arena); | |
| 251 | const basename = if (options.build_file) |bf| std.fs.path.basename(bf) else build_zig_basename; | |
| 252 | ||
| 253 | if (options.build_file) |bf| { | |
| 254 | if (std.fs.path.dirname(bf)) |dirname| { | |
| 255 | const dir = std.fs.cwd().openDir(dirname, .{}) catch |err| { | |
| 256 | std.process.fatal("unable to open directory to build file from argument 'build-file', '{s}': {s}", .{ dirname, @errorName(err) }); | |
| 257 | }; | |
| 258 | return .{ | |
| 259 | .build_zig_basename = basename, | |
| 260 | .directory = .{ .path = dirname, .handle = dir }, | |
| 261 | .cleanup_build_dir = dir, | |
| 262 | }; | |
| 263 | } | |
| 264 | ||
| 265 | return .{ | |
| 266 | .build_zig_basename = basename, | |
| 267 | .directory = .{ .path = null, .handle = std.fs.cwd() }, | |
| 268 | .cleanup_build_dir = null, | |
| 269 | }; | |
| 270 | } | |
| 271 | // Search up parent directories until we find build.zig. | |
| 272 | var dirname: []const u8 = cwd_path; | |
| 273 | while (true) { | |
| 274 | const joined_path = try std.fs.path.join(arena, &[_][]const u8{ dirname, basename }); | |
| 275 | if (std.fs.cwd().access(joined_path, .{})) |_| { | |
| 276 | const dir = std.fs.cwd().openDir(dirname, .{}) catch |err| { | |
| 277 | std.process.fatal("unable to open directory while searching for {s} file, '{s}': {s}", .{ | |
| 278 | basename, dirname, @errorName(err), | |
| 279 | }); | |
| 280 | }; | |
| 281 | return .{ | |
| 282 | .build_zig_basename = basename, | |
| 283 | .directory = .{ | |
| 284 | .path = dirname, | |
| 285 | .handle = dir, | |
| 286 | }, | |
| 287 | .cleanup_build_dir = dir, | |
| 288 | }; | |
| 289 | } else |err| switch (err) { | |
| 290 | error.FileNotFound => { | |
| 291 | dirname = std.fs.path.dirname(dirname) orelse { | |
| 292 | std.log.info("initialize {s} template file with 'zig init'", .{basename}); | |
| 293 | std.log.info("see 'zig --help' for more options", .{}); | |
| 294 | std.process.fatal("no {s} file found, in the current directory or any parent directories", .{ | |
| 295 | basename, | |
| 296 | }); | |
| 297 | }; | |
| 298 | continue; | |
| 299 | }, | |
| 300 | else => |e| return e, | |
| 301 | } | |
| 302 | } | |
| 303 | } | |
| 304 | ||
| 305 | test { | |
| 306 | _ = Fetch; | |
| 307 | } |
lib/std/zig/Package/Fetch.zig created+2413
| ... | ... | @@ -0,0 +1,2413 @@ |
| 1 | //! Represents one independent job whose responsibility is to: | |
| 2 | //! | |
| 3 | //! 1. Check the global zig package cache to see if the hash already exists. | |
| 4 | //! If so, load, parse, and validate the build.zig.zon file therein, and | |
| 5 | //! goto step 8. Likewise if the location is a relative path, treat this | |
| 6 | //! the same as a cache hit. Otherwise, proceed. | |
| 7 | //! 2. Fetch and unpack a URL into a temporary directory. | |
| 8 | //! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed | |
| 9 | //! for the file to be missing, in which case this fetched package is considered | |
| 10 | //! to be a "naked" package. | |
| 11 | //! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by | |
| 12 | //! deleting excluded files. If any files had errors for files that were | |
| 13 | //! ultimately excluded, those errors should be ignored, such as failure to | |
| 14 | //! create symlinks that weren't supposed to be included anyway. | |
| 15 | //! 5. Compute the package hash based on the remaining files in the temporary | |
| 16 | //! directory. | |
| 17 | //! 6. Rename the temporary directory into the global zig package cache | |
| 18 | //! directory. If the hash already exists, delete the temporary directory and | |
| 19 | //! leave the zig package cache directory untouched as it may be in use by the | |
| 20 | //! system. This is done even if the hash is invalid, in case the package with | |
| 21 | //! the different hash is used in the future. | |
| 22 | //! 7. Validate the computed hash against the expected hash. If invalid, | |
| 23 | //! this job is done. | |
| 24 | //! 8. Spawn a new fetch job for each dependency in the manifest file. Use | |
| 25 | //! a mutex and a hash map so that redundant jobs do not get queued up. | |
| 26 | //! | |
| 27 | //! All of this must be done with only referring to the state inside this struct | |
| 28 | //! because this work will be done in a dedicated thread. | |
| 29 | ||
| 30 | arena: std.heap.ArenaAllocator, | |
| 31 | location: Location, | |
| 32 | location_tok: std.zig.Ast.TokenIndex, | |
| 33 | hash_tok: std.zig.Ast.OptionalTokenIndex, | |
| 34 | name_tok: std.zig.Ast.TokenIndex, | |
| 35 | lazy_status: LazyStatus, | |
| 36 | parent_package_root: Cache.Path, | |
| 37 | parent_manifest_ast: ?*const std.zig.Ast, | |
| 38 | prog_node: std.Progress.Node, | |
| 39 | job_queue: *JobQueue, | |
| 40 | /// If true, don't add an error for a missing hash. This flag is not passed | |
| 41 | /// down to recursive dependencies. It's intended to be used only be the CLI. | |
| 42 | omit_missing_hash_error: bool, | |
| 43 | /// If true, don't fail when a manifest file is missing the `paths` field, | |
| 44 | /// which specifies inclusion rules. This is intended to be true for the first | |
| 45 | /// fetch task and false for the recursive dependencies. | |
| 46 | allow_missing_paths_field: bool, | |
| 47 | allow_missing_fingerprint: bool, | |
| 48 | allow_name_string: bool, | |
| 49 | /// If true and URL points to a Git repository, will use the latest commit. | |
| 50 | use_latest_commit: bool, | |
| 51 | ||
| 52 | // Above this are fields provided as inputs to `run`. | |
| 53 | // Below this are fields populated by `run`. | |
| 54 | ||
| 55 | /// This will either be relative to `global_cache`, or to the build root of | |
| 56 | /// the root package. | |
| 57 | package_root: Cache.Path, | |
| 58 | error_bundle: ErrorBundle.Wip, | |
| 59 | manifest: ?Manifest, | |
| 60 | manifest_ast: std.zig.Ast, | |
| 61 | computed_hash: ComputedHash, | |
| 62 | /// Fetch logic notices whether a package has a build.zig file and sets this flag. | |
| 63 | has_build_zig: bool, | |
| 64 | /// Indicates whether the task aborted due to an out-of-memory condition. | |
| 65 | oom_flag: bool, | |
| 66 | /// If `use_latest_commit` was true, this will be set to the commit that was used. | |
| 67 | /// If the resource pointed to by the location is not a Git-repository, this | |
| 68 | /// will be left unchanged. | |
| 69 | latest_commit: ?git.Oid, | |
| 70 | ||
| 71 | userdata: ?*anyopaque = null, | |
| 72 | ||
| 73 | pub const LazyStatus = enum { | |
| 74 | /// Not lazy. | |
| 75 | eager, | |
| 76 | /// Lazy, found. | |
| 77 | available, | |
| 78 | /// Lazy, not found. | |
| 79 | unavailable, | |
| 80 | }; | |
| 81 | ||
| 82 | /// Contains shared state among all `Fetch` tasks. | |
| 83 | pub const JobQueue = struct { | |
| 84 | mutex: std.Thread.Mutex = .{}, | |
| 85 | /// It's an array hash map so that it can be sorted before rendering the | |
| 86 | /// dependencies.zig source file. | |
| 87 | /// Protected by `mutex`. | |
| 88 | table: Table = .{}, | |
| 89 | /// `table` may be missing some tasks such as ones that failed, so this | |
| 90 | /// field contains references to all of them. | |
| 91 | /// Protected by `mutex`. | |
| 92 | all_fetches: std.ArrayListUnmanaged(*Fetch) = .empty, | |
| 93 | ||
| 94 | http_client: *std.http.Client, | |
| 95 | thread_pool: *ThreadPool, | |
| 96 | wait_group: WaitGroup = .{}, | |
| 97 | global_cache: Cache.Directory, | |
| 98 | /// If true then, no fetching occurs, and: | |
| 99 | /// * The `global_cache` directory is assumed to be the direct parent | |
| 100 | /// directory of on-disk packages rather than having the "p/" directory | |
| 101 | /// prefix inside of it. | |
| 102 | /// * An error occurs if any non-lazy packages are not already present in | |
| 103 | /// the package cache directory. | |
| 104 | /// * Missing hash field causes an error, and no fetching occurs so it does | |
| 105 | /// not print the correct hash like usual. | |
| 106 | read_only: bool, | |
| 107 | recursive: bool, | |
| 108 | /// Dumps hash information to stdout which can be used to troubleshoot why | |
| 109 | /// two hashes of the same package do not match. | |
| 110 | /// If this is true, `recursive` must be false. | |
| 111 | debug_hash: bool, | |
| 112 | work_around_btrfs_bug: bool, | |
| 113 | /// Set of hashes that will be additionally fetched even if they are marked | |
| 114 | /// as lazy. | |
| 115 | unlazy_set: UnlazySet = .{}, | |
| 116 | ||
| 117 | pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch); | |
| 118 | pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void); | |
| 119 | ||
| 120 | pub fn deinit(jq: *JobQueue) void { | |
| 121 | if (jq.all_fetches.items.len == 0) return; | |
| 122 | const gpa = jq.all_fetches.items[0].arena.child_allocator; | |
| 123 | jq.table.deinit(gpa); | |
| 124 | // These must be deinitialized in reverse order because subsequent | |
| 125 | // `Fetch` instances are allocated in prior ones' arenas. | |
| 126 | // Sorry, I know it's a bit weird, but it slightly simplifies the | |
| 127 | // critical section. | |
| 128 | while (jq.all_fetches.pop()) |f| f.deinit(); | |
| 129 | jq.all_fetches.deinit(gpa); | |
| 130 | jq.* = undefined; | |
| 131 | } | |
| 132 | ||
| 133 | /// Dumps all subsequent error bundles into the first one. | |
| 134 | pub fn consolidateErrors(jq: *JobQueue) !void { | |
| 135 | const root = &jq.all_fetches.items[0].error_bundle; | |
| 136 | const gpa = root.gpa; | |
| 137 | for (jq.all_fetches.items[1..]) |fetch| { | |
| 138 | if (fetch.error_bundle.root_list.items.len > 0) { | |
| 139 | var bundle = try fetch.error_bundle.toOwnedBundle(""); | |
| 140 | defer bundle.deinit(gpa); | |
| 141 | try root.addBundleAsRoots(bundle); | |
| 142 | } | |
| 143 | } | |
| 144 | } | |
| 145 | ||
| 146 | /// Creates the dependencies.zig source code for the build runner to obtain | |
| 147 | /// via `@import("@dependencies")`. | |
| 148 | pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { | |
| 149 | const keys = jq.table.keys(); | |
| 150 | ||
| 151 | assert(keys.len != 0); // caller should have added the first one | |
| 152 | if (keys.len == 1) { | |
| 153 | // This is the first one. It must have no dependencies. | |
| 154 | return createEmptyDependenciesSource(buf); | |
| 155 | } | |
| 156 | ||
| 157 | try buf.appendSlice("pub const packages = struct {\n"); | |
| 158 | ||
| 159 | // Ensure the generated .zig file is deterministic. | |
| 160 | jq.table.sortUnstable(@as(struct { | |
| 161 | keys: []const Package.Hash, | |
| 162 | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { | |
| 163 | return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); | |
| 164 | } | |
| 165 | }, .{ .keys = keys })); | |
| 166 | ||
| 167 | for (keys, jq.table.values()) |*hash, fetch| { | |
| 168 | if (fetch == jq.all_fetches.items[0]) { | |
| 169 | // The first one is a dummy package for the current project. | |
| 170 | continue; | |
| 171 | } | |
| 172 | ||
| 173 | const hash_slice = hash.toSlice(); | |
| 174 | ||
| 175 | try buf.writer().print( | |
| 176 | \\ pub const {} = struct {{ | |
| 177 | \\ | |
| 178 | , .{std.zig.fmtId(hash_slice)}); | |
| 179 | ||
| 180 | lazy: { | |
| 181 | switch (fetch.lazy_status) { | |
| 182 | .eager => break :lazy, | |
| 183 | .available => { | |
| 184 | try buf.appendSlice( | |
| 185 | \\ pub const available = true; | |
| 186 | \\ | |
| 187 | ); | |
| 188 | break :lazy; | |
| 189 | }, | |
| 190 | .unavailable => { | |
| 191 | try buf.appendSlice( | |
| 192 | \\ pub const available = false; | |
| 193 | \\ }; | |
| 194 | \\ | |
| 195 | ); | |
| 196 | continue; | |
| 197 | }, | |
| 198 | } | |
| 199 | } | |
| 200 | ||
| 201 | try buf.writer().print( | |
| 202 | \\ pub const build_root = "{q}"; | |
| 203 | \\ | |
| 204 | , .{fetch.package_root}); | |
| 205 | ||
| 206 | if (fetch.has_build_zig) { | |
| 207 | try buf.writer().print( | |
| 208 | \\ pub const build_zig = @import("{}"); | |
| 209 | \\ | |
| 210 | , .{std.zig.fmtEscapes(hash_slice)}); | |
| 211 | } | |
| 212 | ||
| 213 | if (fetch.manifest) |*manifest| { | |
| 214 | try buf.appendSlice( | |
| 215 | \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ | |
| 216 | \\ | |
| 217 | ); | |
| 218 | for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { | |
| 219 | const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; | |
| 220 | try buf.writer().print( | |
| 221 | " .{{ \"{}\", \"{}\" }},\n", | |
| 222 | .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, | |
| 223 | ); | |
| 224 | } | |
| 225 | ||
| 226 | try buf.appendSlice( | |
| 227 | \\ }; | |
| 228 | \\ }; | |
| 229 | \\ | |
| 230 | ); | |
| 231 | } else { | |
| 232 | try buf.appendSlice( | |
| 233 | \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; | |
| 234 | \\ }; | |
| 235 | \\ | |
| 236 | ); | |
| 237 | } | |
| 238 | } | |
| 239 | ||
| 240 | try buf.appendSlice( | |
| 241 | \\}; | |
| 242 | \\ | |
| 243 | \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ | |
| 244 | \\ | |
| 245 | ); | |
| 246 | ||
| 247 | const root_fetch = jq.all_fetches.items[0]; | |
| 248 | const root_manifest = &root_fetch.manifest.?; | |
| 249 | ||
| 250 | for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { | |
| 251 | const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; | |
| 252 | try buf.writer().print( | |
| 253 | " .{{ \"{}\", \"{}\" }},\n", | |
| 254 | .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, | |
| 255 | ); | |
| 256 | } | |
| 257 | try buf.appendSlice("};\n"); | |
| 258 | } | |
| 259 | ||
| 260 | pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void { | |
| 261 | try buf.appendSlice( | |
| 262 | \\pub const packages = struct {}; | |
| 263 | \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; | |
| 264 | \\ | |
| 265 | ); | |
| 266 | } | |
| 267 | }; | |
| 268 | ||
| 269 | pub const Location = union(enum) { | |
| 270 | remote: Remote, | |
| 271 | /// A directory found inside the parent package. | |
| 272 | relative_path: Cache.Path, | |
| 273 | /// Recursive Fetch tasks will never use this Location, but it may be | |
| 274 | /// passed in by the CLI. Indicates the file contents here should be copied | |
| 275 | /// into the global package cache. It may be a file relative to the cwd or | |
| 276 | /// absolute, in which case it should be treated exactly like a `file://` | |
| 277 | /// URL, or a directory, in which case it should be treated as an | |
| 278 | /// already-unpacked directory (but still needs to be copied into the | |
| 279 | /// global package cache and have inclusion rules applied). | |
| 280 | path_or_url: []const u8, | |
| 281 | ||
| 282 | pub const Remote = struct { | |
| 283 | url: []const u8, | |
| 284 | /// If this is null it means the user omitted the hash field from a dependency. | |
| 285 | /// It will be an error but the logic should still fetch and print the discovered hash. | |
| 286 | hash: ?Package.Hash, | |
| 287 | }; | |
| 288 | }; | |
| 289 | ||
| 290 | pub const RunError = error{ | |
| 291 | OutOfMemory, | |
| 292 | /// This error code is intended to be handled by inspecting the | |
| 293 | /// `error_bundle` field. | |
| 294 | FetchFailed, | |
| 295 | }; | |
| 296 | ||
| 297 | pub fn run(f: *Fetch) RunError!void { | |
| 298 | const eb = &f.error_bundle; | |
| 299 | const arena = f.arena.allocator(); | |
| 300 | const gpa = f.arena.child_allocator; | |
| 301 | const cache_root = f.job_queue.global_cache; | |
| 302 | ||
| 303 | try eb.init(gpa); | |
| 304 | ||
| 305 | // Check the global zig package cache to see if the hash already exists. If | |
| 306 | // so, load, parse, and validate the build.zig.zon file therein, and skip | |
| 307 | // ahead to queuing up jobs for dependencies. Likewise if the location is a | |
| 308 | // relative path, treat this the same as a cache hit. Otherwise, proceed. | |
| 309 | ||
| 310 | const remote = switch (f.location) { | |
| 311 | .relative_path => |pkg_root| { | |
| 312 | if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( | |
| 313 | f.location_tok, | |
| 314 | try eb.addString("expected path relative to build root; found absolute path"), | |
| 315 | ); | |
| 316 | if (f.hash_tok.unwrap()) |hash_tok| return f.fail( | |
| 317 | hash_tok, | |
| 318 | try eb.addString("path-based dependencies are not hashed"), | |
| 319 | ); | |
| 320 | // Packages fetched by URL may not use relative paths to escape outside the | |
| 321 | // fetched package directory from within the package cache. | |
| 322 | if (pkg_root.root_dir.eql(cache_root)) { | |
| 323 | // `parent_package_root.sub_path` contains a path like this: | |
| 324 | // "p/$hash", or | |
| 325 | // "p/$hash/foo", with possibly more directories after "foo". | |
| 326 | // We want to fail unless the resolved relative path has a | |
| 327 | // prefix of "p/$hash/". | |
| 328 | const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; | |
| 329 | const parent_sub_path = f.parent_package_root.sub_path; | |
| 330 | const end = find_end: { | |
| 331 | if (parent_sub_path.len > prefix_len) { | |
| 332 | // Use `isSep` instead of `indexOfScalarPos` to account for | |
| 333 | // Windows accepting both `\` and `/` as path separators. | |
| 334 | for (parent_sub_path[prefix_len..], prefix_len..) |c, i| { | |
| 335 | if (std.fs.path.isSep(c)) break :find_end i; | |
| 336 | } | |
| 337 | } | |
| 338 | break :find_end parent_sub_path.len; | |
| 339 | }; | |
| 340 | const expected_prefix = parent_sub_path[0..end]; | |
| 341 | if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) { | |
| 342 | return f.fail( | |
| 343 | f.location_tok, | |
| 344 | try eb.printString("dependency path outside project: '{}'", .{pkg_root}), | |
| 345 | ); | |
| 346 | } | |
| 347 | } | |
| 348 | f.package_root = pkg_root; | |
| 349 | try loadManifest(f, pkg_root); | |
| 350 | if (!f.has_build_zig) try checkBuildFileExistence(f); | |
| 351 | if (!f.job_queue.recursive) return; | |
| 352 | return queueJobsForDeps(f); | |
| 353 | }, | |
| 354 | .remote => |remote| remote, | |
| 355 | .path_or_url => |path_or_url| { | |
| 356 | if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| { | |
| 357 | var resource: Resource = .{ .dir = dir }; | |
| 358 | return f.runResource(path_or_url, &resource, null); | |
| 359 | } else |dir_err| { | |
| 360 | const file_err = if (dir_err == error.NotDir) e: { | |
| 361 | if (fs.cwd().openFile(path_or_url, .{})) |file| { | |
| 362 | var resource: Resource = .{ .file = file }; | |
| 363 | return f.runResource(path_or_url, &resource, null); | |
| 364 | } else |err| break :e err; | |
| 365 | } else dir_err; | |
| 366 | ||
| 367 | const uri = std.Uri.parse(path_or_url) catch |uri_err| { | |
| 368 | return f.fail(0, try eb.printString( | |
| 369 | "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", | |
| 370 | .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, | |
| 371 | )); | |
| 372 | }; | |
| 373 | var server_header_buffer: [header_buffer_size]u8 = undefined; | |
| 374 | var resource = try f.initResource(uri, &server_header_buffer); | |
| 375 | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); | |
| 376 | } | |
| 377 | }, | |
| 378 | }; | |
| 379 | ||
| 380 | if (remote.hash) |expected_hash| { | |
| 381 | var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined; | |
| 382 | prefixed_pkg_sub_path_buffer[0] = 'p'; | |
| 383 | prefixed_pkg_sub_path_buffer[1] = fs.path.sep; | |
| 384 | const hash_slice = expected_hash.toSlice(); | |
| 385 | @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); | |
| 386 | const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; | |
| 387 | const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; | |
| 388 | const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; | |
| 389 | if (cache_root.handle.access(pkg_sub_path, .{})) |_| { | |
| 390 | assert(f.lazy_status != .unavailable); | |
| 391 | f.package_root = .{ | |
| 392 | .root_dir = cache_root, | |
| 393 | .sub_path = try arena.dupe(u8, pkg_sub_path), | |
| 394 | }; | |
| 395 | try loadManifest(f, f.package_root); | |
| 396 | try checkBuildFileExistence(f); | |
| 397 | if (!f.job_queue.recursive) return; | |
| 398 | return queueJobsForDeps(f); | |
| 399 | } else |err| switch (err) { | |
| 400 | error.FileNotFound => { | |
| 401 | switch (f.lazy_status) { | |
| 402 | .eager => {}, | |
| 403 | .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { | |
| 404 | f.lazy_status = .unavailable; | |
| 405 | return; | |
| 406 | }, | |
| 407 | .unavailable => unreachable, | |
| 408 | } | |
| 409 | if (f.job_queue.read_only) return f.fail( | |
| 410 | f.name_tok, | |
| 411 | try eb.printString("package not found at '{}{s}'", .{ | |
| 412 | cache_root, pkg_sub_path, | |
| 413 | }), | |
| 414 | ); | |
| 415 | }, | |
| 416 | else => |e| { | |
| 417 | try eb.addRootErrorMessage(.{ | |
| 418 | .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ | |
| 419 | cache_root, pkg_sub_path, @errorName(e), | |
| 420 | }), | |
| 421 | }); | |
| 422 | return error.FetchFailed; | |
| 423 | }, | |
| 424 | } | |
| 425 | } else if (f.job_queue.read_only) { | |
| 426 | try eb.addRootErrorMessage(.{ | |
| 427 | .msg = try eb.addString("dependency is missing hash field"), | |
| 428 | .src_loc = try f.srcLoc(f.location_tok), | |
| 429 | }); | |
| 430 | return error.FetchFailed; | |
| 431 | } | |
| 432 | ||
| 433 | // Fetch and unpack the remote into a temporary directory. | |
| 434 | ||
| 435 | const uri = std.Uri.parse(remote.url) catch |err| return f.fail( | |
| 436 | f.location_tok, | |
| 437 | try eb.printString("invalid URI: {s}", .{@errorName(err)}), | |
| 438 | ); | |
| 439 | var server_header_buffer: [header_buffer_size]u8 = undefined; | |
| 440 | var resource = try f.initResource(uri, &server_header_buffer); | |
| 441 | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); | |
| 442 | } | |
| 443 | ||
| 444 | pub fn deinit(f: *Fetch) void { | |
| 445 | f.error_bundle.deinit(); | |
| 446 | f.arena.deinit(); | |
| 447 | } | |
| 448 | ||
| 449 | /// Consumes `resource`, even if an error is returned. | |
| 450 | fn runResource( | |
| 451 | f: *Fetch, | |
| 452 | uri_path: []const u8, | |
| 453 | resource: *Resource, | |
| 454 | remote_hash: ?Package.Hash, | |
| 455 | ) RunError!void { | |
| 456 | defer resource.deinit(); | |
| 457 | const arena = f.arena.allocator(); | |
| 458 | const eb = &f.error_bundle; | |
| 459 | const s = fs.path.sep_str; | |
| 460 | const cache_root = f.job_queue.global_cache; | |
| 461 | const rand_int = std.crypto.random.int(u64); | |
| 462 | const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); | |
| 463 | ||
| 464 | const package_sub_path = blk: { | |
| 465 | const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); | |
| 466 | var tmp_directory: Cache.Directory = .{ | |
| 467 | .path = tmp_directory_path, | |
| 468 | .handle = handle: { | |
| 469 | const dir = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ | |
| 470 | .iterate = true, | |
| 471 | }) catch |err| { | |
| 472 | try eb.addRootErrorMessage(.{ | |
| 473 | .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ | |
| 474 | tmp_directory_path, @errorName(err), | |
| 475 | }), | |
| 476 | }); | |
| 477 | return error.FetchFailed; | |
| 478 | }; | |
| 479 | break :handle dir; | |
| 480 | }, | |
| 481 | }; | |
| 482 | defer tmp_directory.handle.close(); | |
| 483 | ||
| 484 | // Fetch and unpack a resource into a temporary directory. | |
| 485 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); | |
| 486 | ||
| 487 | var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; | |
| 488 | ||
| 489 | // Apply btrfs workaround if needed. Reopen tmp_directory. | |
| 490 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { | |
| 491 | // https://github.com/ziglang/zig/issues/17095 | |
| 492 | pkg_path.root_dir.handle.close(); | |
| 493 | pkg_path.root_dir.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ | |
| 494 | .iterate = true, | |
| 495 | }) catch @panic("btrfs workaround failed"); | |
| 496 | } | |
| 497 | ||
| 498 | // Load, parse, and validate the unpacked build.zig.zon file. It is allowed | |
| 499 | // for the file to be missing, in which case this fetched package is | |
| 500 | // considered to be a "naked" package. | |
| 501 | try loadManifest(f, pkg_path); | |
| 502 | ||
| 503 | const filter: Filter = .{ | |
| 504 | .include_paths = if (f.manifest) |m| m.paths else .{}, | |
| 505 | }; | |
| 506 | ||
| 507 | // Ignore errors that were excluded by manifest, such as failure to | |
| 508 | // create symlinks that weren't supposed to be included anyway. | |
| 509 | try unpack_result.validate(f, filter); | |
| 510 | ||
| 511 | // Apply the manifest's inclusion rules to the temporary directory by | |
| 512 | // deleting excluded files. | |
| 513 | // Empty directories have already been omitted by `unpackResource`. | |
| 514 | // Compute the package hash based on the remaining files in the temporary | |
| 515 | // directory. | |
| 516 | f.computed_hash = try computeHash(f, pkg_path, filter); | |
| 517 | ||
| 518 | break :blk if (unpack_result.root_dir.len > 0) | |
| 519 | try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) | |
| 520 | else | |
| 521 | tmp_dir_sub_path; | |
| 522 | }; | |
| 523 | ||
| 524 | const computed_package_hash = computedPackageHash(f); | |
| 525 | ||
| 526 | // Rename the temporary directory into the global zig package cache | |
| 527 | // directory. If the hash already exists, delete the temporary directory | |
| 528 | // and leave the zig package cache directory untouched as it may be in use | |
| 529 | // by the system. This is done even if the hash is invalid, in case the | |
| 530 | // package with the different hash is used in the future. | |
| 531 | ||
| 532 | f.package_root = .{ | |
| 533 | .root_dir = cache_root, | |
| 534 | .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), | |
| 535 | }; | |
| 536 | renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { | |
| 537 | const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); | |
| 538 | const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); | |
| 539 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | |
| 540 | "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", | |
| 541 | .{ src, dest, @errorName(err) }, | |
| 542 | ) }); | |
| 543 | return error.FetchFailed; | |
| 544 | }; | |
| 545 | // Remove temporary directory root if not already renamed to global cache. | |
| 546 | if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { | |
| 547 | cache_root.handle.deleteDir(tmp_dir_sub_path) catch {}; | |
| 548 | } | |
| 549 | ||
| 550 | // Validate the computed hash against the expected hash. If invalid, this | |
| 551 | // job is done. | |
| 552 | ||
| 553 | if (remote_hash) |declared_hash| { | |
| 554 | const hash_tok = f.hash_tok.unwrap().?; | |
| 555 | if (declared_hash.isOld()) { | |
| 556 | const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest); | |
| 557 | if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) { | |
| 558 | return f.fail(hash_tok, try eb.printString( | |
| 559 | "hash mismatch: manifest declares {s} but the fetched package has {s}", | |
| 560 | .{ declared_hash.toSlice(), actual_hex }, | |
| 561 | )); | |
| 562 | } | |
| 563 | } else { | |
| 564 | if (!computed_package_hash.eql(&declared_hash)) { | |
| 565 | return f.fail(hash_tok, try eb.printString( | |
| 566 | "hash mismatch: manifest declares {s} but the fetched package has {s}", | |
| 567 | .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, | |
| 568 | )); | |
| 569 | } | |
| 570 | } | |
| 571 | } else if (!f.omit_missing_hash_error) { | |
| 572 | const notes_len = 1; | |
| 573 | try eb.addRootErrorMessage(.{ | |
| 574 | .msg = try eb.addString("dependency is missing hash field"), | |
| 575 | .src_loc = try f.srcLoc(f.location_tok), | |
| 576 | .notes_len = notes_len, | |
| 577 | }); | |
| 578 | const notes_start = try eb.reserveNotes(notes_len); | |
| 579 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 580 | .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), | |
| 581 | })); | |
| 582 | return error.FetchFailed; | |
| 583 | } | |
| 584 | ||
| 585 | // Spawn a new fetch job for each dependency in the manifest file. Use | |
| 586 | // a mutex and a hash map so that redundant jobs do not get queued up. | |
| 587 | if (!f.job_queue.recursive) return; | |
| 588 | return queueJobsForDeps(f); | |
| 589 | } | |
| 590 | ||
| 591 | pub fn computedPackageHash(f: *const Fetch) Package.Hash { | |
| 592 | const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); | |
| 593 | if (f.manifest) |man| { | |
| 594 | var version_buffer: [32]u8 = undefined; | |
| 595 | const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer; | |
| 596 | return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); | |
| 597 | } | |
| 598 | // In the future build.zig.zon fields will be added to allow overriding these values | |
| 599 | // for naked tarballs. | |
| 600 | return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); | |
| 601 | } | |
| 602 | ||
| 603 | /// `computeHash` gets a free check for the existence of `build.zig`, but when | |
| 604 | /// not computing a hash, we need to do a syscall to check for it. | |
| 605 | fn checkBuildFileExistence(f: *Fetch) RunError!void { | |
| 606 | const eb = &f.error_bundle; | |
| 607 | if (f.package_root.access(Package.build_zig_basename, .{})) |_| { | |
| 608 | f.has_build_zig = true; | |
| 609 | } else |err| switch (err) { | |
| 610 | error.FileNotFound => {}, | |
| 611 | else => |e| { | |
| 612 | try eb.addRootErrorMessage(.{ | |
| 613 | .msg = try eb.printString("unable to access '{}{s}': {s}", .{ | |
| 614 | f.package_root, Package.build_zig_basename, @errorName(e), | |
| 615 | }), | |
| 616 | }); | |
| 617 | return error.FetchFailed; | |
| 618 | }, | |
| 619 | } | |
| 620 | } | |
| 621 | ||
| 622 | /// This function populates `f.manifest` or leaves it `null`. | |
| 623 | fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { | |
| 624 | const eb = &f.error_bundle; | |
| 625 | const arena = f.arena.allocator(); | |
| 626 | const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( | |
| 627 | arena, | |
| 628 | try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), | |
| 629 | Manifest.max_bytes, | |
| 630 | null, | |
| 631 | 1, | |
| 632 | 0, | |
| 633 | ) catch |err| switch (err) { | |
| 634 | error.FileNotFound => return, | |
| 635 | else => |e| { | |
| 636 | const file_path = try pkg_root.join(arena, Manifest.basename); | |
| 637 | try eb.addRootErrorMessage(.{ | |
| 638 | .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ | |
| 639 | file_path, @errorName(e), | |
| 640 | }), | |
| 641 | }); | |
| 642 | return error.FetchFailed; | |
| 643 | }, | |
| 644 | }; | |
| 645 | ||
| 646 | const ast = &f.manifest_ast; | |
| 647 | ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); | |
| 648 | ||
| 649 | if (ast.errors.len > 0) { | |
| 650 | const file_path = try std.fmt.allocPrint(arena, "{}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root}); | |
| 651 | try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); | |
| 652 | return error.FetchFailed; | |
| 653 | } | |
| 654 | ||
| 655 | f.manifest = try Manifest.parse(arena, ast.*, .{ | |
| 656 | .allow_missing_paths_field = f.allow_missing_paths_field, | |
| 657 | .allow_missing_fingerprint = f.allow_missing_fingerprint, | |
| 658 | .allow_name_string = f.allow_name_string, | |
| 659 | }); | |
| 660 | const manifest = &f.manifest.?; | |
| 661 | ||
| 662 | if (manifest.errors.len > 0) { | |
| 663 | const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename }); | |
| 664 | try manifest.copyErrorsIntoBundle(ast.*, src_path, eb); | |
| 665 | return error.FetchFailed; | |
| 666 | } | |
| 667 | } | |
| 668 | ||
| 669 | fn queueJobsForDeps(f: *Fetch) RunError!void { | |
| 670 | assert(f.job_queue.recursive); | |
| 671 | ||
| 672 | // If the package does not have a build.zig.zon file then there are no dependencies. | |
| 673 | const manifest = f.manifest orelse return; | |
| 674 | ||
| 675 | const new_fetches, const prog_names = nf: { | |
| 676 | const parent_arena = f.arena.allocator(); | |
| 677 | const gpa = f.arena.child_allocator; | |
| 678 | const cache_root = f.job_queue.global_cache; | |
| 679 | const dep_names = manifest.dependencies.keys(); | |
| 680 | const deps = manifest.dependencies.values(); | |
| 681 | // Grab the new tasks into a temporary buffer so we can unlock that mutex | |
| 682 | // as fast as possible. | |
| 683 | // This overallocates any fetches that get skipped by the `continue` in the | |
| 684 | // loop below. | |
| 685 | const new_fetches = try parent_arena.alloc(Fetch, deps.len); | |
| 686 | const prog_names = try parent_arena.alloc([]const u8, deps.len); | |
| 687 | var new_fetch_index: usize = 0; | |
| 688 | ||
| 689 | f.job_queue.mutex.lock(); | |
| 690 | defer f.job_queue.mutex.unlock(); | |
| 691 | ||
| 692 | try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); | |
| 693 | try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); | |
| 694 | ||
| 695 | // There are four cases here: | |
| 696 | // * Correct hash is provided by manifest. | |
| 697 | // - Hash map already has the entry, no need to add it again. | |
| 698 | // * Incorrect hash is provided by manifest. | |
| 699 | // - Hash mismatch error emitted; `queueJobsForDeps` is not called. | |
| 700 | // * Hash is not provided by manifest. | |
| 701 | // - Hash missing error emitted; `queueJobsForDeps` is not called. | |
| 702 | // * path-based location is used without a hash. | |
| 703 | // - Hash is added to the table based on the path alone before | |
| 704 | // calling run(); no need to add it again. | |
| 705 | // | |
| 706 | // If we add a dep as lazy and then later try to add the same dep as eager, | |
| 707 | // eagerness takes precedence and the existing entry is updated. | |
| 708 | ||
| 709 | for (dep_names, deps) |dep_name, dep| { | |
| 710 | const new_fetch = &new_fetches[new_fetch_index]; | |
| 711 | const location: Location = switch (dep.location) { | |
| 712 | .url => |url| .{ .remote = .{ | |
| 713 | .url = url, | |
| 714 | .hash = h: { | |
| 715 | const h = dep.hash orelse break :h null; | |
| 716 | const pkg_hash: Package.Hash = .fromSlice(h); | |
| 717 | const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); | |
| 718 | if (gop.found_existing) { | |
| 719 | if (!dep.lazy) { | |
| 720 | gop.value_ptr.*.lazy_status = .eager; | |
| 721 | } | |
| 722 | continue; | |
| 723 | } | |
| 724 | gop.value_ptr.* = new_fetch; | |
| 725 | break :h pkg_hash; | |
| 726 | }, | |
| 727 | } }, | |
| 728 | .path => |rel_path| l: { | |
| 729 | // This might produce an invalid path, which is checked for | |
| 730 | // at the beginning of run(). | |
| 731 | const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); | |
| 732 | const pkg_hash = relativePathDigest(new_root, cache_root); | |
| 733 | const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); | |
| 734 | if (gop.found_existing) { | |
| 735 | if (!dep.lazy) { | |
| 736 | gop.value_ptr.*.lazy_status = .eager; | |
| 737 | } | |
| 738 | continue; | |
| 739 | } | |
| 740 | gop.value_ptr.* = new_fetch; | |
| 741 | break :l .{ .relative_path = new_root }; | |
| 742 | }, | |
| 743 | }; | |
| 744 | prog_names[new_fetch_index] = dep_name; | |
| 745 | new_fetch_index += 1; | |
| 746 | f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); | |
| 747 | new_fetch.* = .{ | |
| 748 | .arena = std.heap.ArenaAllocator.init(gpa), | |
| 749 | .location = location, | |
| 750 | .location_tok = dep.location_tok, | |
| 751 | .hash_tok = dep.hash_tok, | |
| 752 | .name_tok = dep.name_tok, | |
| 753 | .lazy_status = if (dep.lazy) .available else .eager, | |
| 754 | .parent_package_root = f.package_root, | |
| 755 | .parent_manifest_ast = &f.manifest_ast, | |
| 756 | .prog_node = f.prog_node, | |
| 757 | .job_queue = f.job_queue, | |
| 758 | .omit_missing_hash_error = false, | |
| 759 | .allow_missing_paths_field = true, | |
| 760 | .allow_missing_fingerprint = true, | |
| 761 | .allow_name_string = true, | |
| 762 | .use_latest_commit = false, | |
| 763 | ||
| 764 | .package_root = undefined, | |
| 765 | .error_bundle = undefined, | |
| 766 | .manifest = null, | |
| 767 | .manifest_ast = undefined, | |
| 768 | .computed_hash = undefined, | |
| 769 | .has_build_zig = false, | |
| 770 | .oom_flag = false, | |
| 771 | .latest_commit = null, | |
| 772 | }; | |
| 773 | } | |
| 774 | ||
| 775 | f.prog_node.increaseEstimatedTotalItems(new_fetch_index); | |
| 776 | ||
| 777 | break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; | |
| 778 | }; | |
| 779 | ||
| 780 | // Now it's time to give tasks to the thread pool. | |
| 781 | const thread_pool = f.job_queue.thread_pool; | |
| 782 | ||
| 783 | for (new_fetches, prog_names) |*new_fetch, prog_name| { | |
| 784 | thread_pool.spawnWg(&f.job_queue.wait_group, workerRun, .{ new_fetch, prog_name }); | |
| 785 | } | |
| 786 | } | |
| 787 | ||
| 788 | pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { | |
| 789 | return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); | |
| 790 | } | |
| 791 | ||
| 792 | pub fn workerRun(f: *Fetch, prog_name: []const u8) void { | |
| 793 | const prog_node = f.prog_node.start(prog_name, 0); | |
| 794 | defer prog_node.end(); | |
| 795 | ||
| 796 | run(f) catch |err| switch (err) { | |
| 797 | error.OutOfMemory => f.oom_flag = true, | |
| 798 | error.FetchFailed => { | |
| 799 | // Nothing to do because the errors are already reported in `error_bundle`, | |
| 800 | // and a reference is kept to the `Fetch` task inside `all_fetches`. | |
| 801 | }, | |
| 802 | }; | |
| 803 | } | |
| 804 | ||
| 805 | fn srcLoc( | |
| 806 | f: *Fetch, | |
| 807 | tok: std.zig.Ast.TokenIndex, | |
| 808 | ) Allocator.Error!ErrorBundle.SourceLocationIndex { | |
| 809 | const ast = f.parent_manifest_ast orelse return .none; | |
| 810 | const eb = &f.error_bundle; | |
| 811 | const start_loc = ast.tokenLocation(0, tok); | |
| 812 | const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); | |
| 813 | const msg_off = 0; | |
| 814 | return eb.addSourceLocation(.{ | |
| 815 | .src_path = src_path, | |
| 816 | .span_start = ast.tokenStart(tok), | |
| 817 | .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), | |
| 818 | .span_main = ast.tokenStart(tok) + msg_off, | |
| 819 | .line = @intCast(start_loc.line), | |
| 820 | .column = @intCast(start_loc.column), | |
| 821 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | |
| 822 | }); | |
| 823 | } | |
| 824 | ||
| 825 | fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { | |
| 826 | const eb = &f.error_bundle; | |
| 827 | try eb.addRootErrorMessage(.{ | |
| 828 | .msg = msg_str, | |
| 829 | .src_loc = try f.srcLoc(msg_tok), | |
| 830 | }); | |
| 831 | return error.FetchFailed; | |
| 832 | } | |
| 833 | ||
| 834 | const Resource = union(enum) { | |
| 835 | file: fs.File, | |
| 836 | http_request: std.http.Client.Request, | |
| 837 | git: Git, | |
| 838 | dir: fs.Dir, | |
| 839 | ||
| 840 | const Git = struct { | |
| 841 | session: git.Session, | |
| 842 | fetch_stream: git.Session.FetchStream, | |
| 843 | want_oid: git.Oid, | |
| 844 | }; | |
| 845 | ||
| 846 | fn deinit(resource: *Resource) void { | |
| 847 | switch (resource.*) { | |
| 848 | .file => |*file| file.close(), | |
| 849 | .http_request => |*req| req.deinit(), | |
| 850 | .git => |*git_resource| { | |
| 851 | git_resource.fetch_stream.deinit(); | |
| 852 | git_resource.session.deinit(); | |
| 853 | }, | |
| 854 | .dir => |*dir| dir.close(), | |
| 855 | } | |
| 856 | resource.* = undefined; | |
| 857 | } | |
| 858 | ||
| 859 | fn reader(resource: *Resource) std.io.AnyReader { | |
| 860 | return .{ | |
| 861 | .context = resource, | |
| 862 | .readFn = read, | |
| 863 | }; | |
| 864 | } | |
| 865 | ||
| 866 | fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { | |
| 867 | const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); | |
| 868 | switch (resource.*) { | |
| 869 | .file => |*f| return f.read(buffer), | |
| 870 | .http_request => |*r| return r.read(buffer), | |
| 871 | .git => |*g| return g.fetch_stream.read(buffer), | |
| 872 | .dir => unreachable, | |
| 873 | } | |
| 874 | } | |
| 875 | }; | |
| 876 | ||
| 877 | const FileType = enum { | |
| 878 | tar, | |
| 879 | @"tar.gz", | |
| 880 | @"tar.xz", | |
| 881 | @"tar.zst", | |
| 882 | git_pack, | |
| 883 | zip, | |
| 884 | ||
| 885 | fn fromPath(file_path: []const u8) ?FileType { | |
| 886 | if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; | |
| 887 | if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; | |
| 888 | if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; | |
| 889 | if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; | |
| 890 | if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; | |
| 891 | if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; | |
| 892 | if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; | |
| 893 | if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; | |
| 894 | return null; | |
| 895 | } | |
| 896 | ||
| 897 | /// Parameter is a content-disposition header value. | |
| 898 | fn fromContentDisposition(cd_header: []const u8) ?FileType { | |
| 899 | const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse | |
| 900 | return null; | |
| 901 | ||
| 902 | var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse | |
| 903 | return null; | |
| 904 | value_start += "filename".len; | |
| 905 | if (cd_header[value_start] == '*') { | |
| 906 | value_start += 1; | |
| 907 | } | |
| 908 | if (cd_header[value_start] != '=') return null; | |
| 909 | value_start += 1; | |
| 910 | ||
| 911 | var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; | |
| 912 | if (cd_header[value_end - 1] == '\"') { | |
| 913 | value_end -= 1; | |
| 914 | } | |
| 915 | return fromPath(cd_header[value_start..value_end]); | |
| 916 | } | |
| 917 | ||
| 918 | test fromContentDisposition { | |
| 919 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); | |
| 920 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); | |
| 921 | try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); | |
| 922 | try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); | |
| 923 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); | |
| 924 | try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); | |
| 925 | ||
| 926 | try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); | |
| 927 | try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); | |
| 928 | try std.testing.expect(fromContentDisposition("attachment; size=42") == null); | |
| 929 | try std.testing.expect(fromContentDisposition("inline; size=42") == null); | |
| 930 | try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); | |
| 931 | try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); | |
| 932 | } | |
| 933 | }; | |
| 934 | ||
| 935 | const header_buffer_size = 16 * 1024; | |
| 936 | ||
| 937 | fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource { | |
| 938 | const gpa = f.arena.child_allocator; | |
| 939 | const arena = f.arena.allocator(); | |
| 940 | const eb = &f.error_bundle; | |
| 941 | ||
| 942 | if (ascii.eqlIgnoreCase(uri.scheme, "file")) { | |
| 943 | const path = try uri.path.toRawMaybeAlloc(arena); | |
| 944 | return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| { | |
| 945 | return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ | |
| 946 | f.parent_package_root, path, @errorName(err), | |
| 947 | })); | |
| 948 | } }; | |
| 949 | } | |
| 950 | ||
| 951 | const http_client = f.job_queue.http_client; | |
| 952 | ||
| 953 | if (ascii.eqlIgnoreCase(uri.scheme, "http") or | |
| 954 | ascii.eqlIgnoreCase(uri.scheme, "https")) | |
| 955 | { | |
| 956 | var req = http_client.open(.GET, uri, .{ | |
| 957 | .server_header_buffer = server_header_buffer, | |
| 958 | }) catch |err| { | |
| 959 | return f.fail(f.location_tok, try eb.printString( | |
| 960 | "unable to connect to server: {s}", | |
| 961 | .{@errorName(err)}, | |
| 962 | )); | |
| 963 | }; | |
| 964 | errdefer req.deinit(); // releases more than memory | |
| 965 | ||
| 966 | req.send() catch |err| { | |
| 967 | return f.fail(f.location_tok, try eb.printString( | |
| 968 | "HTTP request failed: {s}", | |
| 969 | .{@errorName(err)}, | |
| 970 | )); | |
| 971 | }; | |
| 972 | req.wait() catch |err| { | |
| 973 | return f.fail(f.location_tok, try eb.printString( | |
| 974 | "invalid HTTP response: {s}", | |
| 975 | .{@errorName(err)}, | |
| 976 | )); | |
| 977 | }; | |
| 978 | ||
| 979 | if (req.response.status != .ok) { | |
| 980 | return f.fail(f.location_tok, try eb.printString( | |
| 981 | "bad HTTP response code: '{d} {s}'", | |
| 982 | .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, | |
| 983 | )); | |
| 984 | } | |
| 985 | ||
| 986 | return .{ .http_request = req }; | |
| 987 | } | |
| 988 | ||
| 989 | if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or | |
| 990 | ascii.eqlIgnoreCase(uri.scheme, "git+https")) | |
| 991 | { | |
| 992 | var transport_uri = uri; | |
| 993 | transport_uri.scheme = uri.scheme["git+".len..]; | |
| 994 | var session = git.Session.init(gpa, http_client, transport_uri, server_header_buffer) catch |err| { | |
| 995 | return f.fail(f.location_tok, try eb.printString( | |
| 996 | "unable to discover remote git server capabilities: {s}", | |
| 997 | .{@errorName(err)}, | |
| 998 | )); | |
| 999 | }; | |
| 1000 | errdefer session.deinit(); | |
| 1001 | ||
| 1002 | const want_oid = want_oid: { | |
| 1003 | const want_ref = | |
| 1004 | if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; | |
| 1005 | if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} | |
| 1006 | ||
| 1007 | const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); | |
| 1008 | const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); | |
| 1009 | ||
| 1010 | var ref_iterator = session.listRefs(.{ | |
| 1011 | .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, | |
| 1012 | .include_peeled = true, | |
| 1013 | .server_header_buffer = server_header_buffer, | |
| 1014 | }) catch |err| { | |
| 1015 | return f.fail(f.location_tok, try eb.printString( | |
| 1016 | "unable to list refs: {s}", | |
| 1017 | .{@errorName(err)}, | |
| 1018 | )); | |
| 1019 | }; | |
| 1020 | defer ref_iterator.deinit(); | |
| 1021 | while (ref_iterator.next() catch |err| { | |
| 1022 | return f.fail(f.location_tok, try eb.printString( | |
| 1023 | "unable to iterate refs: {s}", | |
| 1024 | .{@errorName(err)}, | |
| 1025 | )); | |
| 1026 | }) |ref| { | |
| 1027 | if (std.mem.eql(u8, ref.name, want_ref) or | |
| 1028 | std.mem.eql(u8, ref.name, want_ref_head) or | |
| 1029 | std.mem.eql(u8, ref.name, want_ref_tag)) | |
| 1030 | { | |
| 1031 | break :want_oid ref.peeled orelse ref.oid; | |
| 1032 | } | |
| 1033 | } | |
| 1034 | return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); | |
| 1035 | }; | |
| 1036 | if (f.use_latest_commit) { | |
| 1037 | f.latest_commit = want_oid; | |
| 1038 | } else if (uri.fragment == null) { | |
| 1039 | const notes_len = 1; | |
| 1040 | try eb.addRootErrorMessage(.{ | |
| 1041 | .msg = try eb.addString("url field is missing an explicit ref"), | |
| 1042 | .src_loc = try f.srcLoc(f.location_tok), | |
| 1043 | .notes_len = notes_len, | |
| 1044 | }); | |
| 1045 | const notes_start = try eb.reserveNotes(notes_len); | |
| 1046 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1047 | .msg = try eb.printString("try .url = \"{;+/}#{}\",", .{ uri, want_oid }), | |
| 1048 | })); | |
| 1049 | return error.FetchFailed; | |
| 1050 | } | |
| 1051 | ||
| 1052 | var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; | |
| 1053 | _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{want_oid}) catch unreachable; | |
| 1054 | var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| { | |
| 1055 | return f.fail(f.location_tok, try eb.printString( | |
| 1056 | "unable to create fetch stream: {s}", | |
| 1057 | .{@errorName(err)}, | |
| 1058 | )); | |
| 1059 | }; | |
| 1060 | errdefer fetch_stream.deinit(); | |
| 1061 | ||
| 1062 | return .{ .git = .{ | |
| 1063 | .session = session, | |
| 1064 | .fetch_stream = fetch_stream, | |
| 1065 | .want_oid = want_oid, | |
| 1066 | } }; | |
| 1067 | } | |
| 1068 | ||
| 1069 | return f.fail(f.location_tok, try eb.printString( | |
| 1070 | "unsupported URL scheme: {s}", | |
| 1071 | .{uri.scheme}, | |
| 1072 | )); | |
| 1073 | } | |
| 1074 | ||
| 1075 | fn unpackResource( | |
| 1076 | f: *Fetch, | |
| 1077 | resource: *Resource, | |
| 1078 | uri_path: []const u8, | |
| 1079 | tmp_directory: Cache.Directory, | |
| 1080 | ) RunError!UnpackResult { | |
| 1081 | const eb = &f.error_bundle; | |
| 1082 | const file_type = switch (resource.*) { | |
| 1083 | .file => FileType.fromPath(uri_path) orelse | |
| 1084 | return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), | |
| 1085 | ||
| 1086 | .http_request => |req| ft: { | |
| 1087 | // Content-Type takes first precedence. | |
| 1088 | const content_type = req.response.content_type orelse | |
| 1089 | return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); | |
| 1090 | ||
| 1091 | // Extract the MIME type, ignoring charset and boundary directives | |
| 1092 | const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; | |
| 1093 | const mime_type = content_type[0..mime_type_end]; | |
| 1094 | ||
| 1095 | if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) | |
| 1096 | break :ft .tar; | |
| 1097 | ||
| 1098 | if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or | |
| 1099 | ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or | |
| 1100 | ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or | |
| 1101 | ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or | |
| 1102 | ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) | |
| 1103 | { | |
| 1104 | break :ft .@"tar.gz"; | |
| 1105 | } | |
| 1106 | ||
| 1107 | if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) | |
| 1108 | break :ft .@"tar.xz"; | |
| 1109 | ||
| 1110 | if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) | |
| 1111 | break :ft .@"tar.zst"; | |
| 1112 | ||
| 1113 | if (ascii.eqlIgnoreCase(mime_type, "application/zip")) | |
| 1114 | break :ft .zip; | |
| 1115 | ||
| 1116 | if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and | |
| 1117 | !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) | |
| 1118 | { | |
| 1119 | return f.fail(f.location_tok, try eb.printString( | |
| 1120 | "unrecognized 'Content-Type' header: '{s}'", | |
| 1121 | .{content_type}, | |
| 1122 | )); | |
| 1123 | } | |
| 1124 | ||
| 1125 | // Next, the filename from 'content-disposition: attachment' takes precedence. | |
| 1126 | if (req.response.content_disposition) |cd_header| { | |
| 1127 | break :ft FileType.fromContentDisposition(cd_header) orelse { | |
| 1128 | return f.fail(f.location_tok, try eb.printString( | |
| 1129 | "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", | |
| 1130 | .{cd_header}, | |
| 1131 | )); | |
| 1132 | }; | |
| 1133 | } | |
| 1134 | ||
| 1135 | // Finally, the path from the URI is used. | |
| 1136 | break :ft FileType.fromPath(uri_path) orelse { | |
| 1137 | return f.fail(f.location_tok, try eb.printString( | |
| 1138 | "unknown file type: '{s}'", | |
| 1139 | .{uri_path}, | |
| 1140 | )); | |
| 1141 | }; | |
| 1142 | }, | |
| 1143 | ||
| 1144 | .git => .git_pack, | |
| 1145 | ||
| 1146 | .dir => |dir| { | |
| 1147 | f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { | |
| 1148 | return f.fail(f.location_tok, try eb.printString( | |
| 1149 | "unable to copy directory '{s}': {s}", | |
| 1150 | .{ uri_path, @errorName(err) }, | |
| 1151 | )); | |
| 1152 | }; | |
| 1153 | return .{}; | |
| 1154 | }, | |
| 1155 | }; | |
| 1156 | ||
| 1157 | switch (file_type) { | |
| 1158 | .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()), | |
| 1159 | .@"tar.gz" => { | |
| 1160 | const reader = resource.reader(); | |
| 1161 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | |
| 1162 | var dcp = std.compress.gzip.decompressor(br.reader()); | |
| 1163 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | |
| 1164 | }, | |
| 1165 | .@"tar.xz" => { | |
| 1166 | const gpa = f.arena.child_allocator; | |
| 1167 | const reader = resource.reader(); | |
| 1168 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | |
| 1169 | var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| { | |
| 1170 | return f.fail(f.location_tok, try eb.printString( | |
| 1171 | "unable to decompress tarball: {s}", | |
| 1172 | .{@errorName(err)}, | |
| 1173 | )); | |
| 1174 | }; | |
| 1175 | defer dcp.deinit(); | |
| 1176 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | |
| 1177 | }, | |
| 1178 | .@"tar.zst" => { | |
| 1179 | const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; | |
| 1180 | const window_buffer = try f.arena.allocator().create([window_size]u8); | |
| 1181 | const reader = resource.reader(); | |
| 1182 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | |
| 1183 | var dcp = std.compress.zstd.decompressor(br.reader(), .{ | |
| 1184 | .window_buffer = window_buffer, | |
| 1185 | }); | |
| 1186 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | |
| 1187 | }, | |
| 1188 | .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { | |
| 1189 | error.FetchFailed => return error.FetchFailed, | |
| 1190 | error.OutOfMemory => return error.OutOfMemory, | |
| 1191 | else => |e| return f.fail(f.location_tok, try eb.printString( | |
| 1192 | "unable to unpack git files: {s}", | |
| 1193 | .{@errorName(e)}, | |
| 1194 | )), | |
| 1195 | }, | |
| 1196 | .zip => return try unzip(f, tmp_directory.handle, resource.reader()), | |
| 1197 | } | |
| 1198 | } | |
| 1199 | ||
| 1200 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { | |
| 1201 | const eb = &f.error_bundle; | |
| 1202 | const arena = f.arena.allocator(); | |
| 1203 | ||
| 1204 | var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; | |
| 1205 | ||
| 1206 | std.tar.pipeToFileSystem(out_dir, reader, .{ | |
| 1207 | .diagnostics = &diagnostics, | |
| 1208 | .strip_components = 0, | |
| 1209 | .mode_mode = .ignore, | |
| 1210 | .exclude_empty_directories = true, | |
| 1211 | }) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1212 | "unable to unpack tarball to temporary directory: {s}", | |
| 1213 | .{@errorName(err)}, | |
| 1214 | )); | |
| 1215 | ||
| 1216 | var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; | |
| 1217 | if (diagnostics.errors.items.len > 0) { | |
| 1218 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); | |
| 1219 | for (diagnostics.errors.items) |item| { | |
| 1220 | switch (item) { | |
| 1221 | .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), | |
| 1222 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), | |
| 1223 | .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), | |
| 1224 | .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 | |
| 1225 | } | |
| 1226 | } | |
| 1227 | } | |
| 1228 | return res; | |
| 1229 | } | |
| 1230 | ||
| 1231 | fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { | |
| 1232 | // We write the entire contents to a file first because zip files | |
| 1233 | // must be processed back to front and they could be too large to | |
| 1234 | // load into memory. | |
| 1235 | ||
| 1236 | const cache_root = f.job_queue.global_cache; | |
| 1237 | ||
| 1238 | // TODO: the downside of this solution is if we get a failure/crash/oom/power out | |
| 1239 | // during this process, we leave behind a zip file that would be | |
| 1240 | // difficult to know if/when it can be cleaned up. | |
| 1241 | // Might be worth it to use a mechanism that enables other processes | |
| 1242 | // to see if the owning process of a file is still alive (on linux this | |
| 1243 | // can be done with file locks). | |
| 1244 | // Coupled with this mechansism, we could also use slots (i.e. zig-cache/tmp/0, | |
| 1245 | // zig-cache/tmp/1, etc) which would mean that subsequent runs would | |
| 1246 | // automatically clean up old dead files. | |
| 1247 | // This could all be done with a simple TmpFile abstraction. | |
| 1248 | const prefix = "tmp/"; | |
| 1249 | const suffix = ".zip"; | |
| 1250 | ||
| 1251 | const random_bytes_count = 20; | |
| 1252 | const random_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count); | |
| 1253 | var zip_path: [prefix.len + random_path_len + suffix.len]u8 = undefined; | |
| 1254 | @memcpy(zip_path[0..prefix.len], prefix); | |
| 1255 | @memcpy(zip_path[prefix.len + random_path_len ..], suffix); | |
| 1256 | { | |
| 1257 | var random_bytes: [random_bytes_count]u8 = undefined; | |
| 1258 | std.crypto.random.bytes(&random_bytes); | |
| 1259 | _ = std.fs.base64_encoder.encode( | |
| 1260 | zip_path[prefix.len..][0..random_path_len], | |
| 1261 | &random_bytes, | |
| 1262 | ); | |
| 1263 | } | |
| 1264 | ||
| 1265 | defer cache_root.handle.deleteFile(&zip_path) catch {}; | |
| 1266 | ||
| 1267 | const eb = &f.error_bundle; | |
| 1268 | ||
| 1269 | { | |
| 1270 | var zip_file = cache_root.handle.createFile( | |
| 1271 | &zip_path, | |
| 1272 | .{}, | |
| 1273 | ) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1274 | "failed to create tmp zip file: {s}", | |
| 1275 | .{@errorName(err)}, | |
| 1276 | )); | |
| 1277 | defer zip_file.close(); | |
| 1278 | var buf: [4096]u8 = undefined; | |
| 1279 | while (true) { | |
| 1280 | const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1281 | "read zip stream failed: {s}", | |
| 1282 | .{@errorName(err)}, | |
| 1283 | )); | |
| 1284 | if (len == 0) break; | |
| 1285 | zip_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1286 | "write temporary zip file failed: {s}", | |
| 1287 | .{@errorName(err)}, | |
| 1288 | )); | |
| 1289 | } | |
| 1290 | } | |
| 1291 | ||
| 1292 | var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; | |
| 1293 | // no need to deinit since we are using an arena allocator | |
| 1294 | ||
| 1295 | { | |
| 1296 | var zip_file = cache_root.handle.openFile( | |
| 1297 | &zip_path, | |
| 1298 | .{}, | |
| 1299 | ) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1300 | "failed to open temporary zip file: {s}", | |
| 1301 | .{@errorName(err)}, | |
| 1302 | )); | |
| 1303 | defer zip_file.close(); | |
| 1304 | ||
| 1305 | std.zip.extract(out_dir, zip_file.seekableStream(), .{ | |
| 1306 | .allow_backslashes = true, | |
| 1307 | .diagnostics = &diagnostics, | |
| 1308 | }) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1309 | "zip extract failed: {s}", | |
| 1310 | .{@errorName(err)}, | |
| 1311 | )); | |
| 1312 | } | |
| 1313 | ||
| 1314 | cache_root.handle.deleteFile(&zip_path) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1315 | "delete temporary zip failed: {s}", | |
| 1316 | .{@errorName(err)}, | |
| 1317 | )); | |
| 1318 | ||
| 1319 | const res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; | |
| 1320 | return res; | |
| 1321 | } | |
| 1322 | ||
| 1323 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult { | |
| 1324 | const arena = f.arena.allocator(); | |
| 1325 | const gpa = f.arena.child_allocator; | |
| 1326 | const object_format: git.Oid.Format = resource.want_oid; | |
| 1327 | ||
| 1328 | var res: UnpackResult = .{}; | |
| 1329 | // The .git directory is used to store the packfile and associated index, but | |
| 1330 | // we do not attempt to replicate the exact structure of a real .git | |
| 1331 | // directory, since that isn't relevant for fetching a package. | |
| 1332 | { | |
| 1333 | var pack_dir = try out_dir.makeOpenPath(".git", .{}); | |
| 1334 | defer pack_dir.close(); | |
| 1335 | var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); | |
| 1336 | defer pack_file.close(); | |
| 1337 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1338 | try fifo.pump(resource.fetch_stream.reader(), pack_file.writer()); | |
| 1339 | try pack_file.sync(); | |
| 1340 | ||
| 1341 | var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); | |
| 1342 | defer index_file.close(); | |
| 1343 | { | |
| 1344 | const index_prog_node = f.prog_node.start("Index pack", 0); | |
| 1345 | defer index_prog_node.end(); | |
| 1346 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); | |
| 1347 | try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer()); | |
| 1348 | try index_buffered_writer.flush(); | |
| 1349 | try index_file.sync(); | |
| 1350 | } | |
| 1351 | ||
| 1352 | { | |
| 1353 | const checkout_prog_node = f.prog_node.start("Checkout", 0); | |
| 1354 | defer checkout_prog_node.end(); | |
| 1355 | var repository = try git.Repository.init(gpa, object_format, pack_file, index_file); | |
| 1356 | defer repository.deinit(); | |
| 1357 | var diagnostics: git.Diagnostics = .{ .allocator = arena }; | |
| 1358 | try repository.checkout(out_dir, resource.want_oid, &diagnostics); | |
| 1359 | ||
| 1360 | if (diagnostics.errors.items.len > 0) { | |
| 1361 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); | |
| 1362 | for (diagnostics.errors.items) |item| { | |
| 1363 | switch (item) { | |
| 1364 | .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), | |
| 1365 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), | |
| 1366 | } | |
| 1367 | } | |
| 1368 | } | |
| 1369 | } | |
| 1370 | } | |
| 1371 | ||
| 1372 | try out_dir.deleteTree(".git"); | |
| 1373 | return res; | |
| 1374 | } | |
| 1375 | ||
| 1376 | fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { | |
| 1377 | const gpa = f.arena.child_allocator; | |
| 1378 | // Recursive directory copy. | |
| 1379 | var it = try dir.walk(gpa); | |
| 1380 | defer it.deinit(); | |
| 1381 | while (try it.next()) |entry| { | |
| 1382 | switch (entry.kind) { | |
| 1383 | .directory => {}, // omit empty directories | |
| 1384 | .file => { | |
| 1385 | dir.copyFile( | |
| 1386 | entry.path, | |
| 1387 | tmp_dir, | |
| 1388 | entry.path, | |
| 1389 | .{}, | |
| 1390 | ) catch |err| switch (err) { | |
| 1391 | error.FileNotFound => { | |
| 1392 | if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); | |
| 1393 | try dir.copyFile(entry.path, tmp_dir, entry.path, .{}); | |
| 1394 | }, | |
| 1395 | else => |e| return e, | |
| 1396 | }; | |
| 1397 | }, | |
| 1398 | .sym_link => { | |
| 1399 | var buf: [fs.max_path_bytes]u8 = undefined; | |
| 1400 | const link_name = try dir.readLink(entry.path, &buf); | |
| 1401 | // TODO: if this would create a symlink to outside | |
| 1402 | // the destination directory, fail with an error instead. | |
| 1403 | tmp_dir.symLink(link_name, entry.path, .{}) catch |err| switch (err) { | |
| 1404 | error.FileNotFound => { | |
| 1405 | if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); | |
| 1406 | try tmp_dir.symLink(link_name, entry.path, .{}); | |
| 1407 | }, | |
| 1408 | else => |e| return e, | |
| 1409 | }; | |
| 1410 | }, | |
| 1411 | else => return error.IllegalFileTypeInPackage, | |
| 1412 | } | |
| 1413 | } | |
| 1414 | } | |
| 1415 | ||
| 1416 | pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { | |
| 1417 | assert(dest_dir_sub_path[1] == fs.path.sep); | |
| 1418 | var handled_missing_dir = false; | |
| 1419 | while (true) { | |
| 1420 | cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { | |
| 1421 | error.FileNotFound => { | |
| 1422 | if (handled_missing_dir) return err; | |
| 1423 | cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { | |
| 1424 | error.PathAlreadyExists => handled_missing_dir = true, | |
| 1425 | else => |e| return e, | |
| 1426 | }; | |
| 1427 | continue; | |
| 1428 | }, | |
| 1429 | error.PathAlreadyExists, error.AccessDenied => { | |
| 1430 | // Package has been already downloaded and may already be in use on the system. | |
| 1431 | cache_dir.deleteTree(tmp_dir_sub_path) catch { | |
| 1432 | // Garbage files leftover in zig-cache/tmp/ is, as they say | |
| 1433 | // on Star Trek, "operating within normal parameters". | |
| 1434 | }; | |
| 1435 | }, | |
| 1436 | else => |e| return e, | |
| 1437 | }; | |
| 1438 | break; | |
| 1439 | } | |
| 1440 | } | |
| 1441 | ||
| 1442 | const ComputedHash = struct { | |
| 1443 | digest: Package.Hash.Digest, | |
| 1444 | total_size: u64, | |
| 1445 | }; | |
| 1446 | ||
| 1447 | /// Assumes that files not included in the package have already been filtered | |
| 1448 | /// prior to calling this function. This ensures that files not protected by | |
| 1449 | /// the hash are not present on the file system. Empty directories are *not | |
| 1450 | /// hashed* and must not be present on the file system when calling this | |
| 1451 | /// function. | |
| 1452 | fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { | |
| 1453 | // All the path name strings need to be in memory for sorting. | |
| 1454 | const arena = f.arena.allocator(); | |
| 1455 | const gpa = f.arena.child_allocator; | |
| 1456 | const eb = &f.error_bundle; | |
| 1457 | const thread_pool = f.job_queue.thread_pool; | |
| 1458 | const root_dir = pkg_path.root_dir.handle; | |
| 1459 | ||
| 1460 | // Collect all files, recursively, then sort. | |
| 1461 | var all_files = std.ArrayList(*HashedFile).init(gpa); | |
| 1462 | defer all_files.deinit(); | |
| 1463 | ||
| 1464 | var deleted_files = std.ArrayList(*DeletedFile).init(gpa); | |
| 1465 | defer deleted_files.deinit(); | |
| 1466 | ||
| 1467 | // Track directories which had any files deleted from them so that empty directories | |
| 1468 | // can be deleted. | |
| 1469 | var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; | |
| 1470 | defer sus_dirs.deinit(gpa); | |
| 1471 | ||
| 1472 | var walker = try root_dir.walk(gpa); | |
| 1473 | defer walker.deinit(); | |
| 1474 | ||
| 1475 | // Total number of bytes of file contents included in the package. | |
| 1476 | var total_size: u64 = 0; | |
| 1477 | ||
| 1478 | { | |
| 1479 | // The final hash will be a hash of each file hashed independently. This | |
| 1480 | // allows hashing in parallel. | |
| 1481 | var wait_group: WaitGroup = .{}; | |
| 1482 | // `computeHash` is called from a worker thread so there must not be | |
| 1483 | // any waiting without working or a deadlock could occur. | |
| 1484 | defer thread_pool.waitAndWork(&wait_group); | |
| 1485 | ||
| 1486 | while (walker.next() catch |err| { | |
| 1487 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | |
| 1488 | "unable to walk temporary directory '{}': {s}", | |
| 1489 | .{ pkg_path, @errorName(err) }, | |
| 1490 | ) }); | |
| 1491 | return error.FetchFailed; | |
| 1492 | }) |entry| { | |
| 1493 | if (entry.kind == .directory) continue; | |
| 1494 | ||
| 1495 | const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); | |
| 1496 | if (!filter.includePath(entry_pkg_path)) { | |
| 1497 | // Delete instead of including in hash calculation. | |
| 1498 | const fs_path = try arena.dupe(u8, entry.path); | |
| 1499 | ||
| 1500 | // Also track the parent directory in case it becomes empty. | |
| 1501 | if (fs.path.dirname(fs_path)) |parent| | |
| 1502 | try sus_dirs.put(gpa, parent, {}); | |
| 1503 | ||
| 1504 | const deleted_file = try arena.create(DeletedFile); | |
| 1505 | deleted_file.* = .{ | |
| 1506 | .fs_path = fs_path, | |
| 1507 | .failure = undefined, // to be populated by the worker | |
| 1508 | }; | |
| 1509 | thread_pool.spawnWg(&wait_group, workerDeleteFile, .{ root_dir, deleted_file }); | |
| 1510 | try deleted_files.append(deleted_file); | |
| 1511 | continue; | |
| 1512 | } | |
| 1513 | ||
| 1514 | const kind: HashedFile.Kind = switch (entry.kind) { | |
| 1515 | .directory => unreachable, | |
| 1516 | .file => .file, | |
| 1517 | .sym_link => .link, | |
| 1518 | else => return f.fail(f.location_tok, try eb.printString( | |
| 1519 | "package contains '{s}' which has illegal file type '{s}'", | |
| 1520 | .{ entry.path, @tagName(entry.kind) }, | |
| 1521 | )), | |
| 1522 | }; | |
| 1523 | ||
| 1524 | if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) | |
| 1525 | f.has_build_zig = true; | |
| 1526 | ||
| 1527 | const fs_path = try arena.dupe(u8, entry.path); | |
| 1528 | const hashed_file = try arena.create(HashedFile); | |
| 1529 | hashed_file.* = .{ | |
| 1530 | .fs_path = fs_path, | |
| 1531 | .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), | |
| 1532 | .kind = kind, | |
| 1533 | .hash = undefined, // to be populated by the worker | |
| 1534 | .failure = undefined, // to be populated by the worker | |
| 1535 | .size = undefined, // to be populated by the worker | |
| 1536 | }; | |
| 1537 | thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file }); | |
| 1538 | try all_files.append(hashed_file); | |
| 1539 | } | |
| 1540 | } | |
| 1541 | ||
| 1542 | { | |
| 1543 | // Sort by length, descending, so that child directories get removed first. | |
| 1544 | sus_dirs.sortUnstable(@as(struct { | |
| 1545 | keys: []const []const u8, | |
| 1546 | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { | |
| 1547 | return ctx.keys[b_index].len < ctx.keys[a_index].len; | |
| 1548 | } | |
| 1549 | }, .{ .keys = sus_dirs.keys() })); | |
| 1550 | ||
| 1551 | // During this loop, more entries will be added, so we must loop by index. | |
| 1552 | var i: usize = 0; | |
| 1553 | while (i < sus_dirs.count()) : (i += 1) { | |
| 1554 | const sus_dir = sus_dirs.keys()[i]; | |
| 1555 | root_dir.deleteDir(sus_dir) catch |err| switch (err) { | |
| 1556 | error.DirNotEmpty => continue, | |
| 1557 | error.FileNotFound => continue, | |
| 1558 | else => |e| { | |
| 1559 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | |
| 1560 | "unable to delete empty directory '{s}': {s}", | |
| 1561 | .{ sus_dir, @errorName(e) }, | |
| 1562 | ) }); | |
| 1563 | return error.FetchFailed; | |
| 1564 | }, | |
| 1565 | }; | |
| 1566 | if (fs.path.dirname(sus_dir)) |parent| { | |
| 1567 | try sus_dirs.put(gpa, parent, {}); | |
| 1568 | } | |
| 1569 | } | |
| 1570 | } | |
| 1571 | ||
| 1572 | std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); | |
| 1573 | ||
| 1574 | var hasher = Package.Hash.Algo.init(.{}); | |
| 1575 | var any_failures = false; | |
| 1576 | for (all_files.items) |hashed_file| { | |
| 1577 | hashed_file.failure catch |err| { | |
| 1578 | any_failures = true; | |
| 1579 | try eb.addRootErrorMessage(.{ | |
| 1580 | .msg = try eb.printString("unable to hash '{s}': {s}", .{ | |
| 1581 | hashed_file.fs_path, @errorName(err), | |
| 1582 | }), | |
| 1583 | }); | |
| 1584 | }; | |
| 1585 | hasher.update(&hashed_file.hash); | |
| 1586 | total_size += hashed_file.size; | |
| 1587 | } | |
| 1588 | for (deleted_files.items) |deleted_file| { | |
| 1589 | deleted_file.failure catch |err| { | |
| 1590 | any_failures = true; | |
| 1591 | try eb.addRootErrorMessage(.{ | |
| 1592 | .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ | |
| 1593 | deleted_file.fs_path, @errorName(err), | |
| 1594 | }), | |
| 1595 | }); | |
| 1596 | }; | |
| 1597 | } | |
| 1598 | ||
| 1599 | if (any_failures) return error.FetchFailed; | |
| 1600 | ||
| 1601 | if (f.job_queue.debug_hash) { | |
| 1602 | assert(!f.job_queue.recursive); | |
| 1603 | // Print something to stdout that can be text diffed to figure out why | |
| 1604 | // the package hash is different. | |
| 1605 | dumpHashInfo(all_files.items) catch |err| { | |
| 1606 | std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)}); | |
| 1607 | std.process.exit(1); | |
| 1608 | }; | |
| 1609 | } | |
| 1610 | ||
| 1611 | return .{ | |
| 1612 | .digest = hasher.finalResult(), | |
| 1613 | .total_size = total_size, | |
| 1614 | }; | |
| 1615 | } | |
| 1616 | ||
| 1617 | fn dumpHashInfo(all_files: []const *const HashedFile) !void { | |
| 1618 | const stdout = std.io.getStdOut(); | |
| 1619 | var bw = std.io.bufferedWriter(stdout.writer()); | |
| 1620 | const w = bw.writer(); | |
| 1621 | ||
| 1622 | for (all_files) |hashed_file| { | |
| 1623 | try w.print("{s}: {s}: {s}\n", .{ | |
| 1624 | @tagName(hashed_file.kind), | |
| 1625 | std.fmt.fmtSliceHexLower(&hashed_file.hash), | |
| 1626 | hashed_file.normalized_path, | |
| 1627 | }); | |
| 1628 | } | |
| 1629 | ||
| 1630 | try bw.flush(); | |
| 1631 | } | |
| 1632 | ||
| 1633 | fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile) void { | |
| 1634 | hashed_file.failure = hashFileFallible(dir, hashed_file); | |
| 1635 | } | |
| 1636 | ||
| 1637 | fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void { | |
| 1638 | deleted_file.failure = deleteFileFallible(dir, deleted_file); | |
| 1639 | } | |
| 1640 | ||
| 1641 | fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { | |
| 1642 | var buf: [8000]u8 = undefined; | |
| 1643 | var hasher = Package.Hash.Algo.init(.{}); | |
| 1644 | hasher.update(hashed_file.normalized_path); | |
| 1645 | var file_size: u64 = 0; | |
| 1646 | ||
| 1647 | switch (hashed_file.kind) { | |
| 1648 | .file => { | |
| 1649 | var file = try dir.openFile(hashed_file.fs_path, .{}); | |
| 1650 | defer file.close(); | |
| 1651 | // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 | |
| 1652 | hasher.update(&.{ 0, 0 }); | |
| 1653 | var file_header: FileHeader = .{}; | |
| 1654 | while (true) { | |
| 1655 | const bytes_read = try file.read(&buf); | |
| 1656 | if (bytes_read == 0) break; | |
| 1657 | file_size += bytes_read; | |
| 1658 | hasher.update(buf[0..bytes_read]); | |
| 1659 | file_header.update(buf[0..bytes_read]); | |
| 1660 | } | |
| 1661 | if (file_header.isExecutable()) { | |
| 1662 | try setExecutable(file); | |
| 1663 | } | |
| 1664 | }, | |
| 1665 | .link => { | |
| 1666 | const link_name = try dir.readLink(hashed_file.fs_path, &buf); | |
| 1667 | if (fs.path.sep != canonical_sep) { | |
| 1668 | // Package hashes are intended to be consistent across | |
| 1669 | // platforms which means we must normalize path separators | |
| 1670 | // inside symlinks. | |
| 1671 | normalizePath(link_name); | |
| 1672 | } | |
| 1673 | hasher.update(link_name); | |
| 1674 | }, | |
| 1675 | } | |
| 1676 | hasher.final(&hashed_file.hash); | |
| 1677 | hashed_file.size = file_size; | |
| 1678 | } | |
| 1679 | ||
| 1680 | fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { | |
| 1681 | try dir.deleteFile(deleted_file.fs_path); | |
| 1682 | } | |
| 1683 | ||
| 1684 | fn setExecutable(file: fs.File) !void { | |
| 1685 | if (!std.fs.has_executable_bit) return; | |
| 1686 | ||
| 1687 | const S = std.posix.S; | |
| 1688 | const mode = fs.File.default_mode | S.IXUSR | S.IXGRP | S.IXOTH; | |
| 1689 | try file.chmod(mode); | |
| 1690 | } | |
| 1691 | ||
| 1692 | const DeletedFile = struct { | |
| 1693 | fs_path: []const u8, | |
| 1694 | failure: Error!void, | |
| 1695 | ||
| 1696 | const Error = | |
| 1697 | fs.Dir.DeleteFileError || | |
| 1698 | fs.Dir.DeleteDirError; | |
| 1699 | }; | |
| 1700 | ||
| 1701 | const HashedFile = struct { | |
| 1702 | fs_path: []const u8, | |
| 1703 | normalized_path: []const u8, | |
| 1704 | hash: Package.Hash.Digest, | |
| 1705 | failure: Error!void, | |
| 1706 | kind: Kind, | |
| 1707 | size: u64, | |
| 1708 | ||
| 1709 | const Error = | |
| 1710 | fs.File.OpenError || | |
| 1711 | fs.File.ReadError || | |
| 1712 | fs.File.StatError || | |
| 1713 | fs.File.ChmodError || | |
| 1714 | fs.Dir.ReadLinkError; | |
| 1715 | ||
| 1716 | const Kind = enum { file, link }; | |
| 1717 | ||
| 1718 | fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { | |
| 1719 | _ = context; | |
| 1720 | return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); | |
| 1721 | } | |
| 1722 | }; | |
| 1723 | ||
| 1724 | /// Strips root directory name from file system path. | |
| 1725 | fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { | |
| 1726 | if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; | |
| 1727 | ||
| 1728 | if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { | |
| 1729 | return fs_path[root_dir.len + 1 ..]; | |
| 1730 | } | |
| 1731 | ||
| 1732 | return fs_path; | |
| 1733 | } | |
| 1734 | ||
| 1735 | /// Make a file system path identical independently of operating system path inconsistencies. | |
| 1736 | /// This converts backslashes into forward slashes. | |
| 1737 | fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { | |
| 1738 | const normalized = try arena.dupe(u8, pkg_path); | |
| 1739 | if (fs.path.sep == canonical_sep) return normalized; | |
| 1740 | normalizePath(normalized); | |
| 1741 | return normalized; | |
| 1742 | } | |
| 1743 | ||
| 1744 | const canonical_sep = fs.path.sep_posix; | |
| 1745 | ||
| 1746 | fn normalizePath(bytes: []u8) void { | |
| 1747 | assert(fs.path.sep != canonical_sep); | |
| 1748 | std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); | |
| 1749 | } | |
| 1750 | ||
| 1751 | const Filter = struct { | |
| 1752 | include_paths: std.StringArrayHashMapUnmanaged(void) = .empty, | |
| 1753 | ||
| 1754 | /// sub_path is relative to the package root. | |
| 1755 | pub fn includePath(self: Filter, sub_path: []const u8) bool { | |
| 1756 | if (self.include_paths.count() == 0) return true; | |
| 1757 | if (self.include_paths.contains("")) return true; | |
| 1758 | if (self.include_paths.contains(".")) return true; | |
| 1759 | if (self.include_paths.contains(sub_path)) return true; | |
| 1760 | ||
| 1761 | // Check if any included paths are parent directories of sub_path. | |
| 1762 | var dirname = sub_path; | |
| 1763 | while (std.fs.path.dirname(dirname)) |next_dirname| { | |
| 1764 | if (self.include_paths.contains(next_dirname)) return true; | |
| 1765 | dirname = next_dirname; | |
| 1766 | } | |
| 1767 | ||
| 1768 | return false; | |
| 1769 | } | |
| 1770 | ||
| 1771 | test includePath { | |
| 1772 | const gpa = std.testing.allocator; | |
| 1773 | var filter: Filter = .{}; | |
| 1774 | defer filter.include_paths.deinit(gpa); | |
| 1775 | ||
| 1776 | try filter.include_paths.put(gpa, "src", {}); | |
| 1777 | try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); | |
| 1778 | try std.testing.expect(!filter.includePath(".gitignore")); | |
| 1779 | } | |
| 1780 | }; | |
| 1781 | ||
| 1782 | pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { | |
| 1783 | if (dep.hash) |h| return .fromSlice(h); | |
| 1784 | ||
| 1785 | switch (dep.location) { | |
| 1786 | .url => return null, | |
| 1787 | .path => |rel_path| { | |
| 1788 | var buf: [fs.max_path_bytes]u8 = undefined; | |
| 1789 | var fba = std.heap.FixedBufferAllocator.init(&buf); | |
| 1790 | const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch | |
| 1791 | return null; | |
| 1792 | return relativePathDigest(new_root, cache_root); | |
| 1793 | }, | |
| 1794 | } | |
| 1795 | } | |
| 1796 | ||
| 1797 | const builtin = @import("builtin"); | |
| 1798 | const std = @import("std"); | |
| 1799 | const fs = std.fs; | |
| 1800 | const assert = std.debug.assert; | |
| 1801 | const ascii = std.ascii; | |
| 1802 | const Allocator = std.mem.Allocator; | |
| 1803 | const Cache = std.Build.Cache; | |
| 1804 | const ThreadPool = std.Thread.Pool; | |
| 1805 | const WaitGroup = std.Thread.WaitGroup; | |
| 1806 | const Fetch = @This(); | |
| 1807 | const git = @import("Fetch/git.zig"); | |
| 1808 | const Package = @import("../Package.zig"); | |
| 1809 | const Manifest = Package.Manifest; | |
| 1810 | const ErrorBundle = std.zig.ErrorBundle; | |
| 1811 | const native_os = builtin.os.tag; | |
| 1812 | ||
| 1813 | test { | |
| 1814 | _ = Filter; | |
| 1815 | _ = FileType; | |
| 1816 | _ = UnpackResult; | |
| 1817 | } | |
| 1818 | ||
| 1819 | // Detects executable header: ELF or Macho-O magic header or shebang line. | |
| 1820 | const FileHeader = struct { | |
| 1821 | header: [4]u8 = undefined, | |
| 1822 | bytes_read: usize = 0, | |
| 1823 | ||
| 1824 | pub fn update(self: *FileHeader, buf: []const u8) void { | |
| 1825 | if (self.bytes_read >= self.header.len) return; | |
| 1826 | const n = @min(self.header.len - self.bytes_read, buf.len); | |
| 1827 | @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); | |
| 1828 | self.bytes_read += n; | |
| 1829 | } | |
| 1830 | ||
| 1831 | fn isScript(self: *FileHeader) bool { | |
| 1832 | const shebang = "#!"; | |
| 1833 | return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); | |
| 1834 | } | |
| 1835 | ||
| 1836 | fn isElf(self: *FileHeader) bool { | |
| 1837 | const elf_magic = std.elf.MAGIC; | |
| 1838 | return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); | |
| 1839 | } | |
| 1840 | ||
| 1841 | fn isMachO(self: *FileHeader) bool { | |
| 1842 | if (self.bytes_read < 4) return false; | |
| 1843 | const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); | |
| 1844 | return magic_number == std.macho.MH_MAGIC or | |
| 1845 | magic_number == std.macho.MH_MAGIC_64 or | |
| 1846 | magic_number == std.macho.FAT_MAGIC or | |
| 1847 | magic_number == std.macho.FAT_MAGIC_64 or | |
| 1848 | magic_number == std.macho.MH_CIGAM or | |
| 1849 | magic_number == std.macho.MH_CIGAM_64 or | |
| 1850 | magic_number == std.macho.FAT_CIGAM or | |
| 1851 | magic_number == std.macho.FAT_CIGAM_64; | |
| 1852 | } | |
| 1853 | ||
| 1854 | pub fn isExecutable(self: *FileHeader) bool { | |
| 1855 | return self.isScript() or self.isElf() or self.isMachO(); | |
| 1856 | } | |
| 1857 | }; | |
| 1858 | ||
| 1859 | test FileHeader { | |
| 1860 | var h: FileHeader = .{}; | |
| 1861 | try std.testing.expect(!h.isExecutable()); | |
| 1862 | ||
| 1863 | const elf_magic = std.elf.MAGIC; | |
| 1864 | h.update(elf_magic[0..2]); | |
| 1865 | try std.testing.expect(!h.isExecutable()); | |
| 1866 | h.update(elf_magic[2..4]); | |
| 1867 | try std.testing.expect(h.isExecutable()); | |
| 1868 | ||
| 1869 | h.update(elf_magic[2..4]); | |
| 1870 | try std.testing.expect(h.isExecutable()); | |
| 1871 | ||
| 1872 | const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; | |
| 1873 | h.bytes_read = 0; | |
| 1874 | h.update(&macho64_magic_bytes); | |
| 1875 | try std.testing.expect(h.isExecutable()); | |
| 1876 | ||
| 1877 | const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; | |
| 1878 | h.bytes_read = 0; | |
| 1879 | h.update(&macho64_cigam_bytes); | |
| 1880 | try std.testing.expect(h.isExecutable()); | |
| 1881 | } | |
| 1882 | ||
| 1883 | // Result of the `unpackResource` operation. Enables collecting errors from | |
| 1884 | // tar/git diagnostic, filtering that errors by manifest inclusion rules and | |
| 1885 | // emitting remaining errors to an `ErrorBundle`. | |
| 1886 | const UnpackResult = struct { | |
| 1887 | errors: []Error = undefined, | |
| 1888 | errors_count: usize = 0, | |
| 1889 | root_error_message: []const u8 = "", | |
| 1890 | ||
| 1891 | // A non empty value means that the package contents are inside a | |
| 1892 | // sub-directory indicated by the named path. | |
| 1893 | root_dir: []const u8 = "", | |
| 1894 | ||
| 1895 | const Error = union(enum) { | |
| 1896 | unable_to_create_sym_link: struct { | |
| 1897 | code: anyerror, | |
| 1898 | file_name: []const u8, | |
| 1899 | link_name: []const u8, | |
| 1900 | }, | |
| 1901 | unable_to_create_file: struct { | |
| 1902 | code: anyerror, | |
| 1903 | file_name: []const u8, | |
| 1904 | }, | |
| 1905 | unsupported_file_type: struct { | |
| 1906 | file_name: []const u8, | |
| 1907 | file_type: u8, | |
| 1908 | }, | |
| 1909 | ||
| 1910 | fn excluded(self: Error, filter: Filter) bool { | |
| 1911 | const file_name = switch (self) { | |
| 1912 | .unable_to_create_file => |info| info.file_name, | |
| 1913 | .unable_to_create_sym_link => |info| info.file_name, | |
| 1914 | .unsupported_file_type => |info| info.file_name, | |
| 1915 | }; | |
| 1916 | return !filter.includePath(file_name); | |
| 1917 | } | |
| 1918 | }; | |
| 1919 | ||
| 1920 | fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { | |
| 1921 | self.root_error_message = try arena.dupe(u8, root_error_message); | |
| 1922 | self.errors = try arena.alloc(UnpackResult.Error, n); | |
| 1923 | } | |
| 1924 | ||
| 1925 | fn hasErrors(self: *UnpackResult) bool { | |
| 1926 | return self.errors_count > 0; | |
| 1927 | } | |
| 1928 | ||
| 1929 | fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { | |
| 1930 | self.errors[self.errors_count] = .{ .unable_to_create_file = .{ | |
| 1931 | .code = err, | |
| 1932 | .file_name = file_name, | |
| 1933 | } }; | |
| 1934 | self.errors_count += 1; | |
| 1935 | } | |
| 1936 | ||
| 1937 | fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { | |
| 1938 | self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ | |
| 1939 | .code = err, | |
| 1940 | .file_name = file_name, | |
| 1941 | .link_name = link_name, | |
| 1942 | } }; | |
| 1943 | self.errors_count += 1; | |
| 1944 | } | |
| 1945 | ||
| 1946 | fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { | |
| 1947 | self.errors[self.errors_count] = .{ .unsupported_file_type = .{ | |
| 1948 | .file_name = file_name, | |
| 1949 | .file_type = file_type, | |
| 1950 | } }; | |
| 1951 | self.errors_count += 1; | |
| 1952 | } | |
| 1953 | ||
| 1954 | fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { | |
| 1955 | if (self.errors_count == 0) return; | |
| 1956 | ||
| 1957 | var unfiltered_errors: u32 = 0; | |
| 1958 | for (self.errors) |item| { | |
| 1959 | if (item.excluded(filter)) continue; | |
| 1960 | unfiltered_errors += 1; | |
| 1961 | } | |
| 1962 | if (unfiltered_errors == 0) return; | |
| 1963 | ||
| 1964 | // Emmit errors to an `ErrorBundle`. | |
| 1965 | const eb = &f.error_bundle; | |
| 1966 | try eb.addRootErrorMessage(.{ | |
| 1967 | .msg = try eb.addString(self.root_error_message), | |
| 1968 | .src_loc = try f.srcLoc(f.location_tok), | |
| 1969 | .notes_len = unfiltered_errors, | |
| 1970 | }); | |
| 1971 | var note_i: u32 = try eb.reserveNotes(unfiltered_errors); | |
| 1972 | for (self.errors) |item| { | |
| 1973 | if (item.excluded(filter)) continue; | |
| 1974 | switch (item) { | |
| 1975 | .unable_to_create_sym_link => |info| { | |
| 1976 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1977 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | |
| 1978 | info.file_name, info.link_name, @errorName(info.code), | |
| 1979 | }), | |
| 1980 | })); | |
| 1981 | }, | |
| 1982 | .unable_to_create_file => |info| { | |
| 1983 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1984 | .msg = try eb.printString("unable to create file '{s}': {s}", .{ | |
| 1985 | info.file_name, @errorName(info.code), | |
| 1986 | }), | |
| 1987 | })); | |
| 1988 | }, | |
| 1989 | .unsupported_file_type => |info| { | |
| 1990 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1991 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ | |
| 1992 | info.file_name, info.file_type, | |
| 1993 | }), | |
| 1994 | })); | |
| 1995 | }, | |
| 1996 | } | |
| 1997 | note_i += 1; | |
| 1998 | } | |
| 1999 | ||
| 2000 | return error.FetchFailed; | |
| 2001 | } | |
| 2002 | ||
| 2003 | test validate { | |
| 2004 | const gpa = std.testing.allocator; | |
| 2005 | var arena_instance = std.heap.ArenaAllocator.init(gpa); | |
| 2006 | defer arena_instance.deinit(); | |
| 2007 | const arena = arena_instance.allocator(); | |
| 2008 | ||
| 2009 | // fill UnpackResult with errors | |
| 2010 | var res: UnpackResult = .{}; | |
| 2011 | try res.allocErrors(arena, 4, "unable to unpack"); | |
| 2012 | try std.testing.expectEqual(0, res.errors_count); | |
| 2013 | res.unableToCreateFile("dir1/file1", error.File1); | |
| 2014 | res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); | |
| 2015 | res.unableToCreateFile("dir1/file3", error.File3); | |
| 2016 | res.unsupportedFileType("dir2/file4", 'x'); | |
| 2017 | try std.testing.expectEqual(4, res.errors_count); | |
| 2018 | ||
| 2019 | // create filter, includes dir2, excludes dir1 | |
| 2020 | var filter: Filter = .{}; | |
| 2021 | try filter.include_paths.put(arena, "dir2", {}); | |
| 2022 | ||
| 2023 | // init Fetch | |
| 2024 | var fetch: Fetch = undefined; | |
| 2025 | fetch.parent_manifest_ast = null; | |
| 2026 | fetch.location_tok = 0; | |
| 2027 | try fetch.error_bundle.init(gpa); | |
| 2028 | defer fetch.error_bundle.deinit(); | |
| 2029 | ||
| 2030 | // validate errors with filter | |
| 2031 | try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); | |
| 2032 | ||
| 2033 | // output errors to string | |
| 2034 | var errors = try fetch.error_bundle.toOwnedBundle(""); | |
| 2035 | defer errors.deinit(gpa); | |
| 2036 | var out = std.ArrayList(u8).init(gpa); | |
| 2037 | defer out.deinit(); | |
| 2038 | try errors.renderToWriter(.{ .ttyconf = .no_color }, out.writer()); | |
| 2039 | try std.testing.expectEqualStrings( | |
| 2040 | \\error: unable to unpack | |
| 2041 | \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError | |
| 2042 | \\ note: file 'dir2/file4' has unsupported type 'x' | |
| 2043 | \\ | |
| 2044 | , out.items); | |
| 2045 | } | |
| 2046 | }; | |
| 2047 | ||
| 2048 | test "zip" { | |
| 2049 | const gpa = std.testing.allocator; | |
| 2050 | var tmp = std.testing.tmpDir(.{}); | |
| 2051 | defer tmp.cleanup(); | |
| 2052 | ||
| 2053 | const test_files = [_]std.zip.testutil.File{ | |
| 2054 | .{ .name = "foo", .content = "this is just foo\n", .compression = .store }, | |
| 2055 | .{ .name = "bar", .content = "another file\n", .compression = .deflate }, | |
| 2056 | }; | |
| 2057 | { | |
| 2058 | var zip_file = try tmp.dir.createFile("test.zip", .{}); | |
| 2059 | defer zip_file.close(); | |
| 2060 | var bw = std.io.bufferedWriter(zip_file.writer()); | |
| 2061 | var store: [test_files.len]std.zip.testutil.FileStore = undefined; | |
| 2062 | try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); | |
| 2063 | try bw.flush(); | |
| 2064 | } | |
| 2065 | ||
| 2066 | const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); | |
| 2067 | defer gpa.free(zip_path); | |
| 2068 | ||
| 2069 | var fb: TestFetchBuilder = undefined; | |
| 2070 | var fetch = try fb.build(gpa, tmp.dir, zip_path); | |
| 2071 | defer fb.deinit(); | |
| 2072 | ||
| 2073 | try fetch.run(); | |
| 2074 | ||
| 2075 | var out = try fb.packageDir(); | |
| 2076 | defer out.close(); | |
| 2077 | ||
| 2078 | try std.zip.testutil.expectFiles(&test_files, out, .{}); | |
| 2079 | } | |
| 2080 | ||
| 2081 | test "zip with one root folder" { | |
| 2082 | const gpa = std.testing.allocator; | |
| 2083 | var tmp = std.testing.tmpDir(.{}); | |
| 2084 | defer tmp.cleanup(); | |
| 2085 | ||
| 2086 | const test_files = [_]std.zip.testutil.File{ | |
| 2087 | .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store }, | |
| 2088 | .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store }, | |
| 2089 | }; | |
| 2090 | { | |
| 2091 | var zip_file = try tmp.dir.createFile("test.zip", .{}); | |
| 2092 | defer zip_file.close(); | |
| 2093 | var bw = std.io.bufferedWriter(zip_file.writer()); | |
| 2094 | var store: [test_files.len]std.zip.testutil.FileStore = undefined; | |
| 2095 | try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); | |
| 2096 | try bw.flush(); | |
| 2097 | } | |
| 2098 | ||
| 2099 | const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); | |
| 2100 | defer gpa.free(zip_path); | |
| 2101 | ||
| 2102 | var fb: TestFetchBuilder = undefined; | |
| 2103 | var fetch = try fb.build(gpa, tmp.dir, zip_path); | |
| 2104 | defer fb.deinit(); | |
| 2105 | ||
| 2106 | try fetch.run(); | |
| 2107 | ||
| 2108 | var out = try fb.packageDir(); | |
| 2109 | defer out.close(); | |
| 2110 | ||
| 2111 | try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" }); | |
| 2112 | } | |
| 2113 | ||
| 2114 | test "tarball with duplicate paths" { | |
| 2115 | // This tarball has duplicate path 'dir1/file1' to simulate case sensitve | |
| 2116 | // file system on any file sytstem. | |
| 2117 | // | |
| 2118 | // duplicate_paths/ | |
| 2119 | // duplicate_paths/dir1/ | |
| 2120 | // duplicate_paths/dir1/file1 | |
| 2121 | // duplicate_paths/dir1/file1 | |
| 2122 | // duplicate_paths/build.zig.zon | |
| 2123 | // duplicate_paths/src/ | |
| 2124 | // duplicate_paths/src/main.zig | |
| 2125 | // duplicate_paths/src/root.zig | |
| 2126 | // duplicate_paths/build.zig | |
| 2127 | // | |
| 2128 | ||
| 2129 | const gpa = std.testing.allocator; | |
| 2130 | var tmp = std.testing.tmpDir(.{}); | |
| 2131 | defer tmp.cleanup(); | |
| 2132 | ||
| 2133 | const tarball_name = "duplicate_paths.tar.gz"; | |
| 2134 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2135 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2136 | defer gpa.free(tarball_path); | |
| 2137 | ||
| 2138 | // Run tarball fetch, expect to fail | |
| 2139 | var fb: TestFetchBuilder = undefined; | |
| 2140 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2141 | defer fb.deinit(); | |
| 2142 | try std.testing.expectError(error.FetchFailed, fetch.run()); | |
| 2143 | ||
| 2144 | try fb.expectFetchErrors(1, | |
| 2145 | \\error: unable to unpack tarball | |
| 2146 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists | |
| 2147 | \\ | |
| 2148 | ); | |
| 2149 | } | |
| 2150 | ||
| 2151 | test "tarball with excluded duplicate paths" { | |
| 2152 | // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. | |
| 2153 | // | |
| 2154 | // .paths = .{ | |
| 2155 | // "build.zig", | |
| 2156 | // "build.zig.zon", | |
| 2157 | // "src", | |
| 2158 | // } | |
| 2159 | // | |
| 2160 | ||
| 2161 | const gpa = std.testing.allocator; | |
| 2162 | var tmp = std.testing.tmpDir(.{}); | |
| 2163 | defer tmp.cleanup(); | |
| 2164 | ||
| 2165 | const tarball_name = "duplicate_paths_excluded.tar.gz"; | |
| 2166 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2167 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2168 | defer gpa.free(tarball_path); | |
| 2169 | ||
| 2170 | // Run tarball fetch, should succeed | |
| 2171 | var fb: TestFetchBuilder = undefined; | |
| 2172 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2173 | defer fb.deinit(); | |
| 2174 | try fetch.run(); | |
| 2175 | ||
| 2176 | const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); | |
| 2177 | try std.testing.expectEqualStrings( | |
| 2178 | "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", | |
| 2179 | &hex_digest, | |
| 2180 | ); | |
| 2181 | ||
| 2182 | const expected_files: []const []const u8 = &.{ | |
| 2183 | "build.zig", | |
| 2184 | "build.zig.zon", | |
| 2185 | "src/main.zig", | |
| 2186 | "src/root.zig", | |
| 2187 | }; | |
| 2188 | try fb.expectPackageFiles(expected_files); | |
| 2189 | } | |
| 2190 | ||
| 2191 | test "tarball without root folder" { | |
| 2192 | // Tarball with root folder. Manifest excludes dir1 and dir2. | |
| 2193 | // | |
| 2194 | // build.zig | |
| 2195 | // build.zig.zon | |
| 2196 | // dir1/ | |
| 2197 | // dir1/file2 | |
| 2198 | // dir1/file1 | |
| 2199 | // dir2/ | |
| 2200 | // dir2/file2 | |
| 2201 | // src/ | |
| 2202 | // src/main.zig | |
| 2203 | // | |
| 2204 | ||
| 2205 | const gpa = std.testing.allocator; | |
| 2206 | var tmp = std.testing.tmpDir(.{}); | |
| 2207 | defer tmp.cleanup(); | |
| 2208 | ||
| 2209 | const tarball_name = "no_root.tar.gz"; | |
| 2210 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2211 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2212 | defer gpa.free(tarball_path); | |
| 2213 | ||
| 2214 | // Run tarball fetch, should succeed | |
| 2215 | var fb: TestFetchBuilder = undefined; | |
| 2216 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2217 | defer fb.deinit(); | |
| 2218 | try fetch.run(); | |
| 2219 | ||
| 2220 | const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); | |
| 2221 | try std.testing.expectEqualStrings( | |
| 2222 | "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", | |
| 2223 | &hex_digest, | |
| 2224 | ); | |
| 2225 | ||
| 2226 | const expected_files: []const []const u8 = &.{ | |
| 2227 | "build.zig", | |
| 2228 | "build.zig.zon", | |
| 2229 | "src/main.zig", | |
| 2230 | }; | |
| 2231 | try fb.expectPackageFiles(expected_files); | |
| 2232 | } | |
| 2233 | ||
| 2234 | test "set executable bit based on file content" { | |
| 2235 | if (!std.fs.has_executable_bit) return error.SkipZigTest; | |
| 2236 | const gpa = std.testing.allocator; | |
| 2237 | var tmp = std.testing.tmpDir(.{}); | |
| 2238 | defer tmp.cleanup(); | |
| 2239 | ||
| 2240 | const tarball_name = "executables.tar.gz"; | |
| 2241 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2242 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2243 | defer gpa.free(tarball_path); | |
| 2244 | ||
| 2245 | // $ tar -tvf executables.tar.gz | |
| 2246 | // drwxrwxr-x 0 executables/ | |
| 2247 | // -rwxrwxr-x 170 executables/hello | |
| 2248 | // lrwxrwxrwx 0 executables/hello_ln -> hello | |
| 2249 | // -rw-rw-r-- 0 executables/file1 | |
| 2250 | // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit | |
| 2251 | // -rwxrwxr-x 7 executables/script_without_shebang | |
| 2252 | // -rwxrwxr-x 17 executables/script | |
| 2253 | ||
| 2254 | var fb: TestFetchBuilder = undefined; | |
| 2255 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2256 | defer fb.deinit(); | |
| 2257 | ||
| 2258 | try fetch.run(); | |
| 2259 | try std.testing.expectEqualStrings( | |
| 2260 | "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", | |
| 2261 | &Package.multiHashHexDigest(fetch.computed_hash.digest), | |
| 2262 | ); | |
| 2263 | ||
| 2264 | var out = try fb.packageDir(); | |
| 2265 | defer out.close(); | |
| 2266 | const S = std.posix.S; | |
| 2267 | // expect executable bit not set | |
| 2268 | try std.testing.expect((try out.statFile("file1")).mode & S.IXUSR == 0); | |
| 2269 | try std.testing.expect((try out.statFile("script_without_shebang")).mode & S.IXUSR == 0); | |
| 2270 | // expect executable bit set | |
| 2271 | try std.testing.expect((try out.statFile("hello")).mode & S.IXUSR != 0); | |
| 2272 | try std.testing.expect((try out.statFile("script")).mode & S.IXUSR != 0); | |
| 2273 | try std.testing.expect((try out.statFile("script_with_shebang_without_exec_bit")).mode & S.IXUSR != 0); | |
| 2274 | try std.testing.expect((try out.statFile("hello_ln")).mode & S.IXUSR != 0); | |
| 2275 | ||
| 2276 | // | |
| 2277 | // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3 | |
| 2278 | // -rw-rw-r-- 1 0 Apr file1 | |
| 2279 | // -rwxrwxr-x 1 170 Apr hello | |
| 2280 | // lrwxrwxrwx 1 5 Apr hello_ln -> hello | |
| 2281 | // -rwxrwxr-x 1 17 Apr script | |
| 2282 | // -rw-rw-r-- 1 7 Apr script_without_shebang | |
| 2283 | // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit | |
| 2284 | } | |
| 2285 | ||
| 2286 | fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void { | |
| 2287 | //const tarball_name = "duplicate_paths_excluded.tar.gz"; | |
| 2288 | const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name); | |
| 2289 | var tmp_file = try dir.createFile(tarball_name, .{}); | |
| 2290 | defer tmp_file.close(); | |
| 2291 | try tmp_file.writeAll(tarball_content); | |
| 2292 | } | |
| 2293 | ||
| 2294 | // Builds Fetch with required dependencies, clears dependencies on deinit(). | |
| 2295 | const TestFetchBuilder = struct { | |
| 2296 | thread_pool: ThreadPool, | |
| 2297 | http_client: std.http.Client, | |
| 2298 | global_cache_directory: Cache.Directory, | |
| 2299 | job_queue: Fetch.JobQueue, | |
| 2300 | fetch: Fetch, | |
| 2301 | ||
| 2302 | fn build( | |
| 2303 | self: *TestFetchBuilder, | |
| 2304 | allocator: std.mem.Allocator, | |
| 2305 | cache_parent_dir: std.fs.Dir, | |
| 2306 | path_or_url: []const u8, | |
| 2307 | ) !*Fetch { | |
| 2308 | const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{}); | |
| 2309 | ||
| 2310 | try self.thread_pool.init(.{ .allocator = allocator }); | |
| 2311 | self.http_client = .{ .allocator = allocator }; | |
| 2312 | self.global_cache_directory = .{ .handle = cache_dir, .path = null }; | |
| 2313 | ||
| 2314 | self.job_queue = .{ | |
| 2315 | .http_client = &self.http_client, | |
| 2316 | .thread_pool = &self.thread_pool, | |
| 2317 | .global_cache = self.global_cache_directory, | |
| 2318 | .recursive = false, | |
| 2319 | .read_only = false, | |
| 2320 | .debug_hash = false, | |
| 2321 | .work_around_btrfs_bug = false, | |
| 2322 | }; | |
| 2323 | ||
| 2324 | self.fetch = .{ | |
| 2325 | .arena = std.heap.ArenaAllocator.init(allocator), | |
| 2326 | .location = .{ .path_or_url = path_or_url }, | |
| 2327 | .location_tok = 0, | |
| 2328 | .hash_tok = .none, | |
| 2329 | .name_tok = 0, | |
| 2330 | .lazy_status = .eager, | |
| 2331 | .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, | |
| 2332 | .parent_manifest_ast = null, | |
| 2333 | .prog_node = std.Progress.Node.none, | |
| 2334 | .job_queue = &self.job_queue, | |
| 2335 | .omit_missing_hash_error = true, | |
| 2336 | .allow_missing_paths_field = false, | |
| 2337 | .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz | |
| 2338 | .allow_name_string = true, // so we can keep using the old testdata .tar.gz | |
| 2339 | .use_latest_commit = true, | |
| 2340 | ||
| 2341 | .package_root = undefined, | |
| 2342 | .error_bundle = undefined, | |
| 2343 | .manifest = null, | |
| 2344 | .manifest_ast = undefined, | |
| 2345 | .computed_hash = undefined, | |
| 2346 | .has_build_zig = false, | |
| 2347 | .oom_flag = false, | |
| 2348 | .latest_commit = null, | |
| 2349 | }; | |
| 2350 | return &self.fetch; | |
| 2351 | } | |
| 2352 | ||
| 2353 | fn deinit(self: *TestFetchBuilder) void { | |
| 2354 | self.fetch.deinit(); | |
| 2355 | self.job_queue.deinit(); | |
| 2356 | self.fetch.prog_node.end(); | |
| 2357 | self.global_cache_directory.handle.close(); | |
| 2358 | self.http_client.deinit(); | |
| 2359 | self.thread_pool.deinit(); | |
| 2360 | } | |
| 2361 | ||
| 2362 | fn packageDir(self: *TestFetchBuilder) !fs.Dir { | |
| 2363 | const root = self.fetch.package_root; | |
| 2364 | return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); | |
| 2365 | } | |
| 2366 | ||
| 2367 | // Test helper, asserts thet package dir constains expected_files. | |
| 2368 | // expected_files must be sorted. | |
| 2369 | fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { | |
| 2370 | var package_dir = try self.packageDir(); | |
| 2371 | defer package_dir.close(); | |
| 2372 | ||
| 2373 | var actual_files: std.ArrayListUnmanaged([]u8) = .empty; | |
| 2374 | defer actual_files.deinit(std.testing.allocator); | |
| 2375 | defer for (actual_files.items) |file| std.testing.allocator.free(file); | |
| 2376 | var walker = try package_dir.walk(std.testing.allocator); | |
| 2377 | defer walker.deinit(); | |
| 2378 | while (try walker.next()) |entry| { | |
| 2379 | if (entry.kind != .file) continue; | |
| 2380 | const path = try std.testing.allocator.dupe(u8, entry.path); | |
| 2381 | errdefer std.testing.allocator.free(path); | |
| 2382 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); | |
| 2383 | try actual_files.append(std.testing.allocator, path); | |
| 2384 | } | |
| 2385 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { | |
| 2386 | fn lessThan(_: void, a: []u8, b: []u8) bool { | |
| 2387 | return std.mem.lessThan(u8, a, b); | |
| 2388 | } | |
| 2389 | }.lessThan); | |
| 2390 | ||
| 2391 | try std.testing.expectEqual(expected_files.len, actual_files.items.len); | |
| 2392 | for (expected_files, 0..) |file_name, i| { | |
| 2393 | try std.testing.expectEqualStrings(file_name, actual_files.items[i]); | |
| 2394 | } | |
| 2395 | try std.testing.expectEqualDeep(expected_files, actual_files.items); | |
| 2396 | } | |
| 2397 | ||
| 2398 | // Test helper, asserts that fetch has failed with `msg` error message. | |
| 2399 | fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { | |
| 2400 | var errors = try self.fetch.error_bundle.toOwnedBundle(""); | |
| 2401 | defer errors.deinit(std.testing.allocator); | |
| 2402 | ||
| 2403 | const em = errors.getErrorMessage(errors.getMessages()[0]); | |
| 2404 | try std.testing.expectEqual(1, em.count); | |
| 2405 | if (notes_len > 0) { | |
| 2406 | try std.testing.expectEqual(notes_len, em.notes_len); | |
| 2407 | } | |
| 2408 | var al = std.ArrayList(u8).init(std.testing.allocator); | |
| 2409 | defer al.deinit(); | |
| 2410 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); | |
| 2411 | try std.testing.expectEqualStrings(msg, al.items); | |
| 2412 | } | |
| 2413 | }; |
lib/std/zig/Package/Fetch/git.zig created+1689
| ... | ... | @@ -0,0 +1,1689 @@ |
| 1 | //! Git support for package fetching. | |
| 2 | //! | |
| 3 | //! This is not intended to support all features of Git: it is limited to the | |
| 4 | //! basic functionality needed to clone a repository for the purpose of fetching | |
| 5 | //! a package. | |
| 6 | ||
| 7 | const std = @import("std"); | |
| 8 | const mem = std.mem; | |
| 9 | const testing = std.testing; | |
| 10 | const Allocator = mem.Allocator; | |
| 11 | const Sha1 = std.crypto.hash.Sha1; | |
| 12 | const Sha256 = std.crypto.hash.sha2.Sha256; | |
| 13 | const assert = std.debug.assert; | |
| 14 | ||
| 15 | /// The ID of a Git object. | |
| 16 | pub const Oid = union(Format) { | |
| 17 | sha1: [Sha1.digest_length]u8, | |
| 18 | sha256: [Sha256.digest_length]u8, | |
| 19 | ||
| 20 | pub const max_formatted_length = len: { | |
| 21 | var max: usize = 0; | |
| 22 | for (std.enums.values(Format)) |f| { | |
| 23 | max = @max(max, f.formattedLength()); | |
| 24 | } | |
| 25 | break :len max; | |
| 26 | }; | |
| 27 | ||
| 28 | pub const Format = enum { | |
| 29 | sha1, | |
| 30 | sha256, | |
| 31 | ||
| 32 | pub fn byteLength(f: Format) usize { | |
| 33 | return switch (f) { | |
| 34 | .sha1 => Sha1.digest_length, | |
| 35 | .sha256 => Sha256.digest_length, | |
| 36 | }; | |
| 37 | } | |
| 38 | ||
| 39 | pub fn formattedLength(f: Format) usize { | |
| 40 | return 2 * f.byteLength(); | |
| 41 | } | |
| 42 | }; | |
| 43 | ||
| 44 | const Hasher = union(Format) { | |
| 45 | sha1: Sha1, | |
| 46 | sha256: Sha256, | |
| 47 | ||
| 48 | fn init(oid_format: Format) Hasher { | |
| 49 | return switch (oid_format) { | |
| 50 | .sha1 => .{ .sha1 = Sha1.init(.{}) }, | |
| 51 | .sha256 => .{ .sha256 = Sha256.init(.{}) }, | |
| 52 | }; | |
| 53 | } | |
| 54 | ||
| 55 | // Must be public for use from HashedReader and HashedWriter. | |
| 56 | pub fn update(hasher: *Hasher, b: []const u8) void { | |
| 57 | switch (hasher.*) { | |
| 58 | inline else => |*inner| inner.update(b), | |
| 59 | } | |
| 60 | } | |
| 61 | ||
| 62 | fn finalResult(hasher: *Hasher) Oid { | |
| 63 | return switch (hasher.*) { | |
| 64 | inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), | |
| 65 | }; | |
| 66 | } | |
| 67 | }; | |
| 68 | ||
| 69 | pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { | |
| 70 | assert(bytes.len == oid_format.byteLength()); | |
| 71 | return switch (oid_format) { | |
| 72 | inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), | |
| 73 | }; | |
| 74 | } | |
| 75 | ||
| 76 | pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { | |
| 77 | return switch (oid_format) { | |
| 78 | inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), | |
| 79 | }; | |
| 80 | } | |
| 81 | ||
| 82 | pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { | |
| 83 | switch (oid_format) { | |
| 84 | inline else => |tag| { | |
| 85 | if (s.len != tag.formattedLength()) return error.InvalidOid; | |
| 86 | var bytes: [tag.byteLength()]u8 = undefined; | |
| 87 | for (&bytes, 0..) |*b, i| { | |
| 88 | b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; | |
| 89 | } | |
| 90 | return @unionInit(Oid, @tagName(tag), bytes); | |
| 91 | }, | |
| 92 | } | |
| 93 | } | |
| 94 | ||
| 95 | test parse { | |
| 96 | try testing.expectEqualSlices( | |
| 97 | u8, | |
| 98 | &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, | |
| 99 | &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, | |
| 100 | ); | |
| 101 | try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); | |
| 102 | try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); | |
| 103 | try testing.expectEqualSlices( | |
| 104 | u8, | |
| 105 | &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, | |
| 106 | &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, | |
| 107 | ); | |
| 108 | try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); | |
| 109 | try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); | |
| 110 | try testing.expectError(error.InvalidOid, parse(.sha1, "master")); | |
| 111 | try testing.expectError(error.InvalidOid, parse(.sha256, "master")); | |
| 112 | try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); | |
| 113 | try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); | |
| 114 | } | |
| 115 | ||
| 116 | pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { | |
| 117 | return for (std.enums.values(Format)) |f| { | |
| 118 | if (s.len == f.formattedLength()) break parse(f, s); | |
| 119 | } else error.InvalidOid; | |
| 120 | } | |
| 121 | ||
| 122 | pub fn format( | |
| 123 | oid: Oid, | |
| 124 | comptime fmt: []const u8, | |
| 125 | options: std.fmt.FormatOptions, | |
| 126 | writer: anytype, | |
| 127 | ) @TypeOf(writer).Error!void { | |
| 128 | _ = fmt; | |
| 129 | _ = options; | |
| 130 | try writer.print("{}", .{std.fmt.fmtSliceHexLower(oid.slice())}); | |
| 131 | } | |
| 132 | ||
| 133 | pub fn slice(oid: *const Oid) []const u8 { | |
| 134 | return switch (oid.*) { | |
| 135 | inline else => |*bytes| bytes, | |
| 136 | }; | |
| 137 | } | |
| 138 | }; | |
| 139 | ||
| 140 | pub const Diagnostics = struct { | |
| 141 | allocator: Allocator, | |
| 142 | errors: std.ArrayListUnmanaged(Error) = .empty, | |
| 143 | ||
| 144 | pub const Error = union(enum) { | |
| 145 | unable_to_create_sym_link: struct { | |
| 146 | code: anyerror, | |
| 147 | file_name: []const u8, | |
| 148 | link_name: []const u8, | |
| 149 | }, | |
| 150 | unable_to_create_file: struct { | |
| 151 | code: anyerror, | |
| 152 | file_name: []const u8, | |
| 153 | }, | |
| 154 | }; | |
| 155 | ||
| 156 | pub fn deinit(d: *Diagnostics) void { | |
| 157 | for (d.errors.items) |item| { | |
| 158 | switch (item) { | |
| 159 | .unable_to_create_sym_link => |info| { | |
| 160 | d.allocator.free(info.file_name); | |
| 161 | d.allocator.free(info.link_name); | |
| 162 | }, | |
| 163 | .unable_to_create_file => |info| { | |
| 164 | d.allocator.free(info.file_name); | |
| 165 | }, | |
| 166 | } | |
| 167 | } | |
| 168 | d.errors.deinit(d.allocator); | |
| 169 | d.* = undefined; | |
| 170 | } | |
| 171 | }; | |
| 172 | ||
| 173 | pub const Repository = struct { | |
| 174 | odb: Odb, | |
| 175 | ||
| 176 | pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { | |
| 177 | return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; | |
| 178 | } | |
| 179 | ||
| 180 | pub fn deinit(repository: *Repository) void { | |
| 181 | repository.odb.deinit(); | |
| 182 | repository.* = undefined; | |
| 183 | } | |
| 184 | ||
| 185 | /// Checks out the repository at `commit_oid` to `worktree`. | |
| 186 | pub fn checkout( | |
| 187 | repository: *Repository, | |
| 188 | worktree: std.fs.Dir, | |
| 189 | commit_oid: Oid, | |
| 190 | diagnostics: *Diagnostics, | |
| 191 | ) !void { | |
| 192 | try repository.odb.seekOid(commit_oid); | |
| 193 | const tree_oid = tree_oid: { | |
| 194 | const commit_object = try repository.odb.readObject(); | |
| 195 | if (commit_object.type != .commit) return error.NotACommit; | |
| 196 | break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); | |
| 197 | }; | |
| 198 | try repository.checkoutTree(worktree, tree_oid, "", diagnostics); | |
| 199 | } | |
| 200 | ||
| 201 | /// Checks out the tree at `tree_oid` to `worktree`. | |
| 202 | fn checkoutTree( | |
| 203 | repository: *Repository, | |
| 204 | dir: std.fs.Dir, | |
| 205 | tree_oid: Oid, | |
| 206 | current_path: []const u8, | |
| 207 | diagnostics: *Diagnostics, | |
| 208 | ) !void { | |
| 209 | try repository.odb.seekOid(tree_oid); | |
| 210 | const tree_object = try repository.odb.readObject(); | |
| 211 | if (tree_object.type != .tree) return error.NotATree; | |
| 212 | // The tree object may be evicted from the object cache while we're | |
| 213 | // iterating over it, so we can make a defensive copy here to make sure | |
| 214 | // it remains valid until we're done with it | |
| 215 | const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); | |
| 216 | defer repository.odb.allocator.free(tree_data); | |
| 217 | ||
| 218 | var tree_iter: TreeIterator = .{ | |
| 219 | .format = repository.odb.format, | |
| 220 | .data = tree_data, | |
| 221 | .pos = 0, | |
| 222 | }; | |
| 223 | while (try tree_iter.next()) |entry| { | |
| 224 | switch (entry.type) { | |
| 225 | .directory => { | |
| 226 | try dir.makeDir(entry.name); | |
| 227 | var subdir = try dir.openDir(entry.name, .{}); | |
| 228 | defer subdir.close(); | |
| 229 | const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); | |
| 230 | defer repository.odb.allocator.free(sub_path); | |
| 231 | try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics); | |
| 232 | }, | |
| 233 | .file => { | |
| 234 | try repository.odb.seekOid(entry.oid); | |
| 235 | const file_object = try repository.odb.readObject(); | |
| 236 | if (file_object.type != .blob) return error.InvalidFile; | |
| 237 | var file = dir.createFile(entry.name, .{ .exclusive = true }) catch |e| { | |
| 238 | const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); | |
| 239 | errdefer diagnostics.allocator.free(file_name); | |
| 240 | try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ | |
| 241 | .code = e, | |
| 242 | .file_name = file_name, | |
| 243 | } }); | |
| 244 | continue; | |
| 245 | }; | |
| 246 | defer file.close(); | |
| 247 | try file.writeAll(file_object.data); | |
| 248 | try file.sync(); | |
| 249 | }, | |
| 250 | .symlink => { | |
| 251 | try repository.odb.seekOid(entry.oid); | |
| 252 | const symlink_object = try repository.odb.readObject(); | |
| 253 | if (symlink_object.type != .blob) return error.InvalidFile; | |
| 254 | const link_name = symlink_object.data; | |
| 255 | dir.symLink(link_name, entry.name, .{}) catch |e| { | |
| 256 | const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); | |
| 257 | errdefer diagnostics.allocator.free(file_name); | |
| 258 | const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); | |
| 259 | errdefer diagnostics.allocator.free(link_name_dup); | |
| 260 | try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ | |
| 261 | .code = e, | |
| 262 | .file_name = file_name, | |
| 263 | .link_name = link_name_dup, | |
| 264 | } }); | |
| 265 | }; | |
| 266 | }, | |
| 267 | .gitlink => { | |
| 268 | // Consistent with git archive behavior, create the directory but | |
| 269 | // do nothing else | |
| 270 | try dir.makeDir(entry.name); | |
| 271 | }, | |
| 272 | } | |
| 273 | } | |
| 274 | } | |
| 275 | ||
| 276 | /// Returns the ID of the tree associated with the given commit (provided as | |
| 277 | /// raw object data). | |
| 278 | fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { | |
| 279 | if (!mem.startsWith(u8, commit_data, "tree ") or | |
| 280 | commit_data.len < "tree ".len + format.formattedLength() + "\n".len or | |
| 281 | commit_data["tree ".len + format.formattedLength()] != '\n') | |
| 282 | { | |
| 283 | return error.InvalidCommit; | |
| 284 | } | |
| 285 | return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); | |
| 286 | } | |
| 287 | ||
| 288 | const TreeIterator = struct { | |
| 289 | format: Oid.Format, | |
| 290 | data: []const u8, | |
| 291 | pos: usize, | |
| 292 | ||
| 293 | const Entry = struct { | |
| 294 | type: Type, | |
| 295 | executable: bool, | |
| 296 | name: [:0]const u8, | |
| 297 | oid: Oid, | |
| 298 | ||
| 299 | const Type = enum(u4) { | |
| 300 | directory = 0o4, | |
| 301 | file = 0o10, | |
| 302 | symlink = 0o12, | |
| 303 | gitlink = 0o16, | |
| 304 | }; | |
| 305 | }; | |
| 306 | ||
| 307 | fn next(iterator: *TreeIterator) !?Entry { | |
| 308 | if (iterator.pos == iterator.data.len) return null; | |
| 309 | ||
| 310 | const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; | |
| 311 | const mode: packed struct { | |
| 312 | permission: u9, | |
| 313 | unused: u3, | |
| 314 | type: u4, | |
| 315 | } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); | |
| 316 | const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; | |
| 317 | const executable = switch (mode.permission) { | |
| 318 | 0 => if (@"type" == .file) return error.InvalidTree else false, | |
| 319 | 0o644 => if (@"type" != .file) return error.InvalidTree else false, | |
| 320 | 0o755 => if (@"type" != .file) return error.InvalidTree else true, | |
| 321 | else => return error.InvalidTree, | |
| 322 | }; | |
| 323 | iterator.pos = mode_end + 1; | |
| 324 | ||
| 325 | const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; | |
| 326 | const name = iterator.data[iterator.pos..name_end :0]; | |
| 327 | iterator.pos = name_end + 1; | |
| 328 | ||
| 329 | const oid_length = iterator.format.byteLength(); | |
| 330 | if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; | |
| 331 | const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); | |
| 332 | iterator.pos += oid_length; | |
| 333 | ||
| 334 | return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; | |
| 335 | } | |
| 336 | }; | |
| 337 | }; | |
| 338 | ||
| 339 | /// A Git object database backed by a packfile. A packfile index is also used | |
| 340 | /// for efficient access to objects in the packfile. | |
| 341 | /// | |
| 342 | /// The format of the packfile and its associated index are documented in | |
| 343 | /// [pack-format](https://git-scm.com/docs/pack-format). | |
| 344 | const Odb = struct { | |
| 345 | format: Oid.Format, | |
| 346 | pack_file: std.fs.File, | |
| 347 | index_header: IndexHeader, | |
| 348 | index_file: std.fs.File, | |
| 349 | cache: ObjectCache = .{}, | |
| 350 | allocator: Allocator, | |
| 351 | ||
| 352 | /// Initializes the database from open pack and index files. | |
| 353 | fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { | |
| 354 | try pack_file.seekTo(0); | |
| 355 | try index_file.seekTo(0); | |
| 356 | const index_header = try IndexHeader.read(index_file.reader()); | |
| 357 | return .{ | |
| 358 | .format = format, | |
| 359 | .pack_file = pack_file, | |
| 360 | .index_header = index_header, | |
| 361 | .index_file = index_file, | |
| 362 | .allocator = allocator, | |
| 363 | }; | |
| 364 | } | |
| 365 | ||
| 366 | fn deinit(odb: *Odb) void { | |
| 367 | odb.cache.deinit(odb.allocator); | |
| 368 | odb.* = undefined; | |
| 369 | } | |
| 370 | ||
| 371 | /// Reads the object at the current position in the database. | |
| 372 | fn readObject(odb: *Odb) !Object { | |
| 373 | var base_offset = try odb.pack_file.getPos(); | |
| 374 | var base_header: EntryHeader = undefined; | |
| 375 | var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; | |
| 376 | defer delta_offsets.deinit(odb.allocator); | |
| 377 | const base_object = while (true) { | |
| 378 | if (odb.cache.get(base_offset)) |base_object| break base_object; | |
| 379 | ||
| 380 | base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); | |
| 381 | switch (base_header) { | |
| 382 | .ofs_delta => |ofs_delta| { | |
| 383 | try delta_offsets.append(odb.allocator, base_offset); | |
| 384 | base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; | |
| 385 | try odb.pack_file.seekTo(base_offset); | |
| 386 | }, | |
| 387 | .ref_delta => |ref_delta| { | |
| 388 | try delta_offsets.append(odb.allocator, base_offset); | |
| 389 | try odb.seekOid(ref_delta.base_object); | |
| 390 | base_offset = try odb.pack_file.getPos(); | |
| 391 | }, | |
| 392 | else => { | |
| 393 | const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); | |
| 394 | errdefer odb.allocator.free(base_data); | |
| 395 | const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; | |
| 396 | try odb.cache.put(odb.allocator, base_offset, base_object); | |
| 397 | break base_object; | |
| 398 | }, | |
| 399 | } | |
| 400 | }; | |
| 401 | ||
| 402 | const base_data = try resolveDeltaChain( | |
| 403 | odb.allocator, | |
| 404 | odb.format, | |
| 405 | odb.pack_file, | |
| 406 | base_object, | |
| 407 | delta_offsets.items, | |
| 408 | &odb.cache, | |
| 409 | ); | |
| 410 | ||
| 411 | return .{ .type = base_object.type, .data = base_data }; | |
| 412 | } | |
| 413 | ||
| 414 | /// Seeks to the beginning of the object with the given ID. | |
| 415 | fn seekOid(odb: *Odb, oid: Oid) !void { | |
| 416 | const oid_length = odb.format.byteLength(); | |
| 417 | const key = oid.slice()[0]; | |
| 418 | var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; | |
| 419 | var end_index = odb.index_header.fan_out_table[key]; | |
| 420 | const found_index = while (start_index < end_index) { | |
| 421 | const mid_index = start_index + (end_index - start_index) / 2; | |
| 422 | try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); | |
| 423 | const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); | |
| 424 | switch (mem.order(u8, mid_oid.slice(), oid.slice())) { | |
| 425 | .lt => start_index = mid_index + 1, | |
| 426 | .gt => end_index = mid_index, | |
| 427 | .eq => break mid_index, | |
| 428 | } | |
| 429 | } else return error.ObjectNotFound; | |
| 430 | ||
| 431 | const n_objects = odb.index_header.fan_out_table[255]; | |
| 432 | const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); | |
| 433 | try odb.index_file.seekTo(offset_values_start + found_index * 4); | |
| 434 | const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big)); | |
| 435 | const pack_offset = pack_offset: { | |
| 436 | if (l1_offset.big) { | |
| 437 | const l2_offset_values_start = offset_values_start + n_objects * 4; | |
| 438 | try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); | |
| 439 | break :pack_offset try odb.index_file.reader().readInt(u64, .big); | |
| 440 | } else { | |
| 441 | break :pack_offset l1_offset.value; | |
| 442 | } | |
| 443 | }; | |
| 444 | ||
| 445 | try odb.pack_file.seekTo(pack_offset); | |
| 446 | } | |
| 447 | }; | |
| 448 | ||
| 449 | const Object = struct { | |
| 450 | type: Type, | |
| 451 | data: []const u8, | |
| 452 | ||
| 453 | const Type = enum { | |
| 454 | commit, | |
| 455 | tree, | |
| 456 | blob, | |
| 457 | tag, | |
| 458 | }; | |
| 459 | }; | |
| 460 | ||
| 461 | /// A cache for object data. | |
| 462 | /// | |
| 463 | /// The purpose of this cache is to speed up resolution of deltas by caching the | |
| 464 | /// results of resolving delta objects, while maintaining a maximum cache size | |
| 465 | /// to avoid excessive memory usage. If the total size of the objects in the | |
| 466 | /// cache exceeds the maximum, the cache will begin evicting the least recently | |
| 467 | /// used objects: when resolving delta chains, the most recently used objects | |
| 468 | /// will likely be more helpful as they will be further along in the chain | |
| 469 | /// (skipping earlier reconstruction steps). | |
| 470 | /// | |
| 471 | /// Object data stored in the cache is managed by the cache. It should not be | |
| 472 | /// freed by the caller at any point after inserting it into the cache. Any | |
| 473 | /// objects remaining in the cache will be freed when the cache itself is freed. | |
| 474 | const ObjectCache = struct { | |
| 475 | objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, | |
| 476 | lru_nodes: LruList = .{}, | |
| 477 | byte_size: usize = 0, | |
| 478 | ||
| 479 | const max_byte_size = 128 * 1024 * 1024; // 128MiB | |
| 480 | /// A list of offsets stored in the cache, with the most recently used | |
| 481 | /// entries at the end. | |
| 482 | const LruList = std.DoublyLinkedList(u64); | |
| 483 | const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; | |
| 484 | ||
| 485 | fn deinit(cache: *ObjectCache, allocator: Allocator) void { | |
| 486 | var object_iterator = cache.objects.iterator(); | |
| 487 | while (object_iterator.next()) |object| { | |
| 488 | allocator.free(object.value_ptr.object.data); | |
| 489 | allocator.destroy(object.value_ptr.lru_node); | |
| 490 | } | |
| 491 | cache.objects.deinit(allocator); | |
| 492 | cache.* = undefined; | |
| 493 | } | |
| 494 | ||
| 495 | /// Gets an object from the cache, moving it to the most recently used | |
| 496 | /// position if it is present. | |
| 497 | fn get(cache: *ObjectCache, offset: u64) ?Object { | |
| 498 | if (cache.objects.get(offset)) |entry| { | |
| 499 | cache.lru_nodes.remove(entry.lru_node); | |
| 500 | cache.lru_nodes.append(entry.lru_node); | |
| 501 | return entry.object; | |
| 502 | } else { | |
| 503 | return null; | |
| 504 | } | |
| 505 | } | |
| 506 | ||
| 507 | /// Puts an object in the cache, possibly evicting older entries if the | |
| 508 | /// cache exceeds its maximum size. Note that, although old objects may | |
| 509 | /// be evicted, the object just added to the cache with this function | |
| 510 | /// will not be evicted before the next call to `put` or `deinit` even if | |
| 511 | /// it exceeds the maximum cache size. | |
| 512 | fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { | |
| 513 | const lru_node = try allocator.create(LruList.Node); | |
| 514 | errdefer allocator.destroy(lru_node); | |
| 515 | lru_node.data = offset; | |
| 516 | ||
| 517 | const gop = try cache.objects.getOrPut(allocator, offset); | |
| 518 | if (gop.found_existing) { | |
| 519 | cache.byte_size -= gop.value_ptr.object.data.len; | |
| 520 | cache.lru_nodes.remove(gop.value_ptr.lru_node); | |
| 521 | allocator.destroy(gop.value_ptr.lru_node); | |
| 522 | allocator.free(gop.value_ptr.object.data); | |
| 523 | } | |
| 524 | gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; | |
| 525 | cache.byte_size += object.data.len; | |
| 526 | cache.lru_nodes.append(lru_node); | |
| 527 | ||
| 528 | while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { | |
| 529 | // The > 1 check is to make sure that we don't evict the most | |
| 530 | // recently added node, even if it by itself happens to exceed the | |
| 531 | // maximum size of the cache. | |
| 532 | const evict_node = cache.lru_nodes.popFirst().?; | |
| 533 | const evict_offset = evict_node.data; | |
| 534 | allocator.destroy(evict_node); | |
| 535 | const evict_object = cache.objects.get(evict_offset).?.object; | |
| 536 | cache.byte_size -= evict_object.data.len; | |
| 537 | allocator.free(evict_object.data); | |
| 538 | _ = cache.objects.remove(evict_offset); | |
| 539 | } | |
| 540 | } | |
| 541 | }; | |
| 542 | ||
| 543 | /// A single pkt-line in the Git protocol. | |
| 544 | /// | |
| 545 | /// The format of a pkt-line is documented in | |
| 546 | /// [protocol-common](https://git-scm.com/docs/protocol-common). The special | |
| 547 | /// meanings of the delimiter and response-end packets are documented in | |
| 548 | /// [protocol-v2](https://git-scm.com/docs/protocol-v2). | |
| 549 | const Packet = union(enum) { | |
| 550 | flush, | |
| 551 | delimiter, | |
| 552 | response_end, | |
| 553 | data: []const u8, | |
| 554 | ||
| 555 | const max_data_length = 65516; | |
| 556 | ||
| 557 | /// Reads a packet in pkt-line format. | |
| 558 | fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { | |
| 559 | const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; | |
| 560 | switch (length) { | |
| 561 | 0 => return .flush, | |
| 562 | 1 => return .delimiter, | |
| 563 | 2 => return .response_end, | |
| 564 | 3 => return error.InvalidPacket, | |
| 565 | else => if (length - 4 > max_data_length) return error.InvalidPacket, | |
| 566 | } | |
| 567 | const data = buf[0 .. length - 4]; | |
| 568 | try reader.readNoEof(data); | |
| 569 | return .{ .data = data }; | |
| 570 | } | |
| 571 | ||
| 572 | /// Writes a packet in pkt-line format. | |
| 573 | fn write(packet: Packet, writer: anytype) !void { | |
| 574 | switch (packet) { | |
| 575 | .flush => try writer.writeAll("0000"), | |
| 576 | .delimiter => try writer.writeAll("0001"), | |
| 577 | .response_end => try writer.writeAll("0002"), | |
| 578 | .data => |data| { | |
| 579 | assert(data.len <= max_data_length); | |
| 580 | try writer.print("{x:0>4}", .{data.len + 4}); | |
| 581 | try writer.writeAll(data); | |
| 582 | }, | |
| 583 | } | |
| 584 | } | |
| 585 | ||
| 586 | /// Returns the normalized form of textual packet data, stripping any | |
| 587 | /// trailing '\n'. | |
| 588 | /// | |
| 589 | /// As documented in | |
| 590 | /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), | |
| 591 | /// non-binary (textual) pkt-line data should contain a trailing '\n', but | |
| 592 | /// is not required to do so (implementations must support both forms). | |
| 593 | fn normalizeText(data: []const u8) []const u8 { | |
| 594 | return if (mem.endsWith(u8, data, "\n")) | |
| 595 | data[0 .. data.len - 1] | |
| 596 | else | |
| 597 | data; | |
| 598 | } | |
| 599 | }; | |
| 600 | ||
| 601 | /// A client session for the Git protocol, currently limited to an HTTP(S) | |
| 602 | /// transport. Only protocol version 2 is supported, as documented in | |
| 603 | /// [protocol-v2](https://git-scm.com/docs/protocol-v2). | |
| 604 | pub const Session = struct { | |
| 605 | transport: *std.http.Client, | |
| 606 | location: Location, | |
| 607 | supports_agent: bool, | |
| 608 | supports_shallow: bool, | |
| 609 | object_format: Oid.Format, | |
| 610 | allocator: Allocator, | |
| 611 | ||
| 612 | const agent = "zig/" ++ @import("builtin").zig_version_string; | |
| 613 | const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); | |
| 614 | ||
| 615 | /// Initializes a client session and discovers the capabilities of the | |
| 616 | /// server for optimal transport. | |
| 617 | pub fn init( | |
| 618 | allocator: Allocator, | |
| 619 | transport: *std.http.Client, | |
| 620 | uri: std.Uri, | |
| 621 | http_headers_buffer: []u8, | |
| 622 | ) !Session { | |
| 623 | var session: Session = .{ | |
| 624 | .transport = transport, | |
| 625 | .location = try .init(allocator, uri), | |
| 626 | .supports_agent = false, | |
| 627 | .supports_shallow = false, | |
| 628 | .object_format = .sha1, | |
| 629 | .allocator = allocator, | |
| 630 | }; | |
| 631 | errdefer session.deinit(); | |
| 632 | var capability_iterator = try session.getCapabilities(http_headers_buffer); | |
| 633 | defer capability_iterator.deinit(); | |
| 634 | while (try capability_iterator.next()) |capability| { | |
| 635 | if (mem.eql(u8, capability.key, "agent")) { | |
| 636 | session.supports_agent = true; | |
| 637 | } else if (mem.eql(u8, capability.key, "fetch")) { | |
| 638 | var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); | |
| 639 | while (feature_iterator.next()) |feature| { | |
| 640 | if (mem.eql(u8, feature, "shallow")) { | |
| 641 | session.supports_shallow = true; | |
| 642 | } | |
| 643 | } | |
| 644 | } else if (mem.eql(u8, capability.key, "object-format")) { | |
| 645 | if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { | |
| 646 | session.object_format = format; | |
| 647 | } | |
| 648 | } | |
| 649 | } | |
| 650 | return session; | |
| 651 | } | |
| 652 | ||
| 653 | pub fn deinit(session: *Session) void { | |
| 654 | session.location.deinit(session.allocator); | |
| 655 | session.* = undefined; | |
| 656 | } | |
| 657 | ||
| 658 | /// An owned `std.Uri` representing the location of the server (base URI). | |
| 659 | const Location = struct { | |
| 660 | uri: std.Uri, | |
| 661 | ||
| 662 | fn init(allocator: Allocator, uri: std.Uri) !Location { | |
| 663 | const scheme = try allocator.dupe(u8, uri.scheme); | |
| 664 | errdefer allocator.free(scheme); | |
| 665 | const user = if (uri.user) |user| try std.fmt.allocPrint(allocator, "{user}", .{user}) else null; | |
| 666 | errdefer if (user) |s| allocator.free(s); | |
| 667 | const password = if (uri.password) |password| try std.fmt.allocPrint(allocator, "{password}", .{password}) else null; | |
| 668 | errdefer if (password) |s| allocator.free(s); | |
| 669 | const host = if (uri.host) |host| try std.fmt.allocPrint(allocator, "{host}", .{host}) else null; | |
| 670 | errdefer if (host) |s| allocator.free(s); | |
| 671 | const path = try std.fmt.allocPrint(allocator, "{path}", .{uri.path}); | |
| 672 | errdefer allocator.free(path); | |
| 673 | // The query and fragment are not used as part of the base server URI. | |
| 674 | return .{ | |
| 675 | .uri = .{ | |
| 676 | .scheme = scheme, | |
| 677 | .user = if (user) |s| .{ .percent_encoded = s } else null, | |
| 678 | .password = if (password) |s| .{ .percent_encoded = s } else null, | |
| 679 | .host = if (host) |s| .{ .percent_encoded = s } else null, | |
| 680 | .port = uri.port, | |
| 681 | .path = .{ .percent_encoded = path }, | |
| 682 | }, | |
| 683 | }; | |
| 684 | } | |
| 685 | ||
| 686 | fn deinit(loc: *Location, allocator: Allocator) void { | |
| 687 | allocator.free(loc.uri.scheme); | |
| 688 | if (loc.uri.user) |user| allocator.free(user.percent_encoded); | |
| 689 | if (loc.uri.password) |password| allocator.free(password.percent_encoded); | |
| 690 | if (loc.uri.host) |host| allocator.free(host.percent_encoded); | |
| 691 | allocator.free(loc.uri.path.percent_encoded); | |
| 692 | } | |
| 693 | }; | |
| 694 | ||
| 695 | /// Returns an iterator over capabilities supported by the server. | |
| 696 | /// | |
| 697 | /// The `session.location` is updated if the server returns a redirect, so | |
| 698 | /// that subsequent session functions do not need to handle redirects. | |
| 699 | fn getCapabilities(session: *Session, http_headers_buffer: []u8) !CapabilityIterator { | |
| 700 | var info_refs_uri = session.location.uri; | |
| 701 | { | |
| 702 | const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); | |
| 703 | defer session.allocator.free(session_uri_path); | |
| 704 | info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "info/refs" }) }; | |
| 705 | } | |
| 706 | defer session.allocator.free(info_refs_uri.path.percent_encoded); | |
| 707 | info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; | |
| 708 | info_refs_uri.fragment = null; | |
| 709 | ||
| 710 | const max_redirects = 3; | |
| 711 | var request = try session.transport.open(.GET, info_refs_uri, .{ | |
| 712 | .redirect_behavior = @enumFromInt(max_redirects), | |
| 713 | .server_header_buffer = http_headers_buffer, | |
| 714 | .extra_headers = &.{ | |
| 715 | .{ .name = "Git-Protocol", .value = "version=2" }, | |
| 716 | }, | |
| 717 | }); | |
| 718 | errdefer request.deinit(); | |
| 719 | try request.send(); | |
| 720 | try request.finish(); | |
| 721 | ||
| 722 | try request.wait(); | |
| 723 | if (request.response.status != .ok) return error.ProtocolError; | |
| 724 | const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; | |
| 725 | if (any_redirects_occurred) { | |
| 726 | const request_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{request.uri.path}); | |
| 727 | defer session.allocator.free(request_uri_path); | |
| 728 | if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; | |
| 729 | var new_uri = request.uri; | |
| 730 | new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; | |
| 731 | const new_location: Location = try .init(session.allocator, new_uri); | |
| 732 | session.location.deinit(session.allocator); | |
| 733 | session.location = new_location; | |
| 734 | } | |
| 735 | ||
| 736 | const reader = request.reader(); | |
| 737 | var buf: [Packet.max_data_length]u8 = undefined; | |
| 738 | var state: enum { response_start, response_content } = .response_start; | |
| 739 | while (true) { | |
| 740 | // Some Git servers (at least GitHub) include an additional | |
| 741 | // '# service=git-upload-pack' informative response before sending | |
| 742 | // the expected 'version 2' packet and capability information. | |
| 743 | // This is not universal: SourceHut, for example, does not do this. | |
| 744 | // Thus, we need to skip any such useless additional responses | |
| 745 | // before we get the one we're actually looking for. The responses | |
| 746 | // will be delimited by flush packets. | |
| 747 | const packet = Packet.read(reader, &buf) catch |e| switch (e) { | |
| 748 | error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found | |
| 749 | else => |other| return other, | |
| 750 | }; | |
| 751 | switch (packet) { | |
| 752 | .flush => state = .response_start, | |
| 753 | .data => |data| switch (state) { | |
| 754 | .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { | |
| 755 | return .{ .request = request }; | |
| 756 | } else { | |
| 757 | state = .response_content; | |
| 758 | }, | |
| 759 | else => {}, | |
| 760 | }, | |
| 761 | else => return error.UnexpectedPacket, | |
| 762 | } | |
| 763 | } | |
| 764 | } | |
| 765 | ||
| 766 | const CapabilityIterator = struct { | |
| 767 | request: std.http.Client.Request, | |
| 768 | buf: [Packet.max_data_length]u8 = undefined, | |
| 769 | ||
| 770 | const Capability = struct { | |
| 771 | key: []const u8, | |
| 772 | value: ?[]const u8 = null, | |
| 773 | ||
| 774 | fn parse(data: []const u8) Capability { | |
| 775 | return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| | |
| 776 | .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } | |
| 777 | else | |
| 778 | .{ .key = data }; | |
| 779 | } | |
| 780 | }; | |
| 781 | ||
| 782 | fn deinit(iterator: *CapabilityIterator) void { | |
| 783 | iterator.request.deinit(); | |
| 784 | iterator.* = undefined; | |
| 785 | } | |
| 786 | ||
| 787 | fn next(iterator: *CapabilityIterator) !?Capability { | |
| 788 | switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { | |
| 789 | .flush => return null, | |
| 790 | .data => |data| return Capability.parse(Packet.normalizeText(data)), | |
| 791 | else => return error.UnexpectedPacket, | |
| 792 | } | |
| 793 | } | |
| 794 | }; | |
| 795 | ||
| 796 | const ListRefsOptions = struct { | |
| 797 | /// The ref prefixes (if any) to use to filter the refs available on the | |
| 798 | /// server. Note that the client must still check the returned refs | |
| 799 | /// against its desired filters itself: the server is not required to | |
| 800 | /// respect these prefix filters and may return other refs as well. | |
| 801 | ref_prefixes: []const []const u8 = &.{}, | |
| 802 | /// Whether to include symref targets for returned symbolic refs. | |
| 803 | include_symrefs: bool = false, | |
| 804 | /// Whether to include the peeled object ID for returned tag refs. | |
| 805 | include_peeled: bool = false, | |
| 806 | server_header_buffer: []u8, | |
| 807 | }; | |
| 808 | ||
| 809 | /// Returns an iterator over refs known to the server. | |
| 810 | pub fn listRefs(session: Session, options: ListRefsOptions) !RefIterator { | |
| 811 | var upload_pack_uri = session.location.uri; | |
| 812 | { | |
| 813 | const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); | |
| 814 | defer session.allocator.free(session_uri_path); | |
| 815 | upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; | |
| 816 | } | |
| 817 | defer session.allocator.free(upload_pack_uri.path.percent_encoded); | |
| 818 | upload_pack_uri.query = null; | |
| 819 | upload_pack_uri.fragment = null; | |
| 820 | ||
| 821 | var body: std.ArrayListUnmanaged(u8) = .empty; | |
| 822 | defer body.deinit(session.allocator); | |
| 823 | const body_writer = body.writer(session.allocator); | |
| 824 | try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); | |
| 825 | if (session.supports_agent) { | |
| 826 | try Packet.write(.{ .data = agent_capability }, body_writer); | |
| 827 | } | |
| 828 | { | |
| 829 | const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); | |
| 830 | defer session.allocator.free(object_format_packet); | |
| 831 | try Packet.write(.{ .data = object_format_packet }, body_writer); | |
| 832 | } | |
| 833 | try Packet.write(.delimiter, body_writer); | |
| 834 | for (options.ref_prefixes) |ref_prefix| { | |
| 835 | const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix}); | |
| 836 | defer session.allocator.free(ref_prefix_packet); | |
| 837 | try Packet.write(.{ .data = ref_prefix_packet }, body_writer); | |
| 838 | } | |
| 839 | if (options.include_symrefs) { | |
| 840 | try Packet.write(.{ .data = "symrefs\n" }, body_writer); | |
| 841 | } | |
| 842 | if (options.include_peeled) { | |
| 843 | try Packet.write(.{ .data = "peel\n" }, body_writer); | |
| 844 | } | |
| 845 | try Packet.write(.flush, body_writer); | |
| 846 | ||
| 847 | var request = try session.transport.open(.POST, upload_pack_uri, .{ | |
| 848 | .redirect_behavior = .unhandled, | |
| 849 | .server_header_buffer = options.server_header_buffer, | |
| 850 | .extra_headers = &.{ | |
| 851 | .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, | |
| 852 | .{ .name = "Git-Protocol", .value = "version=2" }, | |
| 853 | }, | |
| 854 | }); | |
| 855 | errdefer request.deinit(); | |
| 856 | request.transfer_encoding = .{ .content_length = body.items.len }; | |
| 857 | try request.send(); | |
| 858 | try request.writeAll(body.items); | |
| 859 | try request.finish(); | |
| 860 | ||
| 861 | try request.wait(); | |
| 862 | if (request.response.status != .ok) return error.ProtocolError; | |
| 863 | ||
| 864 | return .{ | |
| 865 | .format = session.object_format, | |
| 866 | .request = request, | |
| 867 | }; | |
| 868 | } | |
| 869 | ||
| 870 | pub const RefIterator = struct { | |
| 871 | format: Oid.Format, | |
| 872 | request: std.http.Client.Request, | |
| 873 | buf: [Packet.max_data_length]u8 = undefined, | |
| 874 | ||
| 875 | pub const Ref = struct { | |
| 876 | oid: Oid, | |
| 877 | name: []const u8, | |
| 878 | symref_target: ?[]const u8, | |
| 879 | peeled: ?Oid, | |
| 880 | }; | |
| 881 | ||
| 882 | pub fn deinit(iterator: *RefIterator) void { | |
| 883 | iterator.request.deinit(); | |
| 884 | iterator.* = undefined; | |
| 885 | } | |
| 886 | ||
| 887 | pub fn next(iterator: *RefIterator) !?Ref { | |
| 888 | switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { | |
| 889 | .flush => return null, | |
| 890 | .data => |data| { | |
| 891 | const ref_data = Packet.normalizeText(data); | |
| 892 | const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; | |
| 893 | const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; | |
| 894 | ||
| 895 | const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; | |
| 896 | const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; | |
| 897 | ||
| 898 | var symref_target: ?[]const u8 = null; | |
| 899 | var peeled: ?Oid = null; | |
| 900 | var last_sep_pos = name_sep_pos; | |
| 901 | while (last_sep_pos < ref_data.len) { | |
| 902 | const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; | |
| 903 | const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; | |
| 904 | if (mem.startsWith(u8, attribute, "symref-target:")) { | |
| 905 | symref_target = attribute["symref-target:".len..]; | |
| 906 | } else if (mem.startsWith(u8, attribute, "peeled:")) { | |
| 907 | peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; | |
| 908 | } | |
| 909 | last_sep_pos = next_sep_pos; | |
| 910 | } | |
| 911 | ||
| 912 | return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; | |
| 913 | }, | |
| 914 | else => return error.UnexpectedPacket, | |
| 915 | } | |
| 916 | } | |
| 917 | }; | |
| 918 | ||
| 919 | /// Fetches the given refs from the server. A shallow fetch (depth 1) is | |
| 920 | /// performed if the server supports it. | |
| 921 | pub fn fetch( | |
| 922 | session: Session, | |
| 923 | wants: []const []const u8, | |
| 924 | http_headers_buffer: []u8, | |
| 925 | ) !FetchStream { | |
| 926 | var upload_pack_uri = session.location.uri; | |
| 927 | { | |
| 928 | const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); | |
| 929 | defer session.allocator.free(session_uri_path); | |
| 930 | upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; | |
| 931 | } | |
| 932 | defer session.allocator.free(upload_pack_uri.path.percent_encoded); | |
| 933 | upload_pack_uri.query = null; | |
| 934 | upload_pack_uri.fragment = null; | |
| 935 | ||
| 936 | var body: std.ArrayListUnmanaged(u8) = .empty; | |
| 937 | defer body.deinit(session.allocator); | |
| 938 | const body_writer = body.writer(session.allocator); | |
| 939 | try Packet.write(.{ .data = "command=fetch\n" }, body_writer); | |
| 940 | if (session.supports_agent) { | |
| 941 | try Packet.write(.{ .data = agent_capability }, body_writer); | |
| 942 | } | |
| 943 | { | |
| 944 | const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); | |
| 945 | defer session.allocator.free(object_format_packet); | |
| 946 | try Packet.write(.{ .data = object_format_packet }, body_writer); | |
| 947 | } | |
| 948 | try Packet.write(.delimiter, body_writer); | |
| 949 | // Our packfile parser supports the OFS_DELTA object type | |
| 950 | try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); | |
| 951 | // We do not currently convey server progress information to the user | |
| 952 | try Packet.write(.{ .data = "no-progress\n" }, body_writer); | |
| 953 | if (session.supports_shallow) { | |
| 954 | try Packet.write(.{ .data = "deepen 1\n" }, body_writer); | |
| 955 | } | |
| 956 | for (wants) |want| { | |
| 957 | var buf: [Packet.max_data_length]u8 = undefined; | |
| 958 | const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; | |
| 959 | try Packet.write(.{ .data = arg }, body_writer); | |
| 960 | } | |
| 961 | try Packet.write(.{ .data = "done\n" }, body_writer); | |
| 962 | try Packet.write(.flush, body_writer); | |
| 963 | ||
| 964 | var request = try session.transport.open(.POST, upload_pack_uri, .{ | |
| 965 | .redirect_behavior = .not_allowed, | |
| 966 | .server_header_buffer = http_headers_buffer, | |
| 967 | .extra_headers = &.{ | |
| 968 | .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, | |
| 969 | .{ .name = "Git-Protocol", .value = "version=2" }, | |
| 970 | }, | |
| 971 | }); | |
| 972 | errdefer request.deinit(); | |
| 973 | request.transfer_encoding = .{ .content_length = body.items.len }; | |
| 974 | try request.send(); | |
| 975 | try request.writeAll(body.items); | |
| 976 | try request.finish(); | |
| 977 | ||
| 978 | try request.wait(); | |
| 979 | if (request.response.status != .ok) return error.ProtocolError; | |
| 980 | ||
| 981 | const reader = request.reader(); | |
| 982 | // We are not interested in any of the sections of the returned fetch | |
| 983 | // data other than the packfile section, since we aren't doing anything | |
| 984 | // complex like ref negotiation (this is a fresh clone). | |
| 985 | var state: enum { section_start, section_content } = .section_start; | |
| 986 | while (true) { | |
| 987 | var buf: [Packet.max_data_length]u8 = undefined; | |
| 988 | const packet = try Packet.read(reader, &buf); | |
| 989 | switch (state) { | |
| 990 | .section_start => switch (packet) { | |
| 991 | .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { | |
| 992 | return .{ .request = request }; | |
| 993 | } else { | |
| 994 | state = .section_content; | |
| 995 | }, | |
| 996 | else => return error.UnexpectedPacket, | |
| 997 | }, | |
| 998 | .section_content => switch (packet) { | |
| 999 | .delimiter => state = .section_start, | |
| 1000 | .data => {}, | |
| 1001 | else => return error.UnexpectedPacket, | |
| 1002 | }, | |
| 1003 | } | |
| 1004 | } | |
| 1005 | } | |
| 1006 | ||
| 1007 | pub const FetchStream = struct { | |
| 1008 | request: std.http.Client.Request, | |
| 1009 | buf: [Packet.max_data_length]u8 = undefined, | |
| 1010 | pos: usize = 0, | |
| 1011 | len: usize = 0, | |
| 1012 | ||
| 1013 | pub fn deinit(stream: *FetchStream) void { | |
| 1014 | stream.request.deinit(); | |
| 1015 | } | |
| 1016 | ||
| 1017 | pub const ReadError = std.http.Client.Request.ReadError || error{ | |
| 1018 | InvalidPacket, | |
| 1019 | ProtocolError, | |
| 1020 | UnexpectedPacket, | |
| 1021 | }; | |
| 1022 | pub const Reader = std.io.Reader(*FetchStream, ReadError, read); | |
| 1023 | ||
| 1024 | const StreamCode = enum(u8) { | |
| 1025 | pack_data = 1, | |
| 1026 | progress = 2, | |
| 1027 | fatal_error = 3, | |
| 1028 | _, | |
| 1029 | }; | |
| 1030 | ||
| 1031 | pub fn reader(stream: *FetchStream) Reader { | |
| 1032 | return .{ .context = stream }; | |
| 1033 | } | |
| 1034 | ||
| 1035 | pub fn read(stream: *FetchStream, buf: []u8) !usize { | |
| 1036 | if (stream.pos == stream.len) { | |
| 1037 | while (true) { | |
| 1038 | switch (try Packet.read(stream.request.reader(), &stream.buf)) { | |
| 1039 | .flush => return 0, | |
| 1040 | .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { | |
| 1041 | .pack_data => { | |
| 1042 | stream.pos = 1; | |
| 1043 | stream.len = data.len; | |
| 1044 | break; | |
| 1045 | }, | |
| 1046 | .fatal_error => return error.ProtocolError, | |
| 1047 | else => {}, | |
| 1048 | }, | |
| 1049 | else => return error.UnexpectedPacket, | |
| 1050 | } | |
| 1051 | } | |
| 1052 | } | |
| 1053 | ||
| 1054 | const size = @min(buf.len, stream.len - stream.pos); | |
| 1055 | @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); | |
| 1056 | stream.pos += size; | |
| 1057 | return size; | |
| 1058 | } | |
| 1059 | }; | |
| 1060 | }; | |
| 1061 | ||
| 1062 | const PackHeader = struct { | |
| 1063 | total_objects: u32, | |
| 1064 | ||
| 1065 | const signature = "PACK"; | |
| 1066 | const supported_version = 2; | |
| 1067 | ||
| 1068 | fn read(reader: anytype) !PackHeader { | |
| 1069 | const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { | |
| 1070 | error.EndOfStream => return error.InvalidHeader, | |
| 1071 | else => |other| return other, | |
| 1072 | }; | |
| 1073 | if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; | |
| 1074 | const version = reader.readInt(u32, .big) catch |e| switch (e) { | |
| 1075 | error.EndOfStream => return error.InvalidHeader, | |
| 1076 | else => |other| return other, | |
| 1077 | }; | |
| 1078 | if (version != supported_version) return error.UnsupportedVersion; | |
| 1079 | const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { | |
| 1080 | error.EndOfStream => return error.InvalidHeader, | |
| 1081 | else => |other| return other, | |
| 1082 | }; | |
| 1083 | return .{ .total_objects = total_objects }; | |
| 1084 | } | |
| 1085 | }; | |
| 1086 | ||
| 1087 | const EntryHeader = union(Type) { | |
| 1088 | commit: Undeltified, | |
| 1089 | tree: Undeltified, | |
| 1090 | blob: Undeltified, | |
| 1091 | tag: Undeltified, | |
| 1092 | ofs_delta: OfsDelta, | |
| 1093 | ref_delta: RefDelta, | |
| 1094 | ||
| 1095 | const Type = enum(u3) { | |
| 1096 | commit = 1, | |
| 1097 | tree = 2, | |
| 1098 | blob = 3, | |
| 1099 | tag = 4, | |
| 1100 | ofs_delta = 6, | |
| 1101 | ref_delta = 7, | |
| 1102 | }; | |
| 1103 | ||
| 1104 | const Undeltified = struct { | |
| 1105 | uncompressed_length: u64, | |
| 1106 | }; | |
| 1107 | ||
| 1108 | const OfsDelta = struct { | |
| 1109 | offset: u64, | |
| 1110 | uncompressed_length: u64, | |
| 1111 | }; | |
| 1112 | ||
| 1113 | const RefDelta = struct { | |
| 1114 | base_object: Oid, | |
| 1115 | uncompressed_length: u64, | |
| 1116 | }; | |
| 1117 | ||
| 1118 | fn objectType(header: EntryHeader) Object.Type { | |
| 1119 | return switch (header) { | |
| 1120 | inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), | |
| 1121 | else => unreachable, | |
| 1122 | }; | |
| 1123 | } | |
| 1124 | ||
| 1125 | fn uncompressedLength(header: EntryHeader) u64 { | |
| 1126 | return switch (header) { | |
| 1127 | inline else => |entry| entry.uncompressed_length, | |
| 1128 | }; | |
| 1129 | } | |
| 1130 | ||
| 1131 | fn read(format: Oid.Format, reader: anytype) !EntryHeader { | |
| 1132 | const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; | |
| 1133 | const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { | |
| 1134 | error.EndOfStream => return error.InvalidFormat, | |
| 1135 | else => |other| return other, | |
| 1136 | }); | |
| 1137 | const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; | |
| 1138 | var uncompressed_length: u64 = initial.len; | |
| 1139 | uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; | |
| 1140 | const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; | |
| 1141 | return switch (@"type") { | |
| 1142 | inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ | |
| 1143 | .uncompressed_length = uncompressed_length, | |
| 1144 | }), | |
| 1145 | .ofs_delta => .{ .ofs_delta = .{ | |
| 1146 | .offset = try readOffsetVarInt(reader), | |
| 1147 | .uncompressed_length = uncompressed_length, | |
| 1148 | } }, | |
| 1149 | .ref_delta => .{ .ref_delta = .{ | |
| 1150 | .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { | |
| 1151 | error.EndOfStream => return error.InvalidFormat, | |
| 1152 | else => |other| return other, | |
| 1153 | }, | |
| 1154 | .uncompressed_length = uncompressed_length, | |
| 1155 | } }, | |
| 1156 | }; | |
| 1157 | } | |
| 1158 | }; | |
| 1159 | ||
| 1160 | fn readSizeVarInt(r: anytype) !u64 { | |
| 1161 | const Byte = packed struct { value: u7, has_next: bool }; | |
| 1162 | var b: Byte = @bitCast(try r.readByte()); | |
| 1163 | var value: u64 = b.value; | |
| 1164 | var shift: u6 = 0; | |
| 1165 | while (b.has_next) { | |
| 1166 | b = @bitCast(try r.readByte()); | |
| 1167 | shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; | |
| 1168 | value |= @as(u64, b.value) << shift; | |
| 1169 | } | |
| 1170 | return value; | |
| 1171 | } | |
| 1172 | ||
| 1173 | fn readOffsetVarInt(r: anytype) !u64 { | |
| 1174 | const Byte = packed struct { value: u7, has_next: bool }; | |
| 1175 | var b: Byte = @bitCast(try r.readByte()); | |
| 1176 | var value: u64 = b.value; | |
| 1177 | while (b.has_next) { | |
| 1178 | b = @bitCast(try r.readByte()); | |
| 1179 | value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; | |
| 1180 | value |= b.value; | |
| 1181 | } | |
| 1182 | return value; | |
| 1183 | } | |
| 1184 | ||
| 1185 | const IndexHeader = struct { | |
| 1186 | fan_out_table: [256]u32, | |
| 1187 | ||
| 1188 | const signature = "\xFFtOc"; | |
| 1189 | const supported_version = 2; | |
| 1190 | const size = 4 + 4 + @sizeOf([256]u32); | |
| 1191 | ||
| 1192 | fn read(reader: anytype) !IndexHeader { | |
| 1193 | var header_bytes = try reader.readBytesNoEof(size); | |
| 1194 | if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; | |
| 1195 | const version = mem.readInt(u32, header_bytes[4..8], .big); | |
| 1196 | if (version != supported_version) return error.UnsupportedVersion; | |
| 1197 | ||
| 1198 | var fan_out_table: [256]u32 = undefined; | |
| 1199 | var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); | |
| 1200 | const fan_out_table_reader = fan_out_table_stream.reader(); | |
| 1201 | for (&fan_out_table) |*entry| { | |
| 1202 | entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; | |
| 1203 | } | |
| 1204 | return .{ .fan_out_table = fan_out_table }; | |
| 1205 | } | |
| 1206 | }; | |
| 1207 | ||
| 1208 | const IndexEntry = struct { | |
| 1209 | offset: u64, | |
| 1210 | crc32: u32, | |
| 1211 | }; | |
| 1212 | ||
| 1213 | /// Writes out a version 2 index for the given packfile, as documented in | |
| 1214 | /// [pack-format](https://git-scm.com/docs/pack-format). | |
| 1215 | pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { | |
| 1216 | try pack.seekTo(0); | |
| 1217 | ||
| 1218 | var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; | |
| 1219 | defer index_entries.deinit(allocator); | |
| 1220 | var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty; | |
| 1221 | defer pending_deltas.deinit(allocator); | |
| 1222 | ||
| 1223 | const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); | |
| 1224 | ||
| 1225 | var cache: ObjectCache = .{}; | |
| 1226 | defer cache.deinit(allocator); | |
| 1227 | var remaining_deltas = pending_deltas.items.len; | |
| 1228 | while (remaining_deltas > 0) { | |
| 1229 | var i: usize = remaining_deltas; | |
| 1230 | while (i > 0) { | |
| 1231 | i -= 1; | |
| 1232 | const delta = pending_deltas.items[i]; | |
| 1233 | if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { | |
| 1234 | try index_entries.put(allocator, oid, delta); | |
| 1235 | _ = pending_deltas.swapRemove(i); | |
| 1236 | } | |
| 1237 | } | |
| 1238 | if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; | |
| 1239 | remaining_deltas = pending_deltas.items.len; | |
| 1240 | } | |
| 1241 | ||
| 1242 | var oids: std.ArrayListUnmanaged(Oid) = .empty; | |
| 1243 | defer oids.deinit(allocator); | |
| 1244 | try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); | |
| 1245 | var index_entries_iter = index_entries.iterator(); | |
| 1246 | while (index_entries_iter.next()) |entry| { | |
| 1247 | oids.appendAssumeCapacity(entry.key_ptr.*); | |
| 1248 | } | |
| 1249 | mem.sortUnstable(Oid, oids.items, {}, struct { | |
| 1250 | fn lessThan(_: void, o1: Oid, o2: Oid) bool { | |
| 1251 | return mem.lessThan(u8, o1.slice(), o2.slice()); | |
| 1252 | } | |
| 1253 | }.lessThan); | |
| 1254 | ||
| 1255 | var fan_out_table: [256]u32 = undefined; | |
| 1256 | var count: u32 = 0; | |
| 1257 | var fan_out_index: u8 = 0; | |
| 1258 | for (oids.items) |oid| { | |
| 1259 | const key = oid.slice()[0]; | |
| 1260 | if (key > fan_out_index) { | |
| 1261 | @memset(fan_out_table[fan_out_index..key], count); | |
| 1262 | fan_out_index = key; | |
| 1263 | } | |
| 1264 | count += 1; | |
| 1265 | } | |
| 1266 | @memset(fan_out_table[fan_out_index..], count); | |
| 1267 | ||
| 1268 | var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); | |
| 1269 | const writer = index_hashed_writer.writer(); | |
| 1270 | try writer.writeAll(IndexHeader.signature); | |
| 1271 | try writer.writeInt(u32, IndexHeader.supported_version, .big); | |
| 1272 | for (fan_out_table) |fan_out_entry| { | |
| 1273 | try writer.writeInt(u32, fan_out_entry, .big); | |
| 1274 | } | |
| 1275 | ||
| 1276 | for (oids.items) |oid| { | |
| 1277 | try writer.writeAll(oid.slice()); | |
| 1278 | } | |
| 1279 | ||
| 1280 | for (oids.items) |oid| { | |
| 1281 | try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); | |
| 1282 | } | |
| 1283 | ||
| 1284 | var big_offsets: std.ArrayListUnmanaged(u64) = .empty; | |
| 1285 | defer big_offsets.deinit(allocator); | |
| 1286 | for (oids.items) |oid| { | |
| 1287 | const offset = index_entries.get(oid).?.offset; | |
| 1288 | if (offset <= std.math.maxInt(u31)) { | |
| 1289 | try writer.writeInt(u32, @intCast(offset), .big); | |
| 1290 | } else { | |
| 1291 | const index = big_offsets.items.len; | |
| 1292 | try big_offsets.append(allocator, offset); | |
| 1293 | try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); | |
| 1294 | } | |
| 1295 | } | |
| 1296 | for (big_offsets.items) |offset| { | |
| 1297 | try writer.writeInt(u64, offset, .big); | |
| 1298 | } | |
| 1299 | ||
| 1300 | try writer.writeAll(pack_checksum.slice()); | |
| 1301 | const index_checksum = index_hashed_writer.hasher.finalResult(); | |
| 1302 | try index_writer.writeAll(index_checksum.slice()); | |
| 1303 | } | |
| 1304 | ||
| 1305 | /// Performs the first pass over the packfile data for index construction. | |
| 1306 | /// This will index all non-delta objects, queue delta objects for further | |
| 1307 | /// processing, and return the pack checksum (which is part of the index | |
| 1308 | /// format). | |
| 1309 | fn indexPackFirstPass( | |
| 1310 | allocator: Allocator, | |
| 1311 | format: Oid.Format, | |
| 1312 | pack: std.fs.File, | |
| 1313 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), | |
| 1314 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), | |
| 1315 | ) !Oid { | |
| 1316 | var pack_buffered_reader = std.io.bufferedReader(pack.reader()); | |
| 1317 | var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); | |
| 1318 | var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); | |
| 1319 | const pack_reader = pack_hashed_reader.reader(); | |
| 1320 | ||
| 1321 | const pack_header = try PackHeader.read(pack_reader); | |
| 1322 | ||
| 1323 | var current_entry: u32 = 0; | |
| 1324 | while (current_entry < pack_header.total_objects) : (current_entry += 1) { | |
| 1325 | const entry_offset = pack_counting_reader.bytes_read; | |
| 1326 | var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); | |
| 1327 | const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); | |
| 1328 | switch (entry_header) { | |
| 1329 | .commit, .tree, .blob, .tag => |object| { | |
| 1330 | var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); | |
| 1331 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); | |
| 1332 | var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); | |
| 1333 | const entry_writer = entry_hashed_writer.writer(); | |
| 1334 | // The object header is not included in the pack data but is | |
| 1335 | // part of the object's ID | |
| 1336 | try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); | |
| 1337 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1338 | try fifo.pump(entry_counting_reader.reader(), entry_writer); | |
| 1339 | if (entry_counting_reader.bytes_read != object.uncompressed_length) { | |
| 1340 | return error.InvalidObject; | |
| 1341 | } | |
| 1342 | const oid = entry_hashed_writer.hasher.finalResult(); | |
| 1343 | try index_entries.put(allocator, oid, .{ | |
| 1344 | .offset = entry_offset, | |
| 1345 | .crc32 = entry_crc32_reader.hasher.final(), | |
| 1346 | }); | |
| 1347 | }, | |
| 1348 | inline .ofs_delta, .ref_delta => |delta| { | |
| 1349 | var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); | |
| 1350 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); | |
| 1351 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1352 | try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); | |
| 1353 | if (entry_counting_reader.bytes_read != delta.uncompressed_length) { | |
| 1354 | return error.InvalidObject; | |
| 1355 | } | |
| 1356 | try pending_deltas.append(allocator, .{ | |
| 1357 | .offset = entry_offset, | |
| 1358 | .crc32 = entry_crc32_reader.hasher.final(), | |
| 1359 | }); | |
| 1360 | }, | |
| 1361 | } | |
| 1362 | } | |
| 1363 | ||
| 1364 | const pack_checksum = pack_hashed_reader.hasher.finalResult(); | |
| 1365 | const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); | |
| 1366 | if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { | |
| 1367 | return error.CorruptedPack; | |
| 1368 | } | |
| 1369 | _ = pack_reader.readByte() catch |e| switch (e) { | |
| 1370 | error.EndOfStream => return pack_checksum, | |
| 1371 | else => |other| return other, | |
| 1372 | }; | |
| 1373 | return error.InvalidFormat; | |
| 1374 | } | |
| 1375 | ||
| 1376 | /// Attempts to determine the final object ID of the given deltified object. | |
| 1377 | /// May return null if this is not yet possible (if the delta is a ref-based | |
| 1378 | /// delta and we do not yet know the offset of the base object). | |
| 1379 | fn indexPackHashDelta( | |
| 1380 | allocator: Allocator, | |
| 1381 | format: Oid.Format, | |
| 1382 | pack: std.fs.File, | |
| 1383 | delta: IndexEntry, | |
| 1384 | index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), | |
| 1385 | cache: *ObjectCache, | |
| 1386 | ) !?Oid { | |
| 1387 | // Figure out the chain of deltas to resolve | |
| 1388 | var base_offset = delta.offset; | |
| 1389 | var base_header: EntryHeader = undefined; | |
| 1390 | var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; | |
| 1391 | defer delta_offsets.deinit(allocator); | |
| 1392 | const base_object = while (true) { | |
| 1393 | if (cache.get(base_offset)) |base_object| break base_object; | |
| 1394 | ||
| 1395 | try pack.seekTo(base_offset); | |
| 1396 | base_header = try EntryHeader.read(format, pack.reader()); | |
| 1397 | switch (base_header) { | |
| 1398 | .ofs_delta => |ofs_delta| { | |
| 1399 | try delta_offsets.append(allocator, base_offset); | |
| 1400 | base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; | |
| 1401 | }, | |
| 1402 | .ref_delta => |ref_delta| { | |
| 1403 | try delta_offsets.append(allocator, base_offset); | |
| 1404 | base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; | |
| 1405 | }, | |
| 1406 | else => { | |
| 1407 | const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); | |
| 1408 | errdefer allocator.free(base_data); | |
| 1409 | const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; | |
| 1410 | try cache.put(allocator, base_offset, base_object); | |
| 1411 | break base_object; | |
| 1412 | }, | |
| 1413 | } | |
| 1414 | }; | |
| 1415 | ||
| 1416 | const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); | |
| 1417 | ||
| 1418 | var entry_hasher: Oid.Hasher = .init(format); | |
| 1419 | var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); | |
| 1420 | try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); | |
| 1421 | entry_hasher.update(base_data); | |
| 1422 | return entry_hasher.finalResult(); | |
| 1423 | } | |
| 1424 | ||
| 1425 | /// Resolves a chain of deltas, returning the final base object data. `pack` is | |
| 1426 | /// assumed to be looking at the start of the object data for the base object of | |
| 1427 | /// the chain, and will then apply the deltas in `delta_offsets` in reverse order | |
| 1428 | /// to obtain the final object. | |
| 1429 | fn resolveDeltaChain( | |
| 1430 | allocator: Allocator, | |
| 1431 | format: Oid.Format, | |
| 1432 | pack: std.fs.File, | |
| 1433 | base_object: Object, | |
| 1434 | delta_offsets: []const u64, | |
| 1435 | cache: *ObjectCache, | |
| 1436 | ) ![]const u8 { | |
| 1437 | var base_data = base_object.data; | |
| 1438 | var i: usize = delta_offsets.len; | |
| 1439 | while (i > 0) { | |
| 1440 | i -= 1; | |
| 1441 | ||
| 1442 | const delta_offset = delta_offsets[i]; | |
| 1443 | try pack.seekTo(delta_offset); | |
| 1444 | const delta_header = try EntryHeader.read(format, pack.reader()); | |
| 1445 | const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); | |
| 1446 | defer allocator.free(delta_data); | |
| 1447 | var delta_stream = std.io.fixedBufferStream(delta_data); | |
| 1448 | const delta_reader = delta_stream.reader(); | |
| 1449 | _ = try readSizeVarInt(delta_reader); // base object size | |
| 1450 | const expanded_size = try readSizeVarInt(delta_reader); | |
| 1451 | ||
| 1452 | const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; | |
| 1453 | const expanded_data = try allocator.alloc(u8, expanded_alloc_size); | |
| 1454 | errdefer allocator.free(expanded_data); | |
| 1455 | var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); | |
| 1456 | var base_stream = std.io.fixedBufferStream(base_data); | |
| 1457 | try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); | |
| 1458 | if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; | |
| 1459 | ||
| 1460 | try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); | |
| 1461 | base_data = expanded_data; | |
| 1462 | } | |
| 1463 | return base_data; | |
| 1464 | } | |
| 1465 | ||
| 1466 | /// Reads the complete contents of an object from `reader`. This function may | |
| 1467 | /// read more bytes than required from `reader`, so the reader position after | |
| 1468 | /// returning is not reliable. | |
| 1469 | fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { | |
| 1470 | const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; | |
| 1471 | var buffered_reader = std.io.bufferedReader(reader); | |
| 1472 | var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); | |
| 1473 | const data = try allocator.alloc(u8, alloc_size); | |
| 1474 | errdefer allocator.free(data); | |
| 1475 | try decompress_stream.reader().readNoEof(data); | |
| 1476 | _ = decompress_stream.reader().readByte() catch |e| switch (e) { | |
| 1477 | error.EndOfStream => return data, | |
| 1478 | else => |other| return other, | |
| 1479 | }; | |
| 1480 | return error.InvalidFormat; | |
| 1481 | } | |
| 1482 | ||
| 1483 | /// Expands delta data from `delta_reader` to `writer`. `base_object` must | |
| 1484 | /// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). | |
| 1485 | /// | |
| 1486 | /// The format of the delta data is documented in | |
| 1487 | /// [pack-format](https://git-scm.com/docs/pack-format). | |
| 1488 | fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { | |
| 1489 | while (true) { | |
| 1490 | const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { | |
| 1491 | error.EndOfStream => return, | |
| 1492 | else => |other| return other, | |
| 1493 | }); | |
| 1494 | if (inst.copy) { | |
| 1495 | const available: packed struct { | |
| 1496 | offset1: bool, | |
| 1497 | offset2: bool, | |
| 1498 | offset3: bool, | |
| 1499 | offset4: bool, | |
| 1500 | size1: bool, | |
| 1501 | size2: bool, | |
| 1502 | size3: bool, | |
| 1503 | } = @bitCast(inst.value); | |
| 1504 | const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ | |
| 1505 | .offset1 = if (available.offset1) try delta_reader.readByte() else 0, | |
| 1506 | .offset2 = if (available.offset2) try delta_reader.readByte() else 0, | |
| 1507 | .offset3 = if (available.offset3) try delta_reader.readByte() else 0, | |
| 1508 | .offset4 = if (available.offset4) try delta_reader.readByte() else 0, | |
| 1509 | }; | |
| 1510 | const offset: u32 = @bitCast(offset_parts); | |
| 1511 | const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ | |
| 1512 | .size1 = if (available.size1) try delta_reader.readByte() else 0, | |
| 1513 | .size2 = if (available.size2) try delta_reader.readByte() else 0, | |
| 1514 | .size3 = if (available.size3) try delta_reader.readByte() else 0, | |
| 1515 | }; | |
| 1516 | var size: u24 = @bitCast(size_parts); | |
| 1517 | if (size == 0) size = 0x10000; | |
| 1518 | try base_object.seekTo(offset); | |
| 1519 | var copy_reader = std.io.limitedReader(base_object.reader(), size); | |
| 1520 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1521 | try fifo.pump(copy_reader.reader(), writer); | |
| 1522 | } else if (inst.value != 0) { | |
| 1523 | var data_reader = std.io.limitedReader(delta_reader, inst.value); | |
| 1524 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1525 | try fifo.pump(data_reader.reader(), writer); | |
| 1526 | } else { | |
| 1527 | return error.InvalidDeltaInstruction; | |
| 1528 | } | |
| 1529 | } | |
| 1530 | } | |
| 1531 | ||
| 1532 | /// Runs the packfile indexing and checkout test. | |
| 1533 | /// | |
| 1534 | /// The two testrepo repositories under testdata contain identical commit | |
| 1535 | /// histories and contents. | |
| 1536 | /// | |
| 1537 | /// To verify the contents of the packfiles using Git alone, run the | |
| 1538 | /// following commands in an empty directory: | |
| 1539 | /// | |
| 1540 | /// 1. `git init --object-format=(sha1|sha256)` | |
| 1541 | /// 2. `git unpack-objects <path/to/testrepo.pack` | |
| 1542 | /// 3. `git fsck` - will print one "dangling commit": | |
| 1543 | /// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb` | |
| 1544 | /// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a` | |
| 1545 | /// 4. `git checkout $commit` | |
| 1546 | fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void { | |
| 1547 | const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack"); | |
| 1548 | ||
| 1549 | var git_dir = testing.tmpDir(.{}); | |
| 1550 | defer git_dir.cleanup(); | |
| 1551 | var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true }); | |
| 1552 | defer pack_file.close(); | |
| 1553 | try pack_file.writeAll(testrepo_pack); | |
| 1554 | ||
| 1555 | var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); | |
| 1556 | defer index_file.close(); | |
| 1557 | try indexPack(testing.allocator, format, pack_file, index_file.writer()); | |
| 1558 | ||
| 1559 | // Arbitrary size limit on files read while checking the repository contents | |
| 1560 | // (all files in the test repo are known to be smaller than this) | |
| 1561 | const max_file_size = 8192; | |
| 1562 | ||
| 1563 | const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); | |
| 1564 | defer testing.allocator.free(index_file_data); | |
| 1565 | // testrepo.idx is generated by Git. The index created by this file should | |
| 1566 | // match it exactly. Running `git verify-pack -v testrepo.pack` can verify | |
| 1567 | // this. | |
| 1568 | const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); | |
| 1569 | try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); | |
| 1570 | ||
| 1571 | var repository = try Repository.init(testing.allocator, format, pack_file, index_file); | |
| 1572 | defer repository.deinit(); | |
| 1573 | ||
| 1574 | var worktree = testing.tmpDir(.{ .iterate = true }); | |
| 1575 | defer worktree.cleanup(); | |
| 1576 | ||
| 1577 | const commit_id = try Oid.parse(format, head_commit); | |
| 1578 | ||
| 1579 | var diagnostics: Diagnostics = .{ .allocator = testing.allocator }; | |
| 1580 | defer diagnostics.deinit(); | |
| 1581 | try repository.checkout(worktree.dir, commit_id, &diagnostics); | |
| 1582 | try testing.expect(diagnostics.errors.items.len == 0); | |
| 1583 | ||
| 1584 | const expected_files: []const []const u8 = &.{ | |
| 1585 | "dir/file", | |
| 1586 | "dir/subdir/file", | |
| 1587 | "dir/subdir/file2", | |
| 1588 | "dir2/file", | |
| 1589 | "dir3/file", | |
| 1590 | "dir3/file2", | |
| 1591 | "file", | |
| 1592 | "file2", | |
| 1593 | "file3", | |
| 1594 | "file4", | |
| 1595 | "file5", | |
| 1596 | "file6", | |
| 1597 | "file7", | |
| 1598 | "file8", | |
| 1599 | "file9", | |
| 1600 | }; | |
| 1601 | var actual_files: std.ArrayListUnmanaged([]u8) = .empty; | |
| 1602 | defer actual_files.deinit(testing.allocator); | |
| 1603 | defer for (actual_files.items) |file| testing.allocator.free(file); | |
| 1604 | var walker = try worktree.dir.walk(testing.allocator); | |
| 1605 | defer walker.deinit(); | |
| 1606 | while (try walker.next()) |entry| { | |
| 1607 | if (entry.kind != .file) continue; | |
| 1608 | const path = try testing.allocator.dupe(u8, entry.path); | |
| 1609 | errdefer testing.allocator.free(path); | |
| 1610 | mem.replaceScalar(u8, path, std.fs.path.sep, '/'); | |
| 1611 | try actual_files.append(testing.allocator, path); | |
| 1612 | } | |
| 1613 | mem.sortUnstable([]u8, actual_files.items, {}, struct { | |
| 1614 | fn lessThan(_: void, a: []u8, b: []u8) bool { | |
| 1615 | return mem.lessThan(u8, a, b); | |
| 1616 | } | |
| 1617 | }.lessThan); | |
| 1618 | try testing.expectEqualDeep(expected_files, actual_files.items); | |
| 1619 | ||
| 1620 | const expected_file_contents = | |
| 1621 | \\revision 1 | |
| 1622 | \\revision 2 | |
| 1623 | \\revision 4 | |
| 1624 | \\revision 5 | |
| 1625 | \\revision 7 | |
| 1626 | \\revision 8 | |
| 1627 | \\revision 9 | |
| 1628 | \\revision 10 | |
| 1629 | \\revision 12 | |
| 1630 | \\revision 13 | |
| 1631 | \\revision 14 | |
| 1632 | \\revision 18 | |
| 1633 | \\revision 19 | |
| 1634 | \\ | |
| 1635 | ; | |
| 1636 | const actual_file_contents = try worktree.dir.readFileAlloc(testing.allocator, "file", max_file_size); | |
| 1637 | defer testing.allocator.free(actual_file_contents); | |
| 1638 | try testing.expectEqualStrings(expected_file_contents, actual_file_contents); | |
| 1639 | } | |
| 1640 | ||
| 1641 | test "SHA-1 packfile indexing and checkout" { | |
| 1642 | try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb"); | |
| 1643 | } | |
| 1644 | ||
| 1645 | test "SHA-256 packfile indexing and checkout" { | |
| 1646 | try runRepositoryTest(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"); | |
| 1647 | } | |
| 1648 | ||
| 1649 | /// Checks out a commit of a packfile. Intended for experimenting with and | |
| 1650 | /// benchmarking possible optimizations to the indexing and checkout behavior. | |
| 1651 | pub fn main() !void { | |
| 1652 | const allocator = std.heap.c_allocator; | |
| 1653 | ||
| 1654 | const args = try std.process.argsAlloc(allocator); | |
| 1655 | defer std.process.argsFree(allocator, args); | |
| 1656 | if (args.len != 5) { | |
| 1657 | return error.InvalidArguments; // Arguments: format packfile commit worktree | |
| 1658 | } | |
| 1659 | ||
| 1660 | const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat; | |
| 1661 | ||
| 1662 | var pack_file = try std.fs.cwd().openFile(args[2], .{}); | |
| 1663 | defer pack_file.close(); | |
| 1664 | const commit = try Oid.parse(format, args[3]); | |
| 1665 | var worktree = try std.fs.cwd().makeOpenPath(args[4], .{}); | |
| 1666 | defer worktree.close(); | |
| 1667 | ||
| 1668 | var git_dir = try worktree.makeOpenPath(".git", .{}); | |
| 1669 | defer git_dir.close(); | |
| 1670 | ||
| 1671 | std.debug.print("Starting index...\n", .{}); | |
| 1672 | var index_file = try git_dir.createFile("idx", .{ .read = true }); | |
| 1673 | defer index_file.close(); | |
| 1674 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); | |
| 1675 | try indexPack(allocator, format, pack_file, index_buffered_writer.writer()); | |
| 1676 | try index_buffered_writer.flush(); | |
| 1677 | try index_file.sync(); | |
| 1678 | ||
| 1679 | std.debug.print("Starting checkout...\n", .{}); | |
| 1680 | var repository = try Repository.init(allocator, format, pack_file, index_file); | |
| 1681 | defer repository.deinit(); | |
| 1682 | var diagnostics: Diagnostics = .{ .allocator = allocator }; | |
| 1683 | defer diagnostics.deinit(); | |
| 1684 | try repository.checkout(worktree, commit, &diagnostics); | |
| 1685 | ||
| 1686 | for (diagnostics.errors.items) |err| { | |
| 1687 | std.debug.print("Diagnostic: {}\n", .{err}); | |
| 1688 | } | |
| 1689 | } |
lib/std/zig/Package/Fetch/git/testdata/testrepo-sha1.idx created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/git/testdata/testrepo-sha1.idx differ |
lib/std/zig/Package/Fetch/git/testdata/testrepo-sha1.pack created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/git/testdata/testrepo-sha1.pack differ |
lib/std/zig/Package/Fetch/git/testdata/testrepo-sha256.idx created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/git/testdata/testrepo-sha256.idx differ |
lib/std/zig/Package/Fetch/git/testdata/testrepo-sha256.pack created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/git/testdata/testrepo-sha256.pack differ |
lib/std/zig/Package/Fetch/testdata/duplicate_paths.tar.gz created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/testdata/duplicate_paths.tar.gz differ |
lib/std/zig/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz differ |
lib/std/zig/Package/Fetch/testdata/executables.tar.gz created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/testdata/executables.tar.gz differ |
lib/std/zig/Package/Fetch/testdata/no_root.tar.gz created
| Binary files /dev/null and b/lib/std/zig/Package/Fetch/testdata/no_root.tar.gz differ |
lib/std/zig/Package/Manifest.zig created+704
| ... | ... | @@ -0,0 +1,704 @@ |
| 1 | const Manifest = @This(); | |
| 2 | const std = @import("std"); | |
| 3 | const mem = std.mem; | |
| 4 | const Allocator = std.mem.Allocator; | |
| 5 | const assert = std.debug.assert; | |
| 6 | const Ast = std.zig.Ast; | |
| 7 | const testing = std.testing; | |
| 8 | const Package = @import("../Package.zig"); | |
| 9 | ||
| 10 | pub const max_bytes = 10 * 1024 * 1024; | |
| 11 | pub const basename = "build.zig.zon"; | |
| 12 | pub const max_name_len = 32; | |
| 13 | pub const max_version_len = 32; | |
| 14 | ||
| 15 | pub const Dependency = struct { | |
| 16 | location: Location, | |
| 17 | location_tok: Ast.TokenIndex, | |
| 18 | location_node: Ast.Node.Index, | |
| 19 | hash: ?[]const u8, | |
| 20 | hash_tok: Ast.OptionalTokenIndex, | |
| 21 | hash_node: Ast.Node.OptionalIndex, | |
| 22 | node: Ast.Node.Index, | |
| 23 | name_tok: Ast.TokenIndex, | |
| 24 | lazy: bool, | |
| 25 | ||
| 26 | pub const Location = union(enum) { | |
| 27 | url: []const u8, | |
| 28 | path: []const u8, | |
| 29 | }; | |
| 30 | }; | |
| 31 | ||
| 32 | pub const ErrorMessage = struct { | |
| 33 | msg: []const u8, | |
| 34 | tok: Ast.TokenIndex, | |
| 35 | off: u32, | |
| 36 | }; | |
| 37 | ||
| 38 | name: []const u8, | |
| 39 | id: u32, | |
| 40 | version: std.SemanticVersion, | |
| 41 | version_node: Ast.Node.Index, | |
| 42 | dependencies: std.StringArrayHashMapUnmanaged(Dependency), | |
| 43 | dependencies_node: Ast.Node.OptionalIndex, | |
| 44 | paths: std.StringArrayHashMapUnmanaged(void), | |
| 45 | minimum_zig_version: ?std.SemanticVersion, | |
| 46 | ||
| 47 | errors: []ErrorMessage, | |
| 48 | arena_state: std.heap.ArenaAllocator.State, | |
| 49 | ||
| 50 | pub const ParseOptions = struct { | |
| 51 | allow_missing_paths_field: bool = false, | |
| 52 | /// Deprecated, to be removed after 0.14.0 is tagged. | |
| 53 | allow_name_string: bool = true, | |
| 54 | /// Deprecated, to be removed after 0.14.0 is tagged. | |
| 55 | allow_missing_fingerprint: bool = true, | |
| 56 | }; | |
| 57 | ||
| 58 | pub const Error = Allocator.Error; | |
| 59 | ||
| 60 | pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest { | |
| 61 | const main_node_index = ast.nodeData(.root).node; | |
| 62 | ||
| 63 | var arena_instance = std.heap.ArenaAllocator.init(gpa); | |
| 64 | errdefer arena_instance.deinit(); | |
| 65 | ||
| 66 | var p: Parse = .{ | |
| 67 | .gpa = gpa, | |
| 68 | .ast = ast, | |
| 69 | .arena = arena_instance.allocator(), | |
| 70 | .errors = .{}, | |
| 71 | ||
| 72 | .name = undefined, | |
| 73 | .id = 0, | |
| 74 | .version = undefined, | |
| 75 | .version_node = undefined, | |
| 76 | .dependencies = .{}, | |
| 77 | .dependencies_node = .none, | |
| 78 | .paths = .{}, | |
| 79 | .allow_missing_paths_field = options.allow_missing_paths_field, | |
| 80 | .allow_name_string = options.allow_name_string, | |
| 81 | .allow_missing_fingerprint = options.allow_missing_fingerprint, | |
| 82 | .minimum_zig_version = null, | |
| 83 | .buf = .{}, | |
| 84 | }; | |
| 85 | defer p.buf.deinit(gpa); | |
| 86 | defer p.errors.deinit(gpa); | |
| 87 | defer p.dependencies.deinit(gpa); | |
| 88 | defer p.paths.deinit(gpa); | |
| 89 | ||
| 90 | p.parseRoot(main_node_index) catch |err| switch (err) { | |
| 91 | error.ParseFailure => assert(p.errors.items.len > 0), | |
| 92 | else => |e| return e, | |
| 93 | }; | |
| 94 | ||
| 95 | return .{ | |
| 96 | .name = p.name, | |
| 97 | .id = p.id, | |
| 98 | .version = p.version, | |
| 99 | .version_node = p.version_node, | |
| 100 | .dependencies = try p.dependencies.clone(p.arena), | |
| 101 | .dependencies_node = p.dependencies_node, | |
| 102 | .paths = try p.paths.clone(p.arena), | |
| 103 | .minimum_zig_version = p.minimum_zig_version, | |
| 104 | .errors = try p.arena.dupe(ErrorMessage, p.errors.items), | |
| 105 | .arena_state = arena_instance.state, | |
| 106 | }; | |
| 107 | } | |
| 108 | ||
| 109 | pub fn deinit(man: *Manifest, gpa: Allocator) void { | |
| 110 | man.arena_state.promote(gpa).deinit(); | |
| 111 | man.* = undefined; | |
| 112 | } | |
| 113 | ||
| 114 | pub fn copyErrorsIntoBundle( | |
| 115 | man: Manifest, | |
| 116 | ast: Ast, | |
| 117 | /// ErrorBundle null-terminated string index | |
| 118 | src_path: u32, | |
| 119 | eb: *std.zig.ErrorBundle.Wip, | |
| 120 | ) Allocator.Error!void { | |
| 121 | for (man.errors) |msg| { | |
| 122 | const start_loc = ast.tokenLocation(0, msg.tok); | |
| 123 | ||
| 124 | try eb.addRootErrorMessage(.{ | |
| 125 | .msg = try eb.addString(msg.msg), | |
| 126 | .src_loc = try eb.addSourceLocation(.{ | |
| 127 | .src_path = src_path, | |
| 128 | .span_start = ast.tokenStart(msg.tok), | |
| 129 | .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), | |
| 130 | .span_main = ast.tokenStart(msg.tok) + msg.off, | |
| 131 | .line = @intCast(start_loc.line), | |
| 132 | .column = @intCast(start_loc.column), | |
| 133 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | |
| 134 | }), | |
| 135 | }); | |
| 136 | } | |
| 137 | } | |
| 138 | ||
| 139 | const Parse = struct { | |
| 140 | gpa: Allocator, | |
| 141 | ast: Ast, | |
| 142 | arena: Allocator, | |
| 143 | buf: std.ArrayListUnmanaged(u8), | |
| 144 | errors: std.ArrayListUnmanaged(ErrorMessage), | |
| 145 | ||
| 146 | name: []const u8, | |
| 147 | id: u32, | |
| 148 | version: std.SemanticVersion, | |
| 149 | version_node: Ast.Node.Index, | |
| 150 | dependencies: std.StringArrayHashMapUnmanaged(Dependency), | |
| 151 | dependencies_node: Ast.Node.OptionalIndex, | |
| 152 | paths: std.StringArrayHashMapUnmanaged(void), | |
| 153 | allow_missing_paths_field: bool, | |
| 154 | allow_name_string: bool, | |
| 155 | allow_missing_fingerprint: bool, | |
| 156 | minimum_zig_version: ?std.SemanticVersion, | |
| 157 | ||
| 158 | const InnerError = error{ ParseFailure, OutOfMemory }; | |
| 159 | ||
| 160 | fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { | |
| 161 | const ast = p.ast; | |
| 162 | const main_token = ast.nodeMainToken(node); | |
| 163 | ||
| 164 | var buf: [2]Ast.Node.Index = undefined; | |
| 165 | const struct_init = ast.fullStructInit(&buf, node) orelse { | |
| 166 | return fail(p, main_token, "expected top level expression to be a struct", .{}); | |
| 167 | }; | |
| 168 | ||
| 169 | var have_name = false; | |
| 170 | var have_version = false; | |
| 171 | var have_included_paths = false; | |
| 172 | var fingerprint: ?Package.Fingerprint = null; | |
| 173 | ||
| 174 | for (struct_init.ast.fields) |field_init| { | |
| 175 | const name_token = ast.firstToken(field_init) - 2; | |
| 176 | const field_name = try identifierTokenString(p, name_token); | |
| 177 | // We could get fancy with reflection and comptime logic here but doing | |
| 178 | // things manually provides an opportunity to do any additional verification | |
| 179 | // that is desirable on a per-field basis. | |
| 180 | if (mem.eql(u8, field_name, "dependencies")) { | |
| 181 | p.dependencies_node = field_init.toOptional(); | |
| 182 | try parseDependencies(p, field_init); | |
| 183 | } else if (mem.eql(u8, field_name, "paths")) { | |
| 184 | have_included_paths = true; | |
| 185 | try parseIncludedPaths(p, field_init); | |
| 186 | } else if (mem.eql(u8, field_name, "name")) { | |
| 187 | p.name = try parseName(p, field_init); | |
| 188 | have_name = true; | |
| 189 | } else if (mem.eql(u8, field_name, "fingerprint")) { | |
| 190 | fingerprint = try parseFingerprint(p, field_init); | |
| 191 | } else if (mem.eql(u8, field_name, "version")) { | |
| 192 | p.version_node = field_init; | |
| 193 | const version_text = try parseString(p, field_init); | |
| 194 | if (version_text.len > max_version_len) { | |
| 195 | try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); | |
| 196 | } | |
| 197 | p.version = std.SemanticVersion.parse(version_text) catch |err| v: { | |
| 198 | try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); | |
| 199 | break :v undefined; | |
| 200 | }; | |
| 201 | have_version = true; | |
| 202 | } else if (mem.eql(u8, field_name, "minimum_zig_version")) { | |
| 203 | const version_text = try parseString(p, field_init); | |
| 204 | p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { | |
| 205 | try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); | |
| 206 | break :v null; | |
| 207 | }; | |
| 208 | } else { | |
| 209 | // Ignore unknown fields so that we can add fields in future zig | |
| 210 | // versions without breaking older zig versions. | |
| 211 | } | |
| 212 | } | |
| 213 | ||
| 214 | if (!have_name) { | |
| 215 | try appendError(p, main_token, "missing top-level 'name' field", .{}); | |
| 216 | } else { | |
| 217 | if (fingerprint) |n| { | |
| 218 | if (!n.validate(p.name)) { | |
| 219 | return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ | |
| 220 | n.int(), Package.Fingerprint.generate(p.name).int(), | |
| 221 | }); | |
| 222 | } | |
| 223 | p.id = n.id; | |
| 224 | } else if (!p.allow_missing_fingerprint) { | |
| 225 | try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ | |
| 226 | Package.Fingerprint.generate(p.name).int(), | |
| 227 | }); | |
| 228 | } else { | |
| 229 | p.id = 0; | |
| 230 | } | |
| 231 | } | |
| 232 | ||
| 233 | if (!have_version) { | |
| 234 | try appendError(p, main_token, "missing top-level 'version' field", .{}); | |
| 235 | } | |
| 236 | ||
| 237 | if (!have_included_paths) { | |
| 238 | if (p.allow_missing_paths_field) { | |
| 239 | try p.paths.put(p.gpa, "", {}); | |
| 240 | } else { | |
| 241 | try appendError(p, main_token, "missing top-level 'paths' field", .{}); | |
| 242 | } | |
| 243 | } | |
| 244 | } | |
| 245 | ||
| 246 | fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { | |
| 247 | const ast = p.ast; | |
| 248 | ||
| 249 | var buf: [2]Ast.Node.Index = undefined; | |
| 250 | const struct_init = ast.fullStructInit(&buf, node) orelse { | |
| 251 | const tok = ast.nodeMainToken(node); | |
| 252 | return fail(p, tok, "expected dependencies expression to be a struct", .{}); | |
| 253 | }; | |
| 254 | ||
| 255 | for (struct_init.ast.fields) |field_init| { | |
| 256 | const name_token = ast.firstToken(field_init) - 2; | |
| 257 | const dep_name = try identifierTokenString(p, name_token); | |
| 258 | const dep = try parseDependency(p, field_init); | |
| 259 | try p.dependencies.put(p.gpa, dep_name, dep); | |
| 260 | } | |
| 261 | } | |
| 262 | ||
| 263 | fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { | |
| 264 | const ast = p.ast; | |
| 265 | ||
| 266 | var buf: [2]Ast.Node.Index = undefined; | |
| 267 | const struct_init = ast.fullStructInit(&buf, node) orelse { | |
| 268 | const tok = ast.nodeMainToken(node); | |
| 269 | return fail(p, tok, "expected dependency expression to be a struct", .{}); | |
| 270 | }; | |
| 271 | ||
| 272 | var dep: Dependency = .{ | |
| 273 | .location = undefined, | |
| 274 | .location_tok = undefined, | |
| 275 | .location_node = undefined, | |
| 276 | .hash = null, | |
| 277 | .hash_tok = .none, | |
| 278 | .hash_node = .none, | |
| 279 | .node = node, | |
| 280 | .name_tok = undefined, | |
| 281 | .lazy = false, | |
| 282 | }; | |
| 283 | var has_location = false; | |
| 284 | ||
| 285 | for (struct_init.ast.fields) |field_init| { | |
| 286 | const name_token = ast.firstToken(field_init) - 2; | |
| 287 | dep.name_tok = name_token; | |
| 288 | const field_name = try identifierTokenString(p, name_token); | |
| 289 | // We could get fancy with reflection and comptime logic here but doing | |
| 290 | // things manually provides an opportunity to do any additional verification | |
| 291 | // that is desirable on a per-field basis. | |
| 292 | if (mem.eql(u8, field_name, "url")) { | |
| 293 | if (has_location) { | |
| 294 | return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); | |
| 295 | } | |
| 296 | dep.location = .{ | |
| 297 | .url = parseString(p, field_init) catch |err| switch (err) { | |
| 298 | error.ParseFailure => continue, | |
| 299 | else => |e| return e, | |
| 300 | }, | |
| 301 | }; | |
| 302 | has_location = true; | |
| 303 | dep.location_tok = ast.nodeMainToken(field_init); | |
| 304 | dep.location_node = field_init; | |
| 305 | } else if (mem.eql(u8, field_name, "path")) { | |
| 306 | if (has_location) { | |
| 307 | return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); | |
| 308 | } | |
| 309 | dep.location = .{ | |
| 310 | .path = parseString(p, field_init) catch |err| switch (err) { | |
| 311 | error.ParseFailure => continue, | |
| 312 | else => |e| return e, | |
| 313 | }, | |
| 314 | }; | |
| 315 | has_location = true; | |
| 316 | dep.location_tok = ast.nodeMainToken(field_init); | |
| 317 | dep.location_node = field_init; | |
| 318 | } else if (mem.eql(u8, field_name, "hash")) { | |
| 319 | dep.hash = parseHash(p, field_init) catch |err| switch (err) { | |
| 320 | error.ParseFailure => continue, | |
| 321 | else => |e| return e, | |
| 322 | }; | |
| 323 | dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); | |
| 324 | dep.hash_node = field_init.toOptional(); | |
| 325 | } else if (mem.eql(u8, field_name, "lazy")) { | |
| 326 | dep.lazy = parseBool(p, field_init) catch |err| switch (err) { | |
| 327 | error.ParseFailure => continue, | |
| 328 | else => |e| return e, | |
| 329 | }; | |
| 330 | } else { | |
| 331 | // Ignore unknown fields so that we can add fields in future zig | |
| 332 | // versions without breaking older zig versions. | |
| 333 | } | |
| 334 | } | |
| 335 | ||
| 336 | if (!has_location) { | |
| 337 | try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); | |
| 338 | } | |
| 339 | ||
| 340 | return dep; | |
| 341 | } | |
| 342 | ||
| 343 | fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { | |
| 344 | const ast = p.ast; | |
| 345 | ||
| 346 | var buf: [2]Ast.Node.Index = undefined; | |
| 347 | const array_init = ast.fullArrayInit(&buf, node) orelse { | |
| 348 | const tok = ast.nodeMainToken(node); | |
| 349 | return fail(p, tok, "expected paths expression to be a list of strings", .{}); | |
| 350 | }; | |
| 351 | ||
| 352 | for (array_init.ast.elements) |elem_node| { | |
| 353 | const path_string = try parseString(p, elem_node); | |
| 354 | // This is normalized so that it can be used in string comparisons | |
| 355 | // against file system paths. | |
| 356 | const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); | |
| 357 | try p.paths.put(p.gpa, normalized, {}); | |
| 358 | } | |
| 359 | } | |
| 360 | ||
| 361 | fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { | |
| 362 | const ast = p.ast; | |
| 363 | if (ast.nodeTag(node) != .identifier) { | |
| 364 | return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); | |
| 365 | } | |
| 366 | const ident_token = ast.nodeMainToken(node); | |
| 367 | const token_bytes = ast.tokenSlice(ident_token); | |
| 368 | if (mem.eql(u8, token_bytes, "true")) { | |
| 369 | return true; | |
| 370 | } else if (mem.eql(u8, token_bytes, "false")) { | |
| 371 | return false; | |
| 372 | } else { | |
| 373 | return fail(p, ident_token, "expected boolean", .{}); | |
| 374 | } | |
| 375 | } | |
| 376 | ||
| 377 | fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { | |
| 378 | const ast = p.ast; | |
| 379 | const main_token = ast.nodeMainToken(node); | |
| 380 | if (ast.nodeTag(node) != .number_literal) { | |
| 381 | return fail(p, main_token, "expected integer literal", .{}); | |
| 382 | } | |
| 383 | const token_bytes = ast.tokenSlice(main_token); | |
| 384 | const parsed = std.zig.parseNumberLiteral(token_bytes); | |
| 385 | switch (parsed) { | |
| 386 | .int => |n| return @bitCast(n), | |
| 387 | .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ | |
| 388 | @tagName(parsed), | |
| 389 | }), | |
| 390 | .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), | |
| 391 | } | |
| 392 | } | |
| 393 | ||
| 394 | fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { | |
| 395 | const ast = p.ast; | |
| 396 | const main_token = ast.nodeMainToken(node); | |
| 397 | ||
| 398 | if (p.allow_name_string and ast.nodeTag(node) == .string_literal) { | |
| 399 | const name = try parseString(p, node); | |
| 400 | if (!std.zig.isValidId(name)) | |
| 401 | return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{}); | |
| 402 | ||
| 403 | if (name.len > max_name_len) | |
| 404 | return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ | |
| 405 | std.zig.fmtId(name), max_name_len, | |
| 406 | }); | |
| 407 | ||
| 408 | return name; | |
| 409 | } | |
| 410 | ||
| 411 | if (ast.nodeTag(node) != .enum_literal) | |
| 412 | return fail(p, main_token, "expected enum literal", .{}); | |
| 413 | ||
| 414 | const ident_name = ast.tokenSlice(main_token); | |
| 415 | if (mem.startsWith(u8, ident_name, "@")) | |
| 416 | return fail(p, main_token, "name must be a valid bare zig identifier", .{}); | |
| 417 | ||
| 418 | if (ident_name.len > max_name_len) | |
| 419 | return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ | |
| 420 | std.zig.fmtId(ident_name), max_name_len, | |
| 421 | }); | |
| 422 | ||
| 423 | return ident_name; | |
| 424 | } | |
| 425 | ||
| 426 | fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { | |
| 427 | const ast = p.ast; | |
| 428 | if (ast.nodeTag(node) != .string_literal) { | |
| 429 | return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); | |
| 430 | } | |
| 431 | const str_lit_token = ast.nodeMainToken(node); | |
| 432 | const token_bytes = ast.tokenSlice(str_lit_token); | |
| 433 | p.buf.clearRetainingCapacity(); | |
| 434 | try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); | |
| 435 | const duped = try p.arena.dupe(u8, p.buf.items); | |
| 436 | return duped; | |
| 437 | } | |
| 438 | ||
| 439 | fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { | |
| 440 | const ast = p.ast; | |
| 441 | const tok = ast.nodeMainToken(node); | |
| 442 | const h = try parseString(p, node); | |
| 443 | ||
| 444 | if (h.len > Package.Hash.max_len) { | |
| 445 | return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len}); | |
| 446 | } | |
| 447 | ||
| 448 | return h; | |
| 449 | } | |
| 450 | ||
| 451 | /// TODO: try to DRY this with AstGen.identifierTokenString | |
| 452 | fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { | |
| 453 | const ast = p.ast; | |
| 454 | assert(ast.tokenTag(token) == .identifier); | |
| 455 | const ident_name = ast.tokenSlice(token); | |
| 456 | if (!mem.startsWith(u8, ident_name, "@")) { | |
| 457 | return ident_name; | |
| 458 | } | |
| 459 | p.buf.clearRetainingCapacity(); | |
| 460 | try parseStrLit(p, token, &p.buf, ident_name, 1); | |
| 461 | const duped = try p.arena.dupe(u8, p.buf.items); | |
| 462 | return duped; | |
| 463 | } | |
| 464 | ||
| 465 | /// TODO: try to DRY this with AstGen.parseStrLit | |
| 466 | fn parseStrLit( | |
| 467 | p: *Parse, | |
| 468 | token: Ast.TokenIndex, | |
| 469 | buf: *std.ArrayListUnmanaged(u8), | |
| 470 | bytes: []const u8, | |
| 471 | offset: u32, | |
| 472 | ) InnerError!void { | |
| 473 | const raw_string = bytes[offset..]; | |
| 474 | var buf_managed = buf.toManaged(p.gpa); | |
| 475 | const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); | |
| 476 | buf.* = buf_managed.moveToUnmanaged(); | |
| 477 | switch (try result) { | |
| 478 | .success => {}, | |
| 479 | .failure => |err| try p.appendStrLitError(err, token, bytes, offset), | |
| 480 | } | |
| 481 | } | |
| 482 | ||
| 483 | /// TODO: try to DRY this with AstGen.failWithStrLitError | |
| 484 | fn appendStrLitError( | |
| 485 | p: *Parse, | |
| 486 | err: std.zig.string_literal.Error, | |
| 487 | token: Ast.TokenIndex, | |
| 488 | bytes: []const u8, | |
| 489 | offset: u32, | |
| 490 | ) Allocator.Error!void { | |
| 491 | const raw_string = bytes[offset..]; | |
| 492 | switch (err) { | |
| 493 | .invalid_escape_character => |bad_index| { | |
| 494 | try p.appendErrorOff( | |
| 495 | token, | |
| 496 | offset + @as(u32, @intCast(bad_index)), | |
| 497 | "invalid escape character: '{c}'", | |
| 498 | .{raw_string[bad_index]}, | |
| 499 | ); | |
| 500 | }, | |
| 501 | .expected_hex_digit => |bad_index| { | |
| 502 | try p.appendErrorOff( | |
| 503 | token, | |
| 504 | offset + @as(u32, @intCast(bad_index)), | |
| 505 | "expected hex digit, found '{c}'", | |
| 506 | .{raw_string[bad_index]}, | |
| 507 | ); | |
| 508 | }, | |
| 509 | .empty_unicode_escape_sequence => |bad_index| { | |
| 510 | try p.appendErrorOff( | |
| 511 | token, | |
| 512 | offset + @as(u32, @intCast(bad_index)), | |
| 513 | "empty unicode escape sequence", | |
| 514 | .{}, | |
| 515 | ); | |
| 516 | }, | |
| 517 | .expected_hex_digit_or_rbrace => |bad_index| { | |
| 518 | try p.appendErrorOff( | |
| 519 | token, | |
| 520 | offset + @as(u32, @intCast(bad_index)), | |
| 521 | "expected hex digit or '}}', found '{c}'", | |
| 522 | .{raw_string[bad_index]}, | |
| 523 | ); | |
| 524 | }, | |
| 525 | .invalid_unicode_codepoint => |bad_index| { | |
| 526 | try p.appendErrorOff( | |
| 527 | token, | |
| 528 | offset + @as(u32, @intCast(bad_index)), | |
| 529 | "unicode escape does not correspond to a valid unicode scalar value", | |
| 530 | .{}, | |
| 531 | ); | |
| 532 | }, | |
| 533 | .expected_lbrace => |bad_index| { | |
| 534 | try p.appendErrorOff( | |
| 535 | token, | |
| 536 | offset + @as(u32, @intCast(bad_index)), | |
| 537 | "expected '{{', found '{c}", | |
| 538 | .{raw_string[bad_index]}, | |
| 539 | ); | |
| 540 | }, | |
| 541 | .expected_rbrace => |bad_index| { | |
| 542 | try p.appendErrorOff( | |
| 543 | token, | |
| 544 | offset + @as(u32, @intCast(bad_index)), | |
| 545 | "expected '}}', found '{c}", | |
| 546 | .{raw_string[bad_index]}, | |
| 547 | ); | |
| 548 | }, | |
| 549 | .expected_single_quote => |bad_index| { | |
| 550 | try p.appendErrorOff( | |
| 551 | token, | |
| 552 | offset + @as(u32, @intCast(bad_index)), | |
| 553 | "expected single quote ('), found '{c}", | |
| 554 | .{raw_string[bad_index]}, | |
| 555 | ); | |
| 556 | }, | |
| 557 | .invalid_character => |bad_index| { | |
| 558 | try p.appendErrorOff( | |
| 559 | token, | |
| 560 | offset + @as(u32, @intCast(bad_index)), | |
| 561 | "invalid byte in string or character literal: '{c}'", | |
| 562 | .{raw_string[bad_index]}, | |
| 563 | ); | |
| 564 | }, | |
| 565 | .empty_char_literal => { | |
| 566 | try p.appendErrorOff(token, offset, "empty character literal", .{}); | |
| 567 | }, | |
| 568 | } | |
| 569 | } | |
| 570 | ||
| 571 | fn fail( | |
| 572 | p: *Parse, | |
| 573 | tok: Ast.TokenIndex, | |
| 574 | comptime fmt: []const u8, | |
| 575 | args: anytype, | |
| 576 | ) InnerError { | |
| 577 | try appendError(p, tok, fmt, args); | |
| 578 | return error.ParseFailure; | |
| 579 | } | |
| 580 | ||
| 581 | fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { | |
| 582 | return appendErrorOff(p, tok, 0, fmt, args); | |
| 583 | } | |
| 584 | ||
| 585 | fn appendErrorOff( | |
| 586 | p: *Parse, | |
| 587 | tok: Ast.TokenIndex, | |
| 588 | byte_offset: u32, | |
| 589 | comptime fmt: []const u8, | |
| 590 | args: anytype, | |
| 591 | ) Allocator.Error!void { | |
| 592 | try p.errors.append(p.gpa, .{ | |
| 593 | .msg = try std.fmt.allocPrint(p.arena, fmt, args), | |
| 594 | .tok = tok, | |
| 595 | .off = byte_offset, | |
| 596 | }); | |
| 597 | } | |
| 598 | }; | |
| 599 | ||
| 600 | test "basic" { | |
| 601 | const gpa = testing.allocator; | |
| 602 | ||
| 603 | const example = | |
| 604 | \\.{ | |
| 605 | \\ .name = "foo", | |
| 606 | \\ .version = "3.2.1", | |
| 607 | \\ .paths = .{""}, | |
| 608 | \\ .dependencies = .{ | |
| 609 | \\ .bar = .{ | |
| 610 | \\ .url = "https://example.com/baz.tar.gz", | |
| 611 | \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", | |
| 612 | \\ }, | |
| 613 | \\ }, | |
| 614 | \\} | |
| 615 | ; | |
| 616 | ||
| 617 | var ast = try Ast.parse(gpa, example, .zon); | |
| 618 | defer ast.deinit(gpa); | |
| 619 | ||
| 620 | try testing.expect(ast.errors.len == 0); | |
| 621 | ||
| 622 | var manifest = try Manifest.parse(gpa, ast, .{}); | |
| 623 | defer manifest.deinit(gpa); | |
| 624 | ||
| 625 | try testing.expect(manifest.errors.len == 0); | |
| 626 | try testing.expectEqualStrings("foo", manifest.name); | |
| 627 | ||
| 628 | try testing.expectEqual(@as(std.SemanticVersion, .{ | |
| 629 | .major = 3, | |
| 630 | .minor = 2, | |
| 631 | .patch = 1, | |
| 632 | }), manifest.version); | |
| 633 | ||
| 634 | try testing.expect(manifest.dependencies.count() == 1); | |
| 635 | try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); | |
| 636 | try testing.expectEqualStrings( | |
| 637 | "https://example.com/baz.tar.gz", | |
| 638 | manifest.dependencies.values()[0].location.url, | |
| 639 | ); | |
| 640 | try testing.expectEqualStrings( | |
| 641 | "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", | |
| 642 | manifest.dependencies.values()[0].hash orelse return error.TestFailed, | |
| 643 | ); | |
| 644 | ||
| 645 | try testing.expect(manifest.minimum_zig_version == null); | |
| 646 | } | |
| 647 | ||
| 648 | test "minimum_zig_version" { | |
| 649 | const gpa = testing.allocator; | |
| 650 | ||
| 651 | const example = | |
| 652 | \\.{ | |
| 653 | \\ .name = "foo", | |
| 654 | \\ .version = "3.2.1", | |
| 655 | \\ .paths = .{""}, | |
| 656 | \\ .minimum_zig_version = "0.11.1", | |
| 657 | \\} | |
| 658 | ; | |
| 659 | ||
| 660 | var ast = try Ast.parse(gpa, example, .zon); | |
| 661 | defer ast.deinit(gpa); | |
| 662 | ||
| 663 | try testing.expect(ast.errors.len == 0); | |
| 664 | ||
| 665 | var manifest = try Manifest.parse(gpa, ast, .{}); | |
| 666 | defer manifest.deinit(gpa); | |
| 667 | ||
| 668 | try testing.expect(manifest.errors.len == 0); | |
| 669 | try testing.expect(manifest.dependencies.count() == 0); | |
| 670 | ||
| 671 | try testing.expect(manifest.minimum_zig_version != null); | |
| 672 | ||
| 673 | try testing.expectEqual(@as(std.SemanticVersion, .{ | |
| 674 | .major = 0, | |
| 675 | .minor = 11, | |
| 676 | .patch = 1, | |
| 677 | }), manifest.minimum_zig_version.?); | |
| 678 | } | |
| 679 | ||
| 680 | test "minimum_zig_version - invalid version" { | |
| 681 | const gpa = testing.allocator; | |
| 682 | ||
| 683 | const example = | |
| 684 | \\.{ | |
| 685 | \\ .name = "foo", | |
| 686 | \\ .version = "3.2.1", | |
| 687 | \\ .minimum_zig_version = "X.11.1", | |
| 688 | \\ .paths = .{""}, | |
| 689 | \\} | |
| 690 | ; | |
| 691 | ||
| 692 | var ast = try Ast.parse(gpa, example, .zon); | |
| 693 | defer ast.deinit(gpa); | |
| 694 | ||
| 695 | try testing.expect(ast.errors.len == 0); | |
| 696 | ||
| 697 | var manifest = try Manifest.parse(gpa, ast, .{}); | |
| 698 | defer manifest.deinit(gpa); | |
| 699 | ||
| 700 | try testing.expect(manifest.errors.len == 1); | |
| 701 | try testing.expect(manifest.dependencies.count() == 0); | |
| 702 | ||
| 703 | try testing.expect(manifest.minimum_zig_version == null); | |
| 704 | } |
lib/std/zig/Package/Templates.zig created+90
| ... | ... | @@ -0,0 +1,90 @@ |
| 1 | const std = @import("../../std.zig"); | |
| 2 | const Directory = std.Build.Cache.Directory; | |
| 3 | const fs = std.fs; | |
| 4 | const Allocator = std.mem.Allocator; | |
| 5 | const fatal = std.process.fatal; | |
| 6 | ||
| 7 | const Templates = @This(); | |
| 8 | ||
| 9 | zig_lib_directory: Directory, | |
| 10 | dir: fs.Dir, | |
| 11 | buffer: std.ArrayListUnmanaged(u8), | |
| 12 | ||
| 13 | fn find(gpa: Allocator, zig_lib_directory: Directory) Templates { | |
| 14 | const s = fs.path.sep_str; | |
| 15 | const template_sub_path = "init"; | |
| 16 | const template_dir = zig_lib_directory.handle.openDir(template_sub_path, .{}) catch |err| { | |
| 17 | const path = zig_lib_directory.path orelse "."; | |
| 18 | fatal("unable to open zig project template directory '{s}{s}{s}': {s}", .{ | |
| 19 | path, s, template_sub_path, @errorName(err), | |
| 20 | }); | |
| 21 | }; | |
| 22 | ||
| 23 | return .{ | |
| 24 | .zig_lib_directory = zig_lib_directory, | |
| 25 | .dir = template_dir, | |
| 26 | .buffer = std.ArrayListUnmanaged(u8).init(gpa), | |
| 27 | }; | |
| 28 | } | |
| 29 | ||
| 30 | fn deinit(templates: *Templates, gpa: Allocator) void { | |
| 31 | templates.zig_lib_directory.handle.close(); | |
| 32 | templates.dir.close(); | |
| 33 | templates.buffer.deinit(gpa); | |
| 34 | templates.* = undefined; | |
| 35 | } | |
| 36 | ||
| 37 | fn write( | |
| 38 | templates: *Templates, | |
| 39 | gpa: Allocator, | |
| 40 | out_dir: fs.Dir, | |
| 41 | root_name: []const u8, | |
| 42 | template_path: []const u8, | |
| 43 | fingerprint: std.zig.Package.Fingerprint, | |
| 44 | zig_version_string: []const u8, | |
| 45 | ) !void { | |
| 46 | if (fs.path.dirname(template_path)) |dirname| { | |
| 47 | out_dir.makePath(dirname) catch |err| { | |
| 48 | fatal("unable to make path '{s}': {s}", .{ dirname, @errorName(err) }); | |
| 49 | }; | |
| 50 | } | |
| 51 | ||
| 52 | const max_bytes = 10 * 1024 * 1024; | |
| 53 | const contents = templates.dir.readFileAlloc(gpa, template_path, max_bytes) catch |err| { | |
| 54 | fatal("unable to read template file '{s}': {s}", .{ template_path, @errorName(err) }); | |
| 55 | }; | |
| 56 | defer gpa.free(contents); | |
| 57 | templates.buffer.clearRetainingCapacity(); | |
| 58 | try templates.buffer.ensureUnusedCapacity(gpa, contents.len); | |
| 59 | var i: usize = 0; | |
| 60 | while (i < contents.len) { | |
| 61 | if (contents[i] == '.') { | |
| 62 | if (std.mem.startsWith(u8, contents[i..], ".LITNAME")) { | |
| 63 | try templates.buffer.append(gpa, '.'); | |
| 64 | try templates.buffer.appendSlice(gpa, root_name); | |
| 65 | i += ".LITNAME".len; | |
| 66 | continue; | |
| 67 | } else if (std.mem.startsWith(u8, contents[i..], ".NAME")) { | |
| 68 | try templates.buffer.appendSlice(gpa, root_name); | |
| 69 | i += ".NAME".len; | |
| 70 | continue; | |
| 71 | } else if (std.mem.startsWith(u8, contents[i..], ".FINGERPRINT")) { | |
| 72 | try templates.buffer.writer(gpa).print("0x{x}", .{fingerprint.int()}); | |
| 73 | i += ".FINGERPRINT".len; | |
| 74 | continue; | |
| 75 | } else if (std.mem.startsWith(u8, contents[i..], ".ZIGVER")) { | |
| 76 | try templates.buffer.appendSlice(gpa, zig_version_string); | |
| 77 | i += ".ZIGVER".len; | |
| 78 | continue; | |
| 79 | } | |
| 80 | } | |
| 81 | try templates.buffer.append(gpa, contents[i]); | |
| 82 | i += 1; | |
| 83 | } | |
| 84 | ||
| 85 | return out_dir.writeFile(.{ | |
| 86 | .sub_path = template_path, | |
| 87 | .data = templates.buffer.items, | |
| 88 | .flags = .{ .exclusive = true }, | |
| 89 | }); | |
| 90 | } |
src/Module.zig created+563
| ... | ... | @@ -0,0 +1,563 @@ |
| 1 | //! Corresponds to something that Zig source code can `@import`. | |
| 2 | ||
| 3 | /// Only files inside this directory can be imported. | |
| 4 | root: Cache.Path, | |
| 5 | /// Relative to `root`. May contain path separators. | |
| 6 | root_src_path: []const u8, | |
| 7 | /// Name used in compile errors. Looks like "root.foo.bar". | |
| 8 | fully_qualified_name: []const u8, | |
| 9 | /// The dependency table of this module. Shared dependencies such as 'std', | |
| 10 | /// 'builtin', and 'root' are not specified in every dependency table, but | |
| 11 | /// instead only in the table of `main_mod`. `Module.importFile` is | |
| 12 | /// responsible for detecting these names and using the correct package. | |
| 13 | deps: Deps = .{}, | |
| 14 | ||
| 15 | resolved_target: ResolvedTarget, | |
| 16 | optimize_mode: std.builtin.OptimizeMode, | |
| 17 | code_model: std.builtin.CodeModel, | |
| 18 | single_threaded: bool, | |
| 19 | error_tracing: bool, | |
| 20 | valgrind: bool, | |
| 21 | pic: bool, | |
| 22 | strip: bool, | |
| 23 | omit_frame_pointer: bool, | |
| 24 | stack_check: bool, | |
| 25 | stack_protector: u32, | |
| 26 | red_zone: bool, | |
| 27 | sanitize_c: bool, | |
| 28 | sanitize_thread: bool, | |
| 29 | fuzz: bool, | |
| 30 | unwind_tables: std.builtin.UnwindTables, | |
| 31 | cc_argv: []const []const u8, | |
| 32 | /// (SPIR-V) whether to generate a structured control flow graph or not | |
| 33 | structured_cfg: bool, | |
| 34 | no_builtin: bool, | |
| 35 | ||
| 36 | /// If the module is an `@import("builtin")` module, this is the `File` that | |
| 37 | /// is preallocated for it. Otherwise this field is null. | |
| 38 | builtin_file: ?*File, | |
| 39 | ||
| 40 | pub const Deps = std.StringArrayHashMapUnmanaged(*Module); | |
| 41 | ||
| 42 | pub fn isBuiltin(m: Module) bool { | |
| 43 | return m.builtin_file != null; | |
| 44 | } | |
| 45 | ||
| 46 | pub const Tree = struct { | |
| 47 | /// Each `Package` exposes a `Module` with build.zig as its root source file. | |
| 48 | build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), | |
| 49 | }; | |
| 50 | ||
| 51 | pub const CreateOptions = struct { | |
| 52 | /// Where to store builtin.zig. The global cache directory is used because | |
| 53 | /// it is a pure function based on CLI flags. | |
| 54 | global_cache_directory: Cache.Directory, | |
| 55 | paths: Paths, | |
| 56 | fully_qualified_name: []const u8, | |
| 57 | ||
| 58 | cc_argv: []const []const u8, | |
| 59 | inherited: Inherited, | |
| 60 | global: Compilation.Config, | |
| 61 | /// If this is null then `resolved_target` must be non-null. | |
| 62 | parent: ?*Package.Module, | |
| 63 | ||
| 64 | builtin_mod: ?*Package.Module, | |
| 65 | ||
| 66 | /// Allocated into the given `arena`. Should be shared across all module creations in a Compilation. | |
| 67 | /// Ignored if `builtin_mod` is passed or if `!have_zcu`. | |
| 68 | /// Otherwise, may be `null` only if this Compilation consists of a single module. | |
| 69 | builtin_modules: ?*std.StringHashMapUnmanaged(*Module), | |
| 70 | ||
| 71 | pub const Paths = struct { | |
| 72 | root: Cache.Path, | |
| 73 | /// Relative to `root`. May contain path separators. | |
| 74 | root_src_path: []const u8, | |
| 75 | }; | |
| 76 | ||
| 77 | pub const Inherited = struct { | |
| 78 | /// If this is null then `parent` must be non-null. | |
| 79 | resolved_target: ?ResolvedTarget = null, | |
| 80 | optimize_mode: ?std.builtin.OptimizeMode = null, | |
| 81 | code_model: ?std.builtin.CodeModel = null, | |
| 82 | single_threaded: ?bool = null, | |
| 83 | error_tracing: ?bool = null, | |
| 84 | valgrind: ?bool = null, | |
| 85 | pic: ?bool = null, | |
| 86 | strip: ?bool = null, | |
| 87 | omit_frame_pointer: ?bool = null, | |
| 88 | stack_check: ?bool = null, | |
| 89 | /// null means default. | |
| 90 | /// 0 means no stack protector. | |
| 91 | /// other number means stack protection with that buffer size. | |
| 92 | stack_protector: ?u32 = null, | |
| 93 | red_zone: ?bool = null, | |
| 94 | unwind_tables: ?std.builtin.UnwindTables = null, | |
| 95 | sanitize_c: ?bool = null, | |
| 96 | sanitize_thread: ?bool = null, | |
| 97 | fuzz: ?bool = null, | |
| 98 | structured_cfg: ?bool = null, | |
| 99 | no_builtin: ?bool = null, | |
| 100 | }; | |
| 101 | }; | |
| 102 | ||
| 103 | pub const ResolvedTarget = struct { | |
| 104 | result: std.Target, | |
| 105 | is_native_os: bool, | |
| 106 | is_native_abi: bool, | |
| 107 | llvm_cpu_features: ?[*:0]const u8 = null, | |
| 108 | }; | |
| 109 | ||
| 110 | /// At least one of `parent` and `resolved_target` must be non-null. | |
| 111 | pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { | |
| 112 | if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); | |
| 113 | if (options.inherited.fuzz == true) assert(options.global.any_fuzz); | |
| 114 | if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); | |
| 115 | if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); | |
| 116 | if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); | |
| 117 | ||
| 118 | const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; | |
| 119 | const target = resolved_target.result; | |
| 120 | ||
| 121 | const optimize_mode = options.inherited.optimize_mode orelse | |
| 122 | if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; | |
| 123 | ||
| 124 | const strip = b: { | |
| 125 | if (options.inherited.strip) |x| break :b x; | |
| 126 | if (options.parent) |p| break :b p.strip; | |
| 127 | break :b options.global.root_strip; | |
| 128 | }; | |
| 129 | ||
| 130 | const valgrind = b: { | |
| 131 | if (!target_util.hasValgrindSupport(target)) { | |
| 132 | if (options.inherited.valgrind == true) | |
| 133 | return error.ValgrindUnsupportedOnTarget; | |
| 134 | break :b false; | |
| 135 | } | |
| 136 | if (options.inherited.valgrind) |x| break :b x; | |
| 137 | if (options.parent) |p| break :b p.valgrind; | |
| 138 | if (strip) break :b false; | |
| 139 | break :b optimize_mode == .Debug; | |
| 140 | }; | |
| 141 | ||
| 142 | const zig_backend = target_util.zigBackend(target, options.global.use_llvm); | |
| 143 | ||
| 144 | const single_threaded = b: { | |
| 145 | if (target_util.alwaysSingleThreaded(target)) { | |
| 146 | if (options.inherited.single_threaded == false) | |
| 147 | return error.TargetRequiresSingleThreaded; | |
| 148 | break :b true; | |
| 149 | } | |
| 150 | ||
| 151 | if (options.global.have_zcu) { | |
| 152 | if (!target_util.supportsThreads(target, zig_backend)) { | |
| 153 | if (options.inherited.single_threaded == false) | |
| 154 | return error.BackendRequiresSingleThreaded; | |
| 155 | break :b true; | |
| 156 | } | |
| 157 | } | |
| 158 | ||
| 159 | if (options.inherited.single_threaded) |x| break :b x; | |
| 160 | if (options.parent) |p| break :b p.single_threaded; | |
| 161 | break :b target_util.defaultSingleThreaded(target); | |
| 162 | }; | |
| 163 | ||
| 164 | const error_tracing = b: { | |
| 165 | if (options.inherited.error_tracing) |x| break :b x; | |
| 166 | if (options.parent) |p| break :b p.error_tracing; | |
| 167 | break :b options.global.root_error_tracing; | |
| 168 | }; | |
| 169 | ||
| 170 | const pic = b: { | |
| 171 | if (target_util.requiresPIC(target, options.global.link_libc)) { | |
| 172 | if (options.inherited.pic == false) | |
| 173 | return error.TargetRequiresPic; | |
| 174 | break :b true; | |
| 175 | } | |
| 176 | if (options.global.pie) { | |
| 177 | if (options.inherited.pic == false) | |
| 178 | return error.PieRequiresPic; | |
| 179 | break :b true; | |
| 180 | } | |
| 181 | if (options.global.link_mode == .dynamic) { | |
| 182 | if (options.inherited.pic == false) | |
| 183 | return error.DynamicLinkingRequiresPic; | |
| 184 | break :b true; | |
| 185 | } | |
| 186 | if (options.inherited.pic) |x| break :b x; | |
| 187 | if (options.parent) |p| break :b p.pic; | |
| 188 | break :b false; | |
| 189 | }; | |
| 190 | ||
| 191 | const red_zone = b: { | |
| 192 | if (!target_util.hasRedZone(target)) { | |
| 193 | if (options.inherited.red_zone == true) | |
| 194 | return error.TargetHasNoRedZone; | |
| 195 | break :b false; | |
| 196 | } | |
| 197 | if (options.inherited.red_zone) |x| break :b x; | |
| 198 | if (options.parent) |p| break :b p.red_zone; | |
| 199 | break :b true; | |
| 200 | }; | |
| 201 | ||
| 202 | const omit_frame_pointer = b: { | |
| 203 | if (options.inherited.omit_frame_pointer) |x| break :b x; | |
| 204 | if (options.parent) |p| break :b p.omit_frame_pointer; | |
| 205 | if (optimize_mode == .ReleaseSmall) { | |
| 206 | // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. | |
| 207 | // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) | |
| 208 | // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). | |
| 209 | break :b !target.cpu.arch.isX86(); | |
| 210 | } | |
| 211 | break :b false; | |
| 212 | }; | |
| 213 | ||
| 214 | const sanitize_thread = b: { | |
| 215 | if (options.inherited.sanitize_thread) |x| break :b x; | |
| 216 | if (options.parent) |p| break :b p.sanitize_thread; | |
| 217 | break :b false; | |
| 218 | }; | |
| 219 | ||
| 220 | const unwind_tables = b: { | |
| 221 | if (options.inherited.unwind_tables) |x| break :b x; | |
| 222 | if (options.parent) |p| break :b p.unwind_tables; | |
| 223 | ||
| 224 | break :b target_util.defaultUnwindTables( | |
| 225 | target, | |
| 226 | options.global.link_libunwind, | |
| 227 | sanitize_thread or options.global.any_sanitize_thread, | |
| 228 | ); | |
| 229 | }; | |
| 230 | ||
| 231 | const fuzz = b: { | |
| 232 | if (options.inherited.fuzz) |x| break :b x; | |
| 233 | if (options.parent) |p| break :b p.fuzz; | |
| 234 | break :b false; | |
| 235 | }; | |
| 236 | ||
| 237 | const code_model = b: { | |
| 238 | if (options.inherited.code_model) |x| break :b x; | |
| 239 | if (options.parent) |p| break :b p.code_model; | |
| 240 | break :b .default; | |
| 241 | }; | |
| 242 | ||
| 243 | const is_safe_mode = switch (optimize_mode) { | |
| 244 | .Debug, .ReleaseSafe => true, | |
| 245 | .ReleaseFast, .ReleaseSmall => false, | |
| 246 | }; | |
| 247 | ||
| 248 | const sanitize_c = b: { | |
| 249 | if (options.inherited.sanitize_c) |x| break :b x; | |
| 250 | if (options.parent) |p| break :b p.sanitize_c; | |
| 251 | break :b is_safe_mode; | |
| 252 | }; | |
| 253 | ||
| 254 | const stack_check = b: { | |
| 255 | if (!target_util.supportsStackProbing(target)) { | |
| 256 | if (options.inherited.stack_check == true) | |
| 257 | return error.StackCheckUnsupportedByTarget; | |
| 258 | break :b false; | |
| 259 | } | |
| 260 | if (options.inherited.stack_check) |x| break :b x; | |
| 261 | if (options.parent) |p| break :b p.stack_check; | |
| 262 | break :b is_safe_mode; | |
| 263 | }; | |
| 264 | ||
| 265 | const stack_protector: u32 = sp: { | |
| 266 | const use_zig_backend = options.global.have_zcu or | |
| 267 | (options.global.any_c_source_files and options.global.c_frontend == .aro); | |
| 268 | if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { | |
| 269 | if (options.inherited.stack_protector) |x| { | |
| 270 | if (x > 0) return error.StackProtectorUnsupportedByTarget; | |
| 271 | } | |
| 272 | break :sp 0; | |
| 273 | } | |
| 274 | ||
| 275 | if (options.global.any_c_source_files and options.global.c_frontend == .clang and | |
| 276 | !target_util.clangSupportsStackProtector(target)) | |
| 277 | { | |
| 278 | if (options.inherited.stack_protector) |x| { | |
| 279 | if (x > 0) return error.StackProtectorUnsupportedByTarget; | |
| 280 | } | |
| 281 | break :sp 0; | |
| 282 | } | |
| 283 | ||
| 284 | // This logic is checking for linking libc because otherwise our start code | |
| 285 | // which is trying to set up TLS (i.e. the fs/gs registers) but the stack | |
| 286 | // protection code depends on fs/gs registers being already set up. | |
| 287 | // If we were able to annotate start code, or perhaps the entire std lib, | |
| 288 | // as being exempt from stack protection checks, we could change this logic | |
| 289 | // to supporting stack protection even when not linking libc. | |
| 290 | // TODO file issue about this | |
| 291 | if (!options.global.link_libc) { | |
| 292 | if (options.inherited.stack_protector) |x| { | |
| 293 | if (x > 0) return error.StackProtectorUnavailableWithoutLibC; | |
| 294 | } | |
| 295 | break :sp 0; | |
| 296 | } | |
| 297 | ||
| 298 | if (options.inherited.stack_protector) |x| break :sp x; | |
| 299 | if (options.parent) |p| break :sp p.stack_protector; | |
| 300 | if (!is_safe_mode) break :sp 0; | |
| 301 | ||
| 302 | break :sp target_util.default_stack_protector_buffer_size; | |
| 303 | }; | |
| 304 | ||
| 305 | const structured_cfg = b: { | |
| 306 | if (options.inherited.structured_cfg) |x| break :b x; | |
| 307 | if (options.parent) |p| break :b p.structured_cfg; | |
| 308 | // We always want a structured control flow in shaders. This option is | |
| 309 | // only relevant for OpenCL kernels. | |
| 310 | break :b switch (target.os.tag) { | |
| 311 | .opencl => false, | |
| 312 | else => true, | |
| 313 | }; | |
| 314 | }; | |
| 315 | ||
| 316 | const no_builtin = b: { | |
| 317 | if (options.inherited.no_builtin) |x| break :b x; | |
| 318 | if (options.parent) |p| break :b p.no_builtin; | |
| 319 | ||
| 320 | break :b target.cpu.arch.isBpf(); | |
| 321 | }; | |
| 322 | ||
| 323 | const llvm_cpu_features: ?[*:0]const u8 = b: { | |
| 324 | if (resolved_target.llvm_cpu_features) |x| break :b x; | |
| 325 | if (!options.global.use_llvm) break :b null; | |
| 326 | ||
| 327 | var buf = std.ArrayList(u8).init(arena); | |
| 328 | var disabled_features = std.ArrayList(u8).init(arena); | |
| 329 | defer disabled_features.deinit(); | |
| 330 | ||
| 331 | // Append disabled features after enabled ones, so that their effects aren't overwritten. | |
| 332 | for (target.cpu.arch.allFeaturesList()) |feature| { | |
| 333 | if (feature.llvm_name) |llvm_name| { | |
| 334 | const is_enabled = target.cpu.features.isEnabled(feature.index); | |
| 335 | ||
| 336 | if (is_enabled) { | |
| 337 | try buf.ensureUnusedCapacity(2 + llvm_name.len); | |
| 338 | buf.appendAssumeCapacity('+'); | |
| 339 | buf.appendSliceAssumeCapacity(llvm_name); | |
| 340 | buf.appendAssumeCapacity(','); | |
| 341 | } else { | |
| 342 | try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); | |
| 343 | disabled_features.appendAssumeCapacity('-'); | |
| 344 | disabled_features.appendSliceAssumeCapacity(llvm_name); | |
| 345 | disabled_features.appendAssumeCapacity(','); | |
| 346 | } | |
| 347 | } | |
| 348 | } | |
| 349 | ||
| 350 | try buf.appendSlice(disabled_features.items); | |
| 351 | if (buf.items.len == 0) break :b ""; | |
| 352 | assert(std.mem.endsWith(u8, buf.items, ",")); | |
| 353 | buf.items[buf.items.len - 1] = 0; | |
| 354 | buf.shrinkAndFree(buf.items.len); | |
| 355 | break :b buf.items[0 .. buf.items.len - 1 :0].ptr; | |
| 356 | }; | |
| 357 | ||
| 358 | const mod = try arena.create(Module); | |
| 359 | mod.* = .{ | |
| 360 | .root = options.paths.root, | |
| 361 | .root_src_path = options.paths.root_src_path, | |
| 362 | .fully_qualified_name = options.fully_qualified_name, | |
| 363 | .resolved_target = .{ | |
| 364 | .result = target, | |
| 365 | .is_native_os = resolved_target.is_native_os, | |
| 366 | .is_native_abi = resolved_target.is_native_abi, | |
| 367 | .llvm_cpu_features = llvm_cpu_features, | |
| 368 | }, | |
| 369 | .optimize_mode = optimize_mode, | |
| 370 | .single_threaded = single_threaded, | |
| 371 | .error_tracing = error_tracing, | |
| 372 | .valgrind = valgrind, | |
| 373 | .pic = pic, | |
| 374 | .strip = strip, | |
| 375 | .omit_frame_pointer = omit_frame_pointer, | |
| 376 | .stack_check = stack_check, | |
| 377 | .stack_protector = stack_protector, | |
| 378 | .code_model = code_model, | |
| 379 | .red_zone = red_zone, | |
| 380 | .sanitize_c = sanitize_c, | |
| 381 | .sanitize_thread = sanitize_thread, | |
| 382 | .fuzz = fuzz, | |
| 383 | .unwind_tables = unwind_tables, | |
| 384 | .cc_argv = options.cc_argv, | |
| 385 | .structured_cfg = structured_cfg, | |
| 386 | .no_builtin = no_builtin, | |
| 387 | .builtin_file = null, | |
| 388 | }; | |
| 389 | ||
| 390 | const opt_builtin_mod = options.builtin_mod orelse b: { | |
| 391 | if (!options.global.have_zcu) break :b null; | |
| 392 | ||
| 393 | const generated_builtin_source = try Builtin.generate(.{ | |
| 394 | .target = target, | |
| 395 | .zig_backend = zig_backend, | |
| 396 | .output_mode = options.global.output_mode, | |
| 397 | .link_mode = options.global.link_mode, | |
| 398 | .unwind_tables = unwind_tables, | |
| 399 | .is_test = options.global.is_test, | |
| 400 | .single_threaded = single_threaded, | |
| 401 | .link_libc = options.global.link_libc, | |
| 402 | .link_libcpp = options.global.link_libcpp, | |
| 403 | .optimize_mode = optimize_mode, | |
| 404 | .error_tracing = error_tracing, | |
| 405 | .valgrind = valgrind, | |
| 406 | .sanitize_thread = sanitize_thread, | |
| 407 | .fuzz = fuzz, | |
| 408 | .pic = pic, | |
| 409 | .pie = options.global.pie, | |
| 410 | .strip = strip, | |
| 411 | .code_model = code_model, | |
| 412 | .omit_frame_pointer = omit_frame_pointer, | |
| 413 | .wasi_exec_model = options.global.wasi_exec_model, | |
| 414 | }, arena); | |
| 415 | ||
| 416 | const new = if (options.builtin_modules) |builtins| new: { | |
| 417 | const gop = try builtins.getOrPut(arena, generated_builtin_source); | |
| 418 | if (gop.found_existing) break :b gop.value_ptr.*; | |
| 419 | errdefer builtins.removeByPtr(gop.key_ptr); | |
| 420 | const new = try arena.create(Module); | |
| 421 | gop.value_ptr.* = new; | |
| 422 | break :new new; | |
| 423 | } else try arena.create(Module); | |
| 424 | errdefer if (options.builtin_modules) |builtins| assert(builtins.remove(generated_builtin_source)); | |
| 425 | ||
| 426 | const new_file = try arena.create(File); | |
| 427 | ||
| 428 | const hex_digest = digest: { | |
| 429 | var hasher: Cache.Hasher = Cache.hasher_init; | |
| 430 | hasher.update(generated_builtin_source); | |
| 431 | ||
| 432 | var bin_digest: Cache.BinDigest = undefined; | |
| 433 | hasher.final(&bin_digest); | |
| 434 | ||
| 435 | var hex_digest: Cache.HexDigest = undefined; | |
| 436 | _ = std.fmt.bufPrint( | |
| 437 | &hex_digest, | |
| 438 | "{s}", | |
| 439 | .{std.fmt.fmtSliceHexLower(&bin_digest)}, | |
| 440 | ) catch unreachable; | |
| 441 | ||
| 442 | break :digest hex_digest; | |
| 443 | }; | |
| 444 | ||
| 445 | const builtin_sub_path = try arena.dupe(u8, "b" ++ std.fs.path.sep_str ++ hex_digest); | |
| 446 | ||
| 447 | new.* = .{ | |
| 448 | .root = .{ | |
| 449 | .root_dir = options.global_cache_directory, | |
| 450 | .sub_path = builtin_sub_path, | |
| 451 | }, | |
| 452 | .root_src_path = "builtin.zig", | |
| 453 | .fully_qualified_name = if (options.parent == null) | |
| 454 | "builtin" | |
| 455 | else | |
| 456 | try std.fmt.allocPrint(arena, "{s}.builtin", .{options.fully_qualified_name}), | |
| 457 | .resolved_target = .{ | |
| 458 | .result = target, | |
| 459 | .is_native_os = resolved_target.is_native_os, | |
| 460 | .is_native_abi = resolved_target.is_native_abi, | |
| 461 | .llvm_cpu_features = llvm_cpu_features, | |
| 462 | }, | |
| 463 | .optimize_mode = optimize_mode, | |
| 464 | .single_threaded = single_threaded, | |
| 465 | .error_tracing = error_tracing, | |
| 466 | .valgrind = valgrind, | |
| 467 | .pic = pic, | |
| 468 | .strip = strip, | |
| 469 | .omit_frame_pointer = omit_frame_pointer, | |
| 470 | .stack_check = stack_check, | |
| 471 | .stack_protector = stack_protector, | |
| 472 | .code_model = code_model, | |
| 473 | .red_zone = red_zone, | |
| 474 | .sanitize_c = sanitize_c, | |
| 475 | .sanitize_thread = sanitize_thread, | |
| 476 | .fuzz = fuzz, | |
| 477 | .unwind_tables = unwind_tables, | |
| 478 | .cc_argv = &.{}, | |
| 479 | .structured_cfg = structured_cfg, | |
| 480 | .no_builtin = no_builtin, | |
| 481 | .builtin_file = new_file, | |
| 482 | }; | |
| 483 | new_file.* = .{ | |
| 484 | .sub_file_path = "builtin.zig", | |
| 485 | .stat = undefined, | |
| 486 | .source = generated_builtin_source, | |
| 487 | .tree = null, | |
| 488 | .zir = null, | |
| 489 | .zoir = null, | |
| 490 | .status = .never_loaded, | |
| 491 | .mod = new, | |
| 492 | }; | |
| 493 | break :b new; | |
| 494 | }; | |
| 495 | ||
| 496 | if (opt_builtin_mod) |builtin_mod| { | |
| 497 | try mod.deps.ensureUnusedCapacity(arena, 1); | |
| 498 | mod.deps.putAssumeCapacityNoClobber("builtin", builtin_mod); | |
| 499 | } | |
| 500 | ||
| 501 | return mod; | |
| 502 | } | |
| 503 | ||
| 504 | /// All fields correspond to `CreateOptions`. | |
| 505 | pub const LimitedOptions = struct { | |
| 506 | root: Cache.Path, | |
| 507 | root_src_path: []const u8, | |
| 508 | fully_qualified_name: []const u8, | |
| 509 | }; | |
| 510 | ||
| 511 | /// This one can only be used if the Module will only be used for AstGen and earlier in | |
| 512 | /// the pipeline. Illegal behavior occurs if a limited module touches Sema. | |
| 513 | pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { | |
| 514 | const mod = try gpa.create(Module); | |
| 515 | mod.* = .{ | |
| 516 | .root = options.root, | |
| 517 | .root_src_path = options.root_src_path, | |
| 518 | .fully_qualified_name = options.fully_qualified_name, | |
| 519 | ||
| 520 | .resolved_target = undefined, | |
| 521 | .optimize_mode = undefined, | |
| 522 | .code_model = undefined, | |
| 523 | .single_threaded = undefined, | |
| 524 | .error_tracing = undefined, | |
| 525 | .valgrind = undefined, | |
| 526 | .pic = undefined, | |
| 527 | .strip = undefined, | |
| 528 | .omit_frame_pointer = undefined, | |
| 529 | .stack_check = undefined, | |
| 530 | .stack_protector = undefined, | |
| 531 | .red_zone = undefined, | |
| 532 | .sanitize_c = undefined, | |
| 533 | .sanitize_thread = undefined, | |
| 534 | .fuzz = undefined, | |
| 535 | .unwind_tables = undefined, | |
| 536 | .cc_argv = undefined, | |
| 537 | .structured_cfg = undefined, | |
| 538 | .no_builtin = undefined, | |
| 539 | .builtin_file = null, | |
| 540 | }; | |
| 541 | return mod; | |
| 542 | } | |
| 543 | ||
| 544 | /// Asserts that the module has a builtin module, which is not true for non-zig | |
| 545 | /// modules such as ones only used for `@embedFile`, or the root module when | |
| 546 | /// there is no Zig Compilation Unit. | |
| 547 | pub fn getBuiltinDependency(m: Module) *Module { | |
| 548 | const result = m.deps.values()[0]; | |
| 549 | assert(result.isBuiltin()); | |
| 550 | return result; | |
| 551 | } | |
| 552 | ||
| 553 | const Module = @This(); | |
| 554 | const Package = @import("../Package.zig"); | |
| 555 | const std = @import("std"); | |
| 556 | const Allocator = std.mem.Allocator; | |
| 557 | const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; | |
| 558 | const target_util = @import("../target.zig"); | |
| 559 | const Cache = std.Build.Cache; | |
| 560 | const Builtin = @import("../Builtin.zig"); | |
| 561 | const assert = std.debug.assert; | |
| 562 | const Compilation = @import("../Compilation.zig"); | |
| 563 | const File = @import("../Zcu.zig").File; |
src/Package.zig deleted-200
| ... | ... | @@ -1,200 +0,0 @@ |
| 1 | const std = @import("std"); | |
| 2 | const assert = std.debug.assert; | |
| 3 | ||
| 4 | pub const Module = @import("Package/Module.zig"); | |
| 5 | pub const Fetch = @import("Package/Fetch.zig"); | |
| 6 | pub const build_zig_basename = "build.zig"; | |
| 7 | pub const Manifest = @import("Package/Manifest.zig"); | |
| 8 | ||
| 9 | pub const multihash_len = 1 + 1 + Hash.Algo.digest_length; | |
| 10 | pub const multihash_hex_digest_len = 2 * multihash_len; | |
| 11 | pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; | |
| 12 | ||
| 13 | pub const Fingerprint = packed struct(u64) { | |
| 14 | id: u32, | |
| 15 | checksum: u32, | |
| 16 | ||
| 17 | pub fn generate(name: []const u8) Fingerprint { | |
| 18 | return .{ | |
| 19 | .id = std.crypto.random.intRangeLessThan(u32, 1, 0xffffffff), | |
| 20 | .checksum = std.hash.Crc32.hash(name), | |
| 21 | }; | |
| 22 | } | |
| 23 | ||
| 24 | pub fn validate(n: Fingerprint, name: []const u8) bool { | |
| 25 | switch (n.id) { | |
| 26 | 0x00000000, 0xffffffff => return false, | |
| 27 | else => return std.hash.Crc32.hash(name) == n.checksum, | |
| 28 | } | |
| 29 | } | |
| 30 | ||
| 31 | pub fn int(n: Fingerprint) u64 { | |
| 32 | return @bitCast(n); | |
| 33 | } | |
| 34 | }; | |
| 35 | ||
| 36 | /// A user-readable, file system safe hash that identifies an exact package | |
| 37 | /// snapshot, including file contents. | |
| 38 | /// | |
| 39 | /// The hash is not only to prevent collisions but must resist attacks where | |
| 40 | /// the adversary fully controls the contents being hashed. Thus, it contains | |
| 41 | /// a full SHA-256 digest. | |
| 42 | /// | |
| 43 | /// This data structure can be used to store the legacy hash format too. Legacy | |
| 44 | /// hash format is scheduled to be removed after 0.14.0 is tagged. | |
| 45 | /// | |
| 46 | /// There's also a third way this structure is used. When using path rather than | |
| 47 | /// hash, a unique hash is still needed, so one is computed based on the path. | |
| 48 | pub const Hash = struct { | |
| 49 | /// Maximum size of a package hash. Unused bytes at the end are | |
| 50 | /// filled with zeroes. | |
| 51 | bytes: [max_len]u8, | |
| 52 | ||
| 53 | pub const Algo = std.crypto.hash.sha2.Sha256; | |
| 54 | pub const Digest = [Algo.digest_length]u8; | |
| 55 | ||
| 56 | /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" | |
| 57 | pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; | |
| 58 | ||
| 59 | pub fn fromSlice(s: []const u8) Hash { | |
| 60 | assert(s.len <= max_len); | |
| 61 | var result: Hash = undefined; | |
| 62 | @memcpy(result.bytes[0..s.len], s); | |
| 63 | @memset(result.bytes[s.len..], 0); | |
| 64 | return result; | |
| 65 | } | |
| 66 | ||
| 67 | pub fn toSlice(ph: *const Hash) []const u8 { | |
| 68 | var end: usize = ph.bytes.len; | |
| 69 | while (true) { | |
| 70 | end -= 1; | |
| 71 | if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; | |
| 72 | } | |
| 73 | } | |
| 74 | ||
| 75 | pub fn eql(a: *const Hash, b: *const Hash) bool { | |
| 76 | return std.mem.eql(u8, &a.bytes, &b.bytes); | |
| 77 | } | |
| 78 | ||
| 79 | /// Distinguishes whether the legacy multihash format is being stored here. | |
| 80 | pub fn isOld(h: *const Hash) bool { | |
| 81 | if (h.bytes.len < 2) return false; | |
| 82 | const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false; | |
| 83 | if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false; | |
| 84 | if (h.toSlice().len != multihash_hex_digest_len) return false; | |
| 85 | return std.mem.indexOfScalar(u8, &h.bytes, '-') == null; | |
| 86 | } | |
| 87 | ||
| 88 | test isOld { | |
| 89 | const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7"); | |
| 90 | try std.testing.expect(h.isOld()); | |
| 91 | } | |
| 92 | ||
| 93 | /// Produces "$name-$semver-$hashplus". | |
| 94 | /// * name is the name field from build.zig.zon, asserted to be at most 32 | |
| 95 | /// bytes and assumed be a valid zig identifier | |
| 96 | /// * semver is the version field from build.zig.zon, asserted to be at | |
| 97 | /// most 32 bytes | |
| 98 | /// * hashplus is the following 33-byte array, base64 encoded using -_ to make | |
| 99 | /// it filesystem safe: | |
| 100 | /// - (4 bytes) LE u32 Package ID | |
| 101 | /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated | |
| 102 | /// - (25 bytes) truncated SHA-256 digest of hashed files of the package | |
| 103 | pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { | |
| 104 | assert(name.len <= 32); | |
| 105 | assert(ver.len <= 32); | |
| 106 | var result: Hash = undefined; | |
| 107 | var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes); | |
| 108 | buf.appendSliceAssumeCapacity(name); | |
| 109 | buf.appendAssumeCapacity('-'); | |
| 110 | buf.appendSliceAssumeCapacity(ver); | |
| 111 | buf.appendAssumeCapacity('-'); | |
| 112 | var hashplus: [33]u8 = undefined; | |
| 113 | std.mem.writeInt(u32, hashplus[0..4], id, .little); | |
| 114 | std.mem.writeInt(u32, hashplus[4..8], size, .little); | |
| 115 | hashplus[8..].* = digest[0..25].*; | |
| 116 | _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); | |
| 117 | @memset(buf.unusedCapacitySlice(), 0); | |
| 118 | return result; | |
| 119 | } | |
| 120 | ||
| 121 | /// Produces a unique hash based on the path provided. The result should | |
| 122 | /// not be user-visible. | |
| 123 | pub fn initPath(sub_path: []const u8, is_global: bool) Hash { | |
| 124 | var result: Hash = .{ .bytes = @splat(0) }; | |
| 125 | var i: usize = 0; | |
| 126 | if (is_global) { | |
| 127 | result.bytes[0] = '/'; | |
| 128 | i += 1; | |
| 129 | } | |
| 130 | if (i + sub_path.len <= result.bytes.len) { | |
| 131 | @memcpy(result.bytes[i..][0..sub_path.len], sub_path); | |
| 132 | return result; | |
| 133 | } | |
| 134 | var bin_digest: [Algo.digest_length]u8 = undefined; | |
| 135 | Algo.hash(sub_path, &bin_digest, .{}); | |
| 136 | _ = std.fmt.bufPrint(result.bytes[i..], "{}", .{std.fmt.fmtSliceHexLower(&bin_digest)}) catch unreachable; | |
| 137 | return result; | |
| 138 | } | |
| 139 | }; | |
| 140 | ||
| 141 | pub const MultihashFunction = enum(u16) { | |
| 142 | identity = 0x00, | |
| 143 | sha1 = 0x11, | |
| 144 | @"sha2-256" = 0x12, | |
| 145 | @"sha2-512" = 0x13, | |
| 146 | @"sha3-512" = 0x14, | |
| 147 | @"sha3-384" = 0x15, | |
| 148 | @"sha3-256" = 0x16, | |
| 149 | @"sha3-224" = 0x17, | |
| 150 | @"sha2-384" = 0x20, | |
| 151 | @"sha2-256-trunc254-padded" = 0x1012, | |
| 152 | @"sha2-224" = 0x1013, | |
| 153 | @"sha2-512-224" = 0x1014, | |
| 154 | @"sha2-512-256" = 0x1015, | |
| 155 | @"blake2b-256" = 0xb220, | |
| 156 | _, | |
| 157 | }; | |
| 158 | ||
| 159 | pub const multihash_function: MultihashFunction = switch (Hash.Algo) { | |
| 160 | std.crypto.hash.sha2.Sha256 => .@"sha2-256", | |
| 161 | else => unreachable, | |
| 162 | }; | |
| 163 | ||
| 164 | pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest { | |
| 165 | const hex_charset = std.fmt.hex_charset; | |
| 166 | ||
| 167 | var result: MultiHashHexDigest = undefined; | |
| 168 | ||
| 169 | result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; | |
| 170 | result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; | |
| 171 | ||
| 172 | result[2] = hex_charset[Hash.Algo.digest_length >> 4]; | |
| 173 | result[3] = hex_charset[Hash.Algo.digest_length & 15]; | |
| 174 | ||
| 175 | for (digest, 0..) |byte, i| { | |
| 176 | result[4 + i * 2] = hex_charset[byte >> 4]; | |
| 177 | result[5 + i * 2] = hex_charset[byte & 15]; | |
| 178 | } | |
| 179 | return result; | |
| 180 | } | |
| 181 | ||
| 182 | comptime { | |
| 183 | // We avoid unnecessary uleb128 code in hexDigest by asserting here the | |
| 184 | // values are small enough to be contained in the one-byte encoding. | |
| 185 | assert(@intFromEnum(multihash_function) < 127); | |
| 186 | assert(Hash.Algo.digest_length < 127); | |
| 187 | } | |
| 188 | ||
| 189 | test Hash { | |
| 190 | const example_digest: Hash.Digest = .{ | |
| 191 | 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, | |
| 192 | 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, | |
| 193 | }; | |
| 194 | const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); | |
| 195 | try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); | |
| 196 | } | |
| 197 | ||
| 198 | test { | |
| 199 | _ = Fetch; | |
| 200 | } |
src/Package/Fetch.zig deleted-2421
| ... | ... | @@ -1,2421 +0,0 @@ |
| 1 | //! Represents one independent job whose responsibility is to: | |
| 2 | //! | |
| 3 | //! 1. Check the global zig package cache to see if the hash already exists. | |
| 4 | //! If so, load, parse, and validate the build.zig.zon file therein, and | |
| 5 | //! goto step 8. Likewise if the location is a relative path, treat this | |
| 6 | //! the same as a cache hit. Otherwise, proceed. | |
| 7 | //! 2. Fetch and unpack a URL into a temporary directory. | |
| 8 | //! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed | |
| 9 | //! for the file to be missing, in which case this fetched package is considered | |
| 10 | //! to be a "naked" package. | |
| 11 | //! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by | |
| 12 | //! deleting excluded files. If any files had errors for files that were | |
| 13 | //! ultimately excluded, those errors should be ignored, such as failure to | |
| 14 | //! create symlinks that weren't supposed to be included anyway. | |
| 15 | //! 5. Compute the package hash based on the remaining files in the temporary | |
| 16 | //! directory. | |
| 17 | //! 6. Rename the temporary directory into the global zig package cache | |
| 18 | //! directory. If the hash already exists, delete the temporary directory and | |
| 19 | //! leave the zig package cache directory untouched as it may be in use by the | |
| 20 | //! system. This is done even if the hash is invalid, in case the package with | |
| 21 | //! the different hash is used in the future. | |
| 22 | //! 7. Validate the computed hash against the expected hash. If invalid, | |
| 23 | //! this job is done. | |
| 24 | //! 8. Spawn a new fetch job for each dependency in the manifest file. Use | |
| 25 | //! a mutex and a hash map so that redundant jobs do not get queued up. | |
| 26 | //! | |
| 27 | //! All of this must be done with only referring to the state inside this struct | |
| 28 | //! because this work will be done in a dedicated thread. | |
| 29 | ||
| 30 | arena: std.heap.ArenaAllocator, | |
| 31 | location: Location, | |
| 32 | location_tok: std.zig.Ast.TokenIndex, | |
| 33 | hash_tok: std.zig.Ast.OptionalTokenIndex, | |
| 34 | name_tok: std.zig.Ast.TokenIndex, | |
| 35 | lazy_status: LazyStatus, | |
| 36 | parent_package_root: Cache.Path, | |
| 37 | parent_manifest_ast: ?*const std.zig.Ast, | |
| 38 | prog_node: std.Progress.Node, | |
| 39 | job_queue: *JobQueue, | |
| 40 | /// If true, don't add an error for a missing hash. This flag is not passed | |
| 41 | /// down to recursive dependencies. It's intended to be used only be the CLI. | |
| 42 | omit_missing_hash_error: bool, | |
| 43 | /// If true, don't fail when a manifest file is missing the `paths` field, | |
| 44 | /// which specifies inclusion rules. This is intended to be true for the first | |
| 45 | /// fetch task and false for the recursive dependencies. | |
| 46 | allow_missing_paths_field: bool, | |
| 47 | allow_missing_fingerprint: bool, | |
| 48 | allow_name_string: bool, | |
| 49 | /// If true and URL points to a Git repository, will use the latest commit. | |
| 50 | use_latest_commit: bool, | |
| 51 | ||
| 52 | // Above this are fields provided as inputs to `run`. | |
| 53 | // Below this are fields populated by `run`. | |
| 54 | ||
| 55 | /// This will either be relative to `global_cache`, or to the build root of | |
| 56 | /// the root package. | |
| 57 | package_root: Cache.Path, | |
| 58 | error_bundle: ErrorBundle.Wip, | |
| 59 | manifest: ?Manifest, | |
| 60 | manifest_ast: std.zig.Ast, | |
| 61 | computed_hash: ComputedHash, | |
| 62 | /// Fetch logic notices whether a package has a build.zig file and sets this flag. | |
| 63 | has_build_zig: bool, | |
| 64 | /// Indicates whether the task aborted due to an out-of-memory condition. | |
| 65 | oom_flag: bool, | |
| 66 | /// If `use_latest_commit` was true, this will be set to the commit that was used. | |
| 67 | /// If the resource pointed to by the location is not a Git-repository, this | |
| 68 | /// will be left unchanged. | |
| 69 | latest_commit: ?git.Oid, | |
| 70 | ||
| 71 | // This field is used by the CLI only, untouched by this file. | |
| 72 | ||
| 73 | /// The module for this `Fetch` tasks's package, which exposes `build.zig` as | |
| 74 | /// the root source file. | |
| 75 | module: ?*Package.Module, | |
| 76 | ||
| 77 | pub const LazyStatus = enum { | |
| 78 | /// Not lazy. | |
| 79 | eager, | |
| 80 | /// Lazy, found. | |
| 81 | available, | |
| 82 | /// Lazy, not found. | |
| 83 | unavailable, | |
| 84 | }; | |
| 85 | ||
| 86 | /// Contains shared state among all `Fetch` tasks. | |
| 87 | pub const JobQueue = struct { | |
| 88 | mutex: std.Thread.Mutex = .{}, | |
| 89 | /// It's an array hash map so that it can be sorted before rendering the | |
| 90 | /// dependencies.zig source file. | |
| 91 | /// Protected by `mutex`. | |
| 92 | table: Table = .{}, | |
| 93 | /// `table` may be missing some tasks such as ones that failed, so this | |
| 94 | /// field contains references to all of them. | |
| 95 | /// Protected by `mutex`. | |
| 96 | all_fetches: std.ArrayListUnmanaged(*Fetch) = .empty, | |
| 97 | ||
| 98 | http_client: *std.http.Client, | |
| 99 | thread_pool: *ThreadPool, | |
| 100 | wait_group: WaitGroup = .{}, | |
| 101 | global_cache: Cache.Directory, | |
| 102 | /// If true then, no fetching occurs, and: | |
| 103 | /// * The `global_cache` directory is assumed to be the direct parent | |
| 104 | /// directory of on-disk packages rather than having the "p/" directory | |
| 105 | /// prefix inside of it. | |
| 106 | /// * An error occurs if any non-lazy packages are not already present in | |
| 107 | /// the package cache directory. | |
| 108 | /// * Missing hash field causes an error, and no fetching occurs so it does | |
| 109 | /// not print the correct hash like usual. | |
| 110 | read_only: bool, | |
| 111 | recursive: bool, | |
| 112 | /// Dumps hash information to stdout which can be used to troubleshoot why | |
| 113 | /// two hashes of the same package do not match. | |
| 114 | /// If this is true, `recursive` must be false. | |
| 115 | debug_hash: bool, | |
| 116 | work_around_btrfs_bug: bool, | |
| 117 | /// Set of hashes that will be additionally fetched even if they are marked | |
| 118 | /// as lazy. | |
| 119 | unlazy_set: UnlazySet = .{}, | |
| 120 | ||
| 121 | pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch); | |
| 122 | pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void); | |
| 123 | ||
| 124 | pub fn deinit(jq: *JobQueue) void { | |
| 125 | if (jq.all_fetches.items.len == 0) return; | |
| 126 | const gpa = jq.all_fetches.items[0].arena.child_allocator; | |
| 127 | jq.table.deinit(gpa); | |
| 128 | // These must be deinitialized in reverse order because subsequent | |
| 129 | // `Fetch` instances are allocated in prior ones' arenas. | |
| 130 | // Sorry, I know it's a bit weird, but it slightly simplifies the | |
| 131 | // critical section. | |
| 132 | while (jq.all_fetches.pop()) |f| f.deinit(); | |
| 133 | jq.all_fetches.deinit(gpa); | |
| 134 | jq.* = undefined; | |
| 135 | } | |
| 136 | ||
| 137 | /// Dumps all subsequent error bundles into the first one. | |
| 138 | pub fn consolidateErrors(jq: *JobQueue) !void { | |
| 139 | const root = &jq.all_fetches.items[0].error_bundle; | |
| 140 | const gpa = root.gpa; | |
| 141 | for (jq.all_fetches.items[1..]) |fetch| { | |
| 142 | if (fetch.error_bundle.root_list.items.len > 0) { | |
| 143 | var bundle = try fetch.error_bundle.toOwnedBundle(""); | |
| 144 | defer bundle.deinit(gpa); | |
| 145 | try root.addBundleAsRoots(bundle); | |
| 146 | } | |
| 147 | } | |
| 148 | } | |
| 149 | ||
| 150 | /// Creates the dependencies.zig source code for the build runner to obtain | |
| 151 | /// via `@import("@dependencies")`. | |
| 152 | pub fn createDependenciesSource(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { | |
| 153 | const keys = jq.table.keys(); | |
| 154 | ||
| 155 | assert(keys.len != 0); // caller should have added the first one | |
| 156 | if (keys.len == 1) { | |
| 157 | // This is the first one. It must have no dependencies. | |
| 158 | return createEmptyDependenciesSource(buf); | |
| 159 | } | |
| 160 | ||
| 161 | try buf.appendSlice("pub const packages = struct {\n"); | |
| 162 | ||
| 163 | // Ensure the generated .zig file is deterministic. | |
| 164 | jq.table.sortUnstable(@as(struct { | |
| 165 | keys: []const Package.Hash, | |
| 166 | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { | |
| 167 | return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); | |
| 168 | } | |
| 169 | }, .{ .keys = keys })); | |
| 170 | ||
| 171 | for (keys, jq.table.values()) |*hash, fetch| { | |
| 172 | if (fetch == jq.all_fetches.items[0]) { | |
| 173 | // The first one is a dummy package for the current project. | |
| 174 | continue; | |
| 175 | } | |
| 176 | ||
| 177 | const hash_slice = hash.toSlice(); | |
| 178 | ||
| 179 | try buf.writer().print( | |
| 180 | \\ pub const {} = struct {{ | |
| 181 | \\ | |
| 182 | , .{std.zig.fmtId(hash_slice)}); | |
| 183 | ||
| 184 | lazy: { | |
| 185 | switch (fetch.lazy_status) { | |
| 186 | .eager => break :lazy, | |
| 187 | .available => { | |
| 188 | try buf.appendSlice( | |
| 189 | \\ pub const available = true; | |
| 190 | \\ | |
| 191 | ); | |
| 192 | break :lazy; | |
| 193 | }, | |
| 194 | .unavailable => { | |
| 195 | try buf.appendSlice( | |
| 196 | \\ pub const available = false; | |
| 197 | \\ }; | |
| 198 | \\ | |
| 199 | ); | |
| 200 | continue; | |
| 201 | }, | |
| 202 | } | |
| 203 | } | |
| 204 | ||
| 205 | try buf.writer().print( | |
| 206 | \\ pub const build_root = "{q}"; | |
| 207 | \\ | |
| 208 | , .{fetch.package_root}); | |
| 209 | ||
| 210 | if (fetch.has_build_zig) { | |
| 211 | try buf.writer().print( | |
| 212 | \\ pub const build_zig = @import("{}"); | |
| 213 | \\ | |
| 214 | , .{std.zig.fmtEscapes(hash_slice)}); | |
| 215 | } | |
| 216 | ||
| 217 | if (fetch.manifest) |*manifest| { | |
| 218 | try buf.appendSlice( | |
| 219 | \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ | |
| 220 | \\ | |
| 221 | ); | |
| 222 | for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { | |
| 223 | const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; | |
| 224 | try buf.writer().print( | |
| 225 | " .{{ \"{}\", \"{}\" }},\n", | |
| 226 | .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, | |
| 227 | ); | |
| 228 | } | |
| 229 | ||
| 230 | try buf.appendSlice( | |
| 231 | \\ }; | |
| 232 | \\ }; | |
| 233 | \\ | |
| 234 | ); | |
| 235 | } else { | |
| 236 | try buf.appendSlice( | |
| 237 | \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; | |
| 238 | \\ }; | |
| 239 | \\ | |
| 240 | ); | |
| 241 | } | |
| 242 | } | |
| 243 | ||
| 244 | try buf.appendSlice( | |
| 245 | \\}; | |
| 246 | \\ | |
| 247 | \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ | |
| 248 | \\ | |
| 249 | ); | |
| 250 | ||
| 251 | const root_fetch = jq.all_fetches.items[0]; | |
| 252 | const root_manifest = &root_fetch.manifest.?; | |
| 253 | ||
| 254 | for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { | |
| 255 | const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; | |
| 256 | try buf.writer().print( | |
| 257 | " .{{ \"{}\", \"{}\" }},\n", | |
| 258 | .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, | |
| 259 | ); | |
| 260 | } | |
| 261 | try buf.appendSlice("};\n"); | |
| 262 | } | |
| 263 | ||
| 264 | pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void { | |
| 265 | try buf.appendSlice( | |
| 266 | \\pub const packages = struct {}; | |
| 267 | \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; | |
| 268 | \\ | |
| 269 | ); | |
| 270 | } | |
| 271 | }; | |
| 272 | ||
| 273 | pub const Location = union(enum) { | |
| 274 | remote: Remote, | |
| 275 | /// A directory found inside the parent package. | |
| 276 | relative_path: Cache.Path, | |
| 277 | /// Recursive Fetch tasks will never use this Location, but it may be | |
| 278 | /// passed in by the CLI. Indicates the file contents here should be copied | |
| 279 | /// into the global package cache. It may be a file relative to the cwd or | |
| 280 | /// absolute, in which case it should be treated exactly like a `file://` | |
| 281 | /// URL, or a directory, in which case it should be treated as an | |
| 282 | /// already-unpacked directory (but still needs to be copied into the | |
| 283 | /// global package cache and have inclusion rules applied). | |
| 284 | path_or_url: []const u8, | |
| 285 | ||
| 286 | pub const Remote = struct { | |
| 287 | url: []const u8, | |
| 288 | /// If this is null it means the user omitted the hash field from a dependency. | |
| 289 | /// It will be an error but the logic should still fetch and print the discovered hash. | |
| 290 | hash: ?Package.Hash, | |
| 291 | }; | |
| 292 | }; | |
| 293 | ||
| 294 | pub const RunError = error{ | |
| 295 | OutOfMemory, | |
| 296 | /// This error code is intended to be handled by inspecting the | |
| 297 | /// `error_bundle` field. | |
| 298 | FetchFailed, | |
| 299 | }; | |
| 300 | ||
| 301 | pub fn run(f: *Fetch) RunError!void { | |
| 302 | const eb = &f.error_bundle; | |
| 303 | const arena = f.arena.allocator(); | |
| 304 | const gpa = f.arena.child_allocator; | |
| 305 | const cache_root = f.job_queue.global_cache; | |
| 306 | ||
| 307 | try eb.init(gpa); | |
| 308 | ||
| 309 | // Check the global zig package cache to see if the hash already exists. If | |
| 310 | // so, load, parse, and validate the build.zig.zon file therein, and skip | |
| 311 | // ahead to queuing up jobs for dependencies. Likewise if the location is a | |
| 312 | // relative path, treat this the same as a cache hit. Otherwise, proceed. | |
| 313 | ||
| 314 | const remote = switch (f.location) { | |
| 315 | .relative_path => |pkg_root| { | |
| 316 | if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( | |
| 317 | f.location_tok, | |
| 318 | try eb.addString("expected path relative to build root; found absolute path"), | |
| 319 | ); | |
| 320 | if (f.hash_tok.unwrap()) |hash_tok| return f.fail( | |
| 321 | hash_tok, | |
| 322 | try eb.addString("path-based dependencies are not hashed"), | |
| 323 | ); | |
| 324 | // Packages fetched by URL may not use relative paths to escape outside the | |
| 325 | // fetched package directory from within the package cache. | |
| 326 | if (pkg_root.root_dir.eql(cache_root)) { | |
| 327 | // `parent_package_root.sub_path` contains a path like this: | |
| 328 | // "p/$hash", or | |
| 329 | // "p/$hash/foo", with possibly more directories after "foo". | |
| 330 | // We want to fail unless the resolved relative path has a | |
| 331 | // prefix of "p/$hash/". | |
| 332 | const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; | |
| 333 | const parent_sub_path = f.parent_package_root.sub_path; | |
| 334 | const end = find_end: { | |
| 335 | if (parent_sub_path.len > prefix_len) { | |
| 336 | // Use `isSep` instead of `indexOfScalarPos` to account for | |
| 337 | // Windows accepting both `\` and `/` as path separators. | |
| 338 | for (parent_sub_path[prefix_len..], prefix_len..) |c, i| { | |
| 339 | if (std.fs.path.isSep(c)) break :find_end i; | |
| 340 | } | |
| 341 | } | |
| 342 | break :find_end parent_sub_path.len; | |
| 343 | }; | |
| 344 | const expected_prefix = parent_sub_path[0..end]; | |
| 345 | if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) { | |
| 346 | return f.fail( | |
| 347 | f.location_tok, | |
| 348 | try eb.printString("dependency path outside project: '{}'", .{pkg_root}), | |
| 349 | ); | |
| 350 | } | |
| 351 | } | |
| 352 | f.package_root = pkg_root; | |
| 353 | try loadManifest(f, pkg_root); | |
| 354 | if (!f.has_build_zig) try checkBuildFileExistence(f); | |
| 355 | if (!f.job_queue.recursive) return; | |
| 356 | return queueJobsForDeps(f); | |
| 357 | }, | |
| 358 | .remote => |remote| remote, | |
| 359 | .path_or_url => |path_or_url| { | |
| 360 | if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| { | |
| 361 | var resource: Resource = .{ .dir = dir }; | |
| 362 | return f.runResource(path_or_url, &resource, null); | |
| 363 | } else |dir_err| { | |
| 364 | const file_err = if (dir_err == error.NotDir) e: { | |
| 365 | if (fs.cwd().openFile(path_or_url, .{})) |file| { | |
| 366 | var resource: Resource = .{ .file = file }; | |
| 367 | return f.runResource(path_or_url, &resource, null); | |
| 368 | } else |err| break :e err; | |
| 369 | } else dir_err; | |
| 370 | ||
| 371 | const uri = std.Uri.parse(path_or_url) catch |uri_err| { | |
| 372 | return f.fail(0, try eb.printString( | |
| 373 | "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", | |
| 374 | .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, | |
| 375 | )); | |
| 376 | }; | |
| 377 | var server_header_buffer: [header_buffer_size]u8 = undefined; | |
| 378 | var resource = try f.initResource(uri, &server_header_buffer); | |
| 379 | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); | |
| 380 | } | |
| 381 | }, | |
| 382 | }; | |
| 383 | ||
| 384 | if (remote.hash) |expected_hash| { | |
| 385 | var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined; | |
| 386 | prefixed_pkg_sub_path_buffer[0] = 'p'; | |
| 387 | prefixed_pkg_sub_path_buffer[1] = fs.path.sep; | |
| 388 | const hash_slice = expected_hash.toSlice(); | |
| 389 | @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); | |
| 390 | const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; | |
| 391 | const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; | |
| 392 | const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; | |
| 393 | if (cache_root.handle.access(pkg_sub_path, .{})) |_| { | |
| 394 | assert(f.lazy_status != .unavailable); | |
| 395 | f.package_root = .{ | |
| 396 | .root_dir = cache_root, | |
| 397 | .sub_path = try arena.dupe(u8, pkg_sub_path), | |
| 398 | }; | |
| 399 | try loadManifest(f, f.package_root); | |
| 400 | try checkBuildFileExistence(f); | |
| 401 | if (!f.job_queue.recursive) return; | |
| 402 | return queueJobsForDeps(f); | |
| 403 | } else |err| switch (err) { | |
| 404 | error.FileNotFound => { | |
| 405 | switch (f.lazy_status) { | |
| 406 | .eager => {}, | |
| 407 | .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { | |
| 408 | f.lazy_status = .unavailable; | |
| 409 | return; | |
| 410 | }, | |
| 411 | .unavailable => unreachable, | |
| 412 | } | |
| 413 | if (f.job_queue.read_only) return f.fail( | |
| 414 | f.name_tok, | |
| 415 | try eb.printString("package not found at '{}{s}'", .{ | |
| 416 | cache_root, pkg_sub_path, | |
| 417 | }), | |
| 418 | ); | |
| 419 | }, | |
| 420 | else => |e| { | |
| 421 | try eb.addRootErrorMessage(.{ | |
| 422 | .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ | |
| 423 | cache_root, pkg_sub_path, @errorName(e), | |
| 424 | }), | |
| 425 | }); | |
| 426 | return error.FetchFailed; | |
| 427 | }, | |
| 428 | } | |
| 429 | } else if (f.job_queue.read_only) { | |
| 430 | try eb.addRootErrorMessage(.{ | |
| 431 | .msg = try eb.addString("dependency is missing hash field"), | |
| 432 | .src_loc = try f.srcLoc(f.location_tok), | |
| 433 | }); | |
| 434 | return error.FetchFailed; | |
| 435 | } | |
| 436 | ||
| 437 | // Fetch and unpack the remote into a temporary directory. | |
| 438 | ||
| 439 | const uri = std.Uri.parse(remote.url) catch |err| return f.fail( | |
| 440 | f.location_tok, | |
| 441 | try eb.printString("invalid URI: {s}", .{@errorName(err)}), | |
| 442 | ); | |
| 443 | var server_header_buffer: [header_buffer_size]u8 = undefined; | |
| 444 | var resource = try f.initResource(uri, &server_header_buffer); | |
| 445 | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); | |
| 446 | } | |
| 447 | ||
| 448 | pub fn deinit(f: *Fetch) void { | |
| 449 | f.error_bundle.deinit(); | |
| 450 | f.arena.deinit(); | |
| 451 | } | |
| 452 | ||
| 453 | /// Consumes `resource`, even if an error is returned. | |
| 454 | fn runResource( | |
| 455 | f: *Fetch, | |
| 456 | uri_path: []const u8, | |
| 457 | resource: *Resource, | |
| 458 | remote_hash: ?Package.Hash, | |
| 459 | ) RunError!void { | |
| 460 | defer resource.deinit(); | |
| 461 | const arena = f.arena.allocator(); | |
| 462 | const eb = &f.error_bundle; | |
| 463 | const s = fs.path.sep_str; | |
| 464 | const cache_root = f.job_queue.global_cache; | |
| 465 | const rand_int = std.crypto.random.int(u64); | |
| 466 | const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); | |
| 467 | ||
| 468 | const package_sub_path = blk: { | |
| 469 | const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); | |
| 470 | var tmp_directory: Cache.Directory = .{ | |
| 471 | .path = tmp_directory_path, | |
| 472 | .handle = handle: { | |
| 473 | const dir = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ | |
| 474 | .iterate = true, | |
| 475 | }) catch |err| { | |
| 476 | try eb.addRootErrorMessage(.{ | |
| 477 | .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ | |
| 478 | tmp_directory_path, @errorName(err), | |
| 479 | }), | |
| 480 | }); | |
| 481 | return error.FetchFailed; | |
| 482 | }; | |
| 483 | break :handle dir; | |
| 484 | }, | |
| 485 | }; | |
| 486 | defer tmp_directory.handle.close(); | |
| 487 | ||
| 488 | // Fetch and unpack a resource into a temporary directory. | |
| 489 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); | |
| 490 | ||
| 491 | var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; | |
| 492 | ||
| 493 | // Apply btrfs workaround if needed. Reopen tmp_directory. | |
| 494 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { | |
| 495 | // https://github.com/ziglang/zig/issues/17095 | |
| 496 | pkg_path.root_dir.handle.close(); | |
| 497 | pkg_path.root_dir.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{ | |
| 498 | .iterate = true, | |
| 499 | }) catch @panic("btrfs workaround failed"); | |
| 500 | } | |
| 501 | ||
| 502 | // Load, parse, and validate the unpacked build.zig.zon file. It is allowed | |
| 503 | // for the file to be missing, in which case this fetched package is | |
| 504 | // considered to be a "naked" package. | |
| 505 | try loadManifest(f, pkg_path); | |
| 506 | ||
| 507 | const filter: Filter = .{ | |
| 508 | .include_paths = if (f.manifest) |m| m.paths else .{}, | |
| 509 | }; | |
| 510 | ||
| 511 | // Ignore errors that were excluded by manifest, such as failure to | |
| 512 | // create symlinks that weren't supposed to be included anyway. | |
| 513 | try unpack_result.validate(f, filter); | |
| 514 | ||
| 515 | // Apply the manifest's inclusion rules to the temporary directory by | |
| 516 | // deleting excluded files. | |
| 517 | // Empty directories have already been omitted by `unpackResource`. | |
| 518 | // Compute the package hash based on the remaining files in the temporary | |
| 519 | // directory. | |
| 520 | f.computed_hash = try computeHash(f, pkg_path, filter); | |
| 521 | ||
| 522 | break :blk if (unpack_result.root_dir.len > 0) | |
| 523 | try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) | |
| 524 | else | |
| 525 | tmp_dir_sub_path; | |
| 526 | }; | |
| 527 | ||
| 528 | const computed_package_hash = computedPackageHash(f); | |
| 529 | ||
| 530 | // Rename the temporary directory into the global zig package cache | |
| 531 | // directory. If the hash already exists, delete the temporary directory | |
| 532 | // and leave the zig package cache directory untouched as it may be in use | |
| 533 | // by the system. This is done even if the hash is invalid, in case the | |
| 534 | // package with the different hash is used in the future. | |
| 535 | ||
| 536 | f.package_root = .{ | |
| 537 | .root_dir = cache_root, | |
| 538 | .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), | |
| 539 | }; | |
| 540 | renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { | |
| 541 | const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); | |
| 542 | const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); | |
| 543 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | |
| 544 | "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", | |
| 545 | .{ src, dest, @errorName(err) }, | |
| 546 | ) }); | |
| 547 | return error.FetchFailed; | |
| 548 | }; | |
| 549 | // Remove temporary directory root if not already renamed to global cache. | |
| 550 | if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { | |
| 551 | cache_root.handle.deleteDir(tmp_dir_sub_path) catch {}; | |
| 552 | } | |
| 553 | ||
| 554 | // Validate the computed hash against the expected hash. If invalid, this | |
| 555 | // job is done. | |
| 556 | ||
| 557 | if (remote_hash) |declared_hash| { | |
| 558 | const hash_tok = f.hash_tok.unwrap().?; | |
| 559 | if (declared_hash.isOld()) { | |
| 560 | const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest); | |
| 561 | if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) { | |
| 562 | return f.fail(hash_tok, try eb.printString( | |
| 563 | "hash mismatch: manifest declares {s} but the fetched package has {s}", | |
| 564 | .{ declared_hash.toSlice(), actual_hex }, | |
| 565 | )); | |
| 566 | } | |
| 567 | } else { | |
| 568 | if (!computed_package_hash.eql(&declared_hash)) { | |
| 569 | return f.fail(hash_tok, try eb.printString( | |
| 570 | "hash mismatch: manifest declares {s} but the fetched package has {s}", | |
| 571 | .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, | |
| 572 | )); | |
| 573 | } | |
| 574 | } | |
| 575 | } else if (!f.omit_missing_hash_error) { | |
| 576 | const notes_len = 1; | |
| 577 | try eb.addRootErrorMessage(.{ | |
| 578 | .msg = try eb.addString("dependency is missing hash field"), | |
| 579 | .src_loc = try f.srcLoc(f.location_tok), | |
| 580 | .notes_len = notes_len, | |
| 581 | }); | |
| 582 | const notes_start = try eb.reserveNotes(notes_len); | |
| 583 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 584 | .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), | |
| 585 | })); | |
| 586 | return error.FetchFailed; | |
| 587 | } | |
| 588 | ||
| 589 | // Spawn a new fetch job for each dependency in the manifest file. Use | |
| 590 | // a mutex and a hash map so that redundant jobs do not get queued up. | |
| 591 | if (!f.job_queue.recursive) return; | |
| 592 | return queueJobsForDeps(f); | |
| 593 | } | |
| 594 | ||
| 595 | pub fn computedPackageHash(f: *const Fetch) Package.Hash { | |
| 596 | const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); | |
| 597 | if (f.manifest) |man| { | |
| 598 | var version_buffer: [32]u8 = undefined; | |
| 599 | const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer; | |
| 600 | return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); | |
| 601 | } | |
| 602 | // In the future build.zig.zon fields will be added to allow overriding these values | |
| 603 | // for naked tarballs. | |
| 604 | return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); | |
| 605 | } | |
| 606 | ||
| 607 | /// `computeHash` gets a free check for the existence of `build.zig`, but when | |
| 608 | /// not computing a hash, we need to do a syscall to check for it. | |
| 609 | fn checkBuildFileExistence(f: *Fetch) RunError!void { | |
| 610 | const eb = &f.error_bundle; | |
| 611 | if (f.package_root.access(Package.build_zig_basename, .{})) |_| { | |
| 612 | f.has_build_zig = true; | |
| 613 | } else |err| switch (err) { | |
| 614 | error.FileNotFound => {}, | |
| 615 | else => |e| { | |
| 616 | try eb.addRootErrorMessage(.{ | |
| 617 | .msg = try eb.printString("unable to access '{}{s}': {s}", .{ | |
| 618 | f.package_root, Package.build_zig_basename, @errorName(e), | |
| 619 | }), | |
| 620 | }); | |
| 621 | return error.FetchFailed; | |
| 622 | }, | |
| 623 | } | |
| 624 | } | |
| 625 | ||
| 626 | /// This function populates `f.manifest` or leaves it `null`. | |
| 627 | fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { | |
| 628 | const eb = &f.error_bundle; | |
| 629 | const arena = f.arena.allocator(); | |
| 630 | const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( | |
| 631 | arena, | |
| 632 | try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), | |
| 633 | Manifest.max_bytes, | |
| 634 | null, | |
| 635 | 1, | |
| 636 | 0, | |
| 637 | ) catch |err| switch (err) { | |
| 638 | error.FileNotFound => return, | |
| 639 | else => |e| { | |
| 640 | const file_path = try pkg_root.join(arena, Manifest.basename); | |
| 641 | try eb.addRootErrorMessage(.{ | |
| 642 | .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ | |
| 643 | file_path, @errorName(e), | |
| 644 | }), | |
| 645 | }); | |
| 646 | return error.FetchFailed; | |
| 647 | }, | |
| 648 | }; | |
| 649 | ||
| 650 | const ast = &f.manifest_ast; | |
| 651 | ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); | |
| 652 | ||
| 653 | if (ast.errors.len > 0) { | |
| 654 | const file_path = try std.fmt.allocPrint(arena, "{}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root}); | |
| 655 | try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); | |
| 656 | return error.FetchFailed; | |
| 657 | } | |
| 658 | ||
| 659 | f.manifest = try Manifest.parse(arena, ast.*, .{ | |
| 660 | .allow_missing_paths_field = f.allow_missing_paths_field, | |
| 661 | .allow_missing_fingerprint = f.allow_missing_fingerprint, | |
| 662 | .allow_name_string = f.allow_name_string, | |
| 663 | }); | |
| 664 | const manifest = &f.manifest.?; | |
| 665 | ||
| 666 | if (manifest.errors.len > 0) { | |
| 667 | const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename }); | |
| 668 | try manifest.copyErrorsIntoBundle(ast.*, src_path, eb); | |
| 669 | return error.FetchFailed; | |
| 670 | } | |
| 671 | } | |
| 672 | ||
| 673 | fn queueJobsForDeps(f: *Fetch) RunError!void { | |
| 674 | assert(f.job_queue.recursive); | |
| 675 | ||
| 676 | // If the package does not have a build.zig.zon file then there are no dependencies. | |
| 677 | const manifest = f.manifest orelse return; | |
| 678 | ||
| 679 | const new_fetches, const prog_names = nf: { | |
| 680 | const parent_arena = f.arena.allocator(); | |
| 681 | const gpa = f.arena.child_allocator; | |
| 682 | const cache_root = f.job_queue.global_cache; | |
| 683 | const dep_names = manifest.dependencies.keys(); | |
| 684 | const deps = manifest.dependencies.values(); | |
| 685 | // Grab the new tasks into a temporary buffer so we can unlock that mutex | |
| 686 | // as fast as possible. | |
| 687 | // This overallocates any fetches that get skipped by the `continue` in the | |
| 688 | // loop below. | |
| 689 | const new_fetches = try parent_arena.alloc(Fetch, deps.len); | |
| 690 | const prog_names = try parent_arena.alloc([]const u8, deps.len); | |
| 691 | var new_fetch_index: usize = 0; | |
| 692 | ||
| 693 | f.job_queue.mutex.lock(); | |
| 694 | defer f.job_queue.mutex.unlock(); | |
| 695 | ||
| 696 | try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); | |
| 697 | try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); | |
| 698 | ||
| 699 | // There are four cases here: | |
| 700 | // * Correct hash is provided by manifest. | |
| 701 | // - Hash map already has the entry, no need to add it again. | |
| 702 | // * Incorrect hash is provided by manifest. | |
| 703 | // - Hash mismatch error emitted; `queueJobsForDeps` is not called. | |
| 704 | // * Hash is not provided by manifest. | |
| 705 | // - Hash missing error emitted; `queueJobsForDeps` is not called. | |
| 706 | // * path-based location is used without a hash. | |
| 707 | // - Hash is added to the table based on the path alone before | |
| 708 | // calling run(); no need to add it again. | |
| 709 | // | |
| 710 | // If we add a dep as lazy and then later try to add the same dep as eager, | |
| 711 | // eagerness takes precedence and the existing entry is updated. | |
| 712 | ||
| 713 | for (dep_names, deps) |dep_name, dep| { | |
| 714 | const new_fetch = &new_fetches[new_fetch_index]; | |
| 715 | const location: Location = switch (dep.location) { | |
| 716 | .url => |url| .{ .remote = .{ | |
| 717 | .url = url, | |
| 718 | .hash = h: { | |
| 719 | const h = dep.hash orelse break :h null; | |
| 720 | const pkg_hash: Package.Hash = .fromSlice(h); | |
| 721 | const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); | |
| 722 | if (gop.found_existing) { | |
| 723 | if (!dep.lazy) { | |
| 724 | gop.value_ptr.*.lazy_status = .eager; | |
| 725 | } | |
| 726 | continue; | |
| 727 | } | |
| 728 | gop.value_ptr.* = new_fetch; | |
| 729 | break :h pkg_hash; | |
| 730 | }, | |
| 731 | } }, | |
| 732 | .path => |rel_path| l: { | |
| 733 | // This might produce an invalid path, which is checked for | |
| 734 | // at the beginning of run(). | |
| 735 | const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); | |
| 736 | const pkg_hash = relativePathDigest(new_root, cache_root); | |
| 737 | const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); | |
| 738 | if (gop.found_existing) { | |
| 739 | if (!dep.lazy) { | |
| 740 | gop.value_ptr.*.lazy_status = .eager; | |
| 741 | } | |
| 742 | continue; | |
| 743 | } | |
| 744 | gop.value_ptr.* = new_fetch; | |
| 745 | break :l .{ .relative_path = new_root }; | |
| 746 | }, | |
| 747 | }; | |
| 748 | prog_names[new_fetch_index] = dep_name; | |
| 749 | new_fetch_index += 1; | |
| 750 | f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); | |
| 751 | new_fetch.* = .{ | |
| 752 | .arena = std.heap.ArenaAllocator.init(gpa), | |
| 753 | .location = location, | |
| 754 | .location_tok = dep.location_tok, | |
| 755 | .hash_tok = dep.hash_tok, | |
| 756 | .name_tok = dep.name_tok, | |
| 757 | .lazy_status = if (dep.lazy) .available else .eager, | |
| 758 | .parent_package_root = f.package_root, | |
| 759 | .parent_manifest_ast = &f.manifest_ast, | |
| 760 | .prog_node = f.prog_node, | |
| 761 | .job_queue = f.job_queue, | |
| 762 | .omit_missing_hash_error = false, | |
| 763 | .allow_missing_paths_field = true, | |
| 764 | .allow_missing_fingerprint = true, | |
| 765 | .allow_name_string = true, | |
| 766 | .use_latest_commit = false, | |
| 767 | ||
| 768 | .package_root = undefined, | |
| 769 | .error_bundle = undefined, | |
| 770 | .manifest = null, | |
| 771 | .manifest_ast = undefined, | |
| 772 | .computed_hash = undefined, | |
| 773 | .has_build_zig = false, | |
| 774 | .oom_flag = false, | |
| 775 | .latest_commit = null, | |
| 776 | ||
| 777 | .module = null, | |
| 778 | }; | |
| 779 | } | |
| 780 | ||
| 781 | f.prog_node.increaseEstimatedTotalItems(new_fetch_index); | |
| 782 | ||
| 783 | break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; | |
| 784 | }; | |
| 785 | ||
| 786 | // Now it's time to give tasks to the thread pool. | |
| 787 | const thread_pool = f.job_queue.thread_pool; | |
| 788 | ||
| 789 | for (new_fetches, prog_names) |*new_fetch, prog_name| { | |
| 790 | thread_pool.spawnWg(&f.job_queue.wait_group, workerRun, .{ new_fetch, prog_name }); | |
| 791 | } | |
| 792 | } | |
| 793 | ||
| 794 | pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { | |
| 795 | return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); | |
| 796 | } | |
| 797 | ||
| 798 | pub fn workerRun(f: *Fetch, prog_name: []const u8) void { | |
| 799 | const prog_node = f.prog_node.start(prog_name, 0); | |
| 800 | defer prog_node.end(); | |
| 801 | ||
| 802 | run(f) catch |err| switch (err) { | |
| 803 | error.OutOfMemory => f.oom_flag = true, | |
| 804 | error.FetchFailed => { | |
| 805 | // Nothing to do because the errors are already reported in `error_bundle`, | |
| 806 | // and a reference is kept to the `Fetch` task inside `all_fetches`. | |
| 807 | }, | |
| 808 | }; | |
| 809 | } | |
| 810 | ||
| 811 | fn srcLoc( | |
| 812 | f: *Fetch, | |
| 813 | tok: std.zig.Ast.TokenIndex, | |
| 814 | ) Allocator.Error!ErrorBundle.SourceLocationIndex { | |
| 815 | const ast = f.parent_manifest_ast orelse return .none; | |
| 816 | const eb = &f.error_bundle; | |
| 817 | const start_loc = ast.tokenLocation(0, tok); | |
| 818 | const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); | |
| 819 | const msg_off = 0; | |
| 820 | return eb.addSourceLocation(.{ | |
| 821 | .src_path = src_path, | |
| 822 | .span_start = ast.tokenStart(tok), | |
| 823 | .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), | |
| 824 | .span_main = ast.tokenStart(tok) + msg_off, | |
| 825 | .line = @intCast(start_loc.line), | |
| 826 | .column = @intCast(start_loc.column), | |
| 827 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | |
| 828 | }); | |
| 829 | } | |
| 830 | ||
| 831 | fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { | |
| 832 | const eb = &f.error_bundle; | |
| 833 | try eb.addRootErrorMessage(.{ | |
| 834 | .msg = msg_str, | |
| 835 | .src_loc = try f.srcLoc(msg_tok), | |
| 836 | }); | |
| 837 | return error.FetchFailed; | |
| 838 | } | |
| 839 | ||
| 840 | const Resource = union(enum) { | |
| 841 | file: fs.File, | |
| 842 | http_request: std.http.Client.Request, | |
| 843 | git: Git, | |
| 844 | dir: fs.Dir, | |
| 845 | ||
| 846 | const Git = struct { | |
| 847 | session: git.Session, | |
| 848 | fetch_stream: git.Session.FetchStream, | |
| 849 | want_oid: git.Oid, | |
| 850 | }; | |
| 851 | ||
| 852 | fn deinit(resource: *Resource) void { | |
| 853 | switch (resource.*) { | |
| 854 | .file => |*file| file.close(), | |
| 855 | .http_request => |*req| req.deinit(), | |
| 856 | .git => |*git_resource| { | |
| 857 | git_resource.fetch_stream.deinit(); | |
| 858 | git_resource.session.deinit(); | |
| 859 | }, | |
| 860 | .dir => |*dir| dir.close(), | |
| 861 | } | |
| 862 | resource.* = undefined; | |
| 863 | } | |
| 864 | ||
| 865 | fn reader(resource: *Resource) std.io.AnyReader { | |
| 866 | return .{ | |
| 867 | .context = resource, | |
| 868 | .readFn = read, | |
| 869 | }; | |
| 870 | } | |
| 871 | ||
| 872 | fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { | |
| 873 | const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); | |
| 874 | switch (resource.*) { | |
| 875 | .file => |*f| return f.read(buffer), | |
| 876 | .http_request => |*r| return r.read(buffer), | |
| 877 | .git => |*g| return g.fetch_stream.read(buffer), | |
| 878 | .dir => unreachable, | |
| 879 | } | |
| 880 | } | |
| 881 | }; | |
| 882 | ||
| 883 | const FileType = enum { | |
| 884 | tar, | |
| 885 | @"tar.gz", | |
| 886 | @"tar.xz", | |
| 887 | @"tar.zst", | |
| 888 | git_pack, | |
| 889 | zip, | |
| 890 | ||
| 891 | fn fromPath(file_path: []const u8) ?FileType { | |
| 892 | if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; | |
| 893 | if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; | |
| 894 | if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; | |
| 895 | if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; | |
| 896 | if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; | |
| 897 | if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; | |
| 898 | if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; | |
| 899 | if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; | |
| 900 | return null; | |
| 901 | } | |
| 902 | ||
| 903 | /// Parameter is a content-disposition header value. | |
| 904 | fn fromContentDisposition(cd_header: []const u8) ?FileType { | |
| 905 | const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse | |
| 906 | return null; | |
| 907 | ||
| 908 | var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse | |
| 909 | return null; | |
| 910 | value_start += "filename".len; | |
| 911 | if (cd_header[value_start] == '*') { | |
| 912 | value_start += 1; | |
| 913 | } | |
| 914 | if (cd_header[value_start] != '=') return null; | |
| 915 | value_start += 1; | |
| 916 | ||
| 917 | var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; | |
| 918 | if (cd_header[value_end - 1] == '\"') { | |
| 919 | value_end -= 1; | |
| 920 | } | |
| 921 | return fromPath(cd_header[value_start..value_end]); | |
| 922 | } | |
| 923 | ||
| 924 | test fromContentDisposition { | |
| 925 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); | |
| 926 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); | |
| 927 | try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); | |
| 928 | try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); | |
| 929 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); | |
| 930 | try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); | |
| 931 | ||
| 932 | try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); | |
| 933 | try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); | |
| 934 | try std.testing.expect(fromContentDisposition("attachment; size=42") == null); | |
| 935 | try std.testing.expect(fromContentDisposition("inline; size=42") == null); | |
| 936 | try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); | |
| 937 | try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); | |
| 938 | } | |
| 939 | }; | |
| 940 | ||
| 941 | const header_buffer_size = 16 * 1024; | |
| 942 | ||
| 943 | fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource { | |
| 944 | const gpa = f.arena.child_allocator; | |
| 945 | const arena = f.arena.allocator(); | |
| 946 | const eb = &f.error_bundle; | |
| 947 | ||
| 948 | if (ascii.eqlIgnoreCase(uri.scheme, "file")) { | |
| 949 | const path = try uri.path.toRawMaybeAlloc(arena); | |
| 950 | return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| { | |
| 951 | return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ | |
| 952 | f.parent_package_root, path, @errorName(err), | |
| 953 | })); | |
| 954 | } }; | |
| 955 | } | |
| 956 | ||
| 957 | const http_client = f.job_queue.http_client; | |
| 958 | ||
| 959 | if (ascii.eqlIgnoreCase(uri.scheme, "http") or | |
| 960 | ascii.eqlIgnoreCase(uri.scheme, "https")) | |
| 961 | { | |
| 962 | var req = http_client.open(.GET, uri, .{ | |
| 963 | .server_header_buffer = server_header_buffer, | |
| 964 | }) catch |err| { | |
| 965 | return f.fail(f.location_tok, try eb.printString( | |
| 966 | "unable to connect to server: {s}", | |
| 967 | .{@errorName(err)}, | |
| 968 | )); | |
| 969 | }; | |
| 970 | errdefer req.deinit(); // releases more than memory | |
| 971 | ||
| 972 | req.send() catch |err| { | |
| 973 | return f.fail(f.location_tok, try eb.printString( | |
| 974 | "HTTP request failed: {s}", | |
| 975 | .{@errorName(err)}, | |
| 976 | )); | |
| 977 | }; | |
| 978 | req.wait() catch |err| { | |
| 979 | return f.fail(f.location_tok, try eb.printString( | |
| 980 | "invalid HTTP response: {s}", | |
| 981 | .{@errorName(err)}, | |
| 982 | )); | |
| 983 | }; | |
| 984 | ||
| 985 | if (req.response.status != .ok) { | |
| 986 | return f.fail(f.location_tok, try eb.printString( | |
| 987 | "bad HTTP response code: '{d} {s}'", | |
| 988 | .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, | |
| 989 | )); | |
| 990 | } | |
| 991 | ||
| 992 | return .{ .http_request = req }; | |
| 993 | } | |
| 994 | ||
| 995 | if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or | |
| 996 | ascii.eqlIgnoreCase(uri.scheme, "git+https")) | |
| 997 | { | |
| 998 | var transport_uri = uri; | |
| 999 | transport_uri.scheme = uri.scheme["git+".len..]; | |
| 1000 | var session = git.Session.init(gpa, http_client, transport_uri, server_header_buffer) catch |err| { | |
| 1001 | return f.fail(f.location_tok, try eb.printString( | |
| 1002 | "unable to discover remote git server capabilities: {s}", | |
| 1003 | .{@errorName(err)}, | |
| 1004 | )); | |
| 1005 | }; | |
| 1006 | errdefer session.deinit(); | |
| 1007 | ||
| 1008 | const want_oid = want_oid: { | |
| 1009 | const want_ref = | |
| 1010 | if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; | |
| 1011 | if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} | |
| 1012 | ||
| 1013 | const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); | |
| 1014 | const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); | |
| 1015 | ||
| 1016 | var ref_iterator = session.listRefs(.{ | |
| 1017 | .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, | |
| 1018 | .include_peeled = true, | |
| 1019 | .server_header_buffer = server_header_buffer, | |
| 1020 | }) catch |err| { | |
| 1021 | return f.fail(f.location_tok, try eb.printString( | |
| 1022 | "unable to list refs: {s}", | |
| 1023 | .{@errorName(err)}, | |
| 1024 | )); | |
| 1025 | }; | |
| 1026 | defer ref_iterator.deinit(); | |
| 1027 | while (ref_iterator.next() catch |err| { | |
| 1028 | return f.fail(f.location_tok, try eb.printString( | |
| 1029 | "unable to iterate refs: {s}", | |
| 1030 | .{@errorName(err)}, | |
| 1031 | )); | |
| 1032 | }) |ref| { | |
| 1033 | if (std.mem.eql(u8, ref.name, want_ref) or | |
| 1034 | std.mem.eql(u8, ref.name, want_ref_head) or | |
| 1035 | std.mem.eql(u8, ref.name, want_ref_tag)) | |
| 1036 | { | |
| 1037 | break :want_oid ref.peeled orelse ref.oid; | |
| 1038 | } | |
| 1039 | } | |
| 1040 | return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); | |
| 1041 | }; | |
| 1042 | if (f.use_latest_commit) { | |
| 1043 | f.latest_commit = want_oid; | |
| 1044 | } else if (uri.fragment == null) { | |
| 1045 | const notes_len = 1; | |
| 1046 | try eb.addRootErrorMessage(.{ | |
| 1047 | .msg = try eb.addString("url field is missing an explicit ref"), | |
| 1048 | .src_loc = try f.srcLoc(f.location_tok), | |
| 1049 | .notes_len = notes_len, | |
| 1050 | }); | |
| 1051 | const notes_start = try eb.reserveNotes(notes_len); | |
| 1052 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1053 | .msg = try eb.printString("try .url = \"{;+/}#{}\",", .{ uri, want_oid }), | |
| 1054 | })); | |
| 1055 | return error.FetchFailed; | |
| 1056 | } | |
| 1057 | ||
| 1058 | var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; | |
| 1059 | _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{want_oid}) catch unreachable; | |
| 1060 | var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| { | |
| 1061 | return f.fail(f.location_tok, try eb.printString( | |
| 1062 | "unable to create fetch stream: {s}", | |
| 1063 | .{@errorName(err)}, | |
| 1064 | )); | |
| 1065 | }; | |
| 1066 | errdefer fetch_stream.deinit(); | |
| 1067 | ||
| 1068 | return .{ .git = .{ | |
| 1069 | .session = session, | |
| 1070 | .fetch_stream = fetch_stream, | |
| 1071 | .want_oid = want_oid, | |
| 1072 | } }; | |
| 1073 | } | |
| 1074 | ||
| 1075 | return f.fail(f.location_tok, try eb.printString( | |
| 1076 | "unsupported URL scheme: {s}", | |
| 1077 | .{uri.scheme}, | |
| 1078 | )); | |
| 1079 | } | |
| 1080 | ||
| 1081 | fn unpackResource( | |
| 1082 | f: *Fetch, | |
| 1083 | resource: *Resource, | |
| 1084 | uri_path: []const u8, | |
| 1085 | tmp_directory: Cache.Directory, | |
| 1086 | ) RunError!UnpackResult { | |
| 1087 | const eb = &f.error_bundle; | |
| 1088 | const file_type = switch (resource.*) { | |
| 1089 | .file => FileType.fromPath(uri_path) orelse | |
| 1090 | return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), | |
| 1091 | ||
| 1092 | .http_request => |req| ft: { | |
| 1093 | // Content-Type takes first precedence. | |
| 1094 | const content_type = req.response.content_type orelse | |
| 1095 | return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); | |
| 1096 | ||
| 1097 | // Extract the MIME type, ignoring charset and boundary directives | |
| 1098 | const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; | |
| 1099 | const mime_type = content_type[0..mime_type_end]; | |
| 1100 | ||
| 1101 | if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) | |
| 1102 | break :ft .tar; | |
| 1103 | ||
| 1104 | if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or | |
| 1105 | ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or | |
| 1106 | ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or | |
| 1107 | ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or | |
| 1108 | ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) | |
| 1109 | { | |
| 1110 | break :ft .@"tar.gz"; | |
| 1111 | } | |
| 1112 | ||
| 1113 | if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) | |
| 1114 | break :ft .@"tar.xz"; | |
| 1115 | ||
| 1116 | if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) | |
| 1117 | break :ft .@"tar.zst"; | |
| 1118 | ||
| 1119 | if (ascii.eqlIgnoreCase(mime_type, "application/zip")) | |
| 1120 | break :ft .zip; | |
| 1121 | ||
| 1122 | if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and | |
| 1123 | !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) | |
| 1124 | { | |
| 1125 | return f.fail(f.location_tok, try eb.printString( | |
| 1126 | "unrecognized 'Content-Type' header: '{s}'", | |
| 1127 | .{content_type}, | |
| 1128 | )); | |
| 1129 | } | |
| 1130 | ||
| 1131 | // Next, the filename from 'content-disposition: attachment' takes precedence. | |
| 1132 | if (req.response.content_disposition) |cd_header| { | |
| 1133 | break :ft FileType.fromContentDisposition(cd_header) orelse { | |
| 1134 | return f.fail(f.location_tok, try eb.printString( | |
| 1135 | "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", | |
| 1136 | .{cd_header}, | |
| 1137 | )); | |
| 1138 | }; | |
| 1139 | } | |
| 1140 | ||
| 1141 | // Finally, the path from the URI is used. | |
| 1142 | break :ft FileType.fromPath(uri_path) orelse { | |
| 1143 | return f.fail(f.location_tok, try eb.printString( | |
| 1144 | "unknown file type: '{s}'", | |
| 1145 | .{uri_path}, | |
| 1146 | )); | |
| 1147 | }; | |
| 1148 | }, | |
| 1149 | ||
| 1150 | .git => .git_pack, | |
| 1151 | ||
| 1152 | .dir => |dir| { | |
| 1153 | f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { | |
| 1154 | return f.fail(f.location_tok, try eb.printString( | |
| 1155 | "unable to copy directory '{s}': {s}", | |
| 1156 | .{ uri_path, @errorName(err) }, | |
| 1157 | )); | |
| 1158 | }; | |
| 1159 | return .{}; | |
| 1160 | }, | |
| 1161 | }; | |
| 1162 | ||
| 1163 | switch (file_type) { | |
| 1164 | .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()), | |
| 1165 | .@"tar.gz" => { | |
| 1166 | const reader = resource.reader(); | |
| 1167 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | |
| 1168 | var dcp = std.compress.gzip.decompressor(br.reader()); | |
| 1169 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | |
| 1170 | }, | |
| 1171 | .@"tar.xz" => { | |
| 1172 | const gpa = f.arena.child_allocator; | |
| 1173 | const reader = resource.reader(); | |
| 1174 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | |
| 1175 | var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| { | |
| 1176 | return f.fail(f.location_tok, try eb.printString( | |
| 1177 | "unable to decompress tarball: {s}", | |
| 1178 | .{@errorName(err)}, | |
| 1179 | )); | |
| 1180 | }; | |
| 1181 | defer dcp.deinit(); | |
| 1182 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | |
| 1183 | }, | |
| 1184 | .@"tar.zst" => { | |
| 1185 | const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; | |
| 1186 | const window_buffer = try f.arena.allocator().create([window_size]u8); | |
| 1187 | const reader = resource.reader(); | |
| 1188 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | |
| 1189 | var dcp = std.compress.zstd.decompressor(br.reader(), .{ | |
| 1190 | .window_buffer = window_buffer, | |
| 1191 | }); | |
| 1192 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); | |
| 1193 | }, | |
| 1194 | .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { | |
| 1195 | error.FetchFailed => return error.FetchFailed, | |
| 1196 | error.OutOfMemory => return error.OutOfMemory, | |
| 1197 | else => |e| return f.fail(f.location_tok, try eb.printString( | |
| 1198 | "unable to unpack git files: {s}", | |
| 1199 | .{@errorName(e)}, | |
| 1200 | )), | |
| 1201 | }, | |
| 1202 | .zip => return try unzip(f, tmp_directory.handle, resource.reader()), | |
| 1203 | } | |
| 1204 | } | |
| 1205 | ||
| 1206 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { | |
| 1207 | const eb = &f.error_bundle; | |
| 1208 | const arena = f.arena.allocator(); | |
| 1209 | ||
| 1210 | var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; | |
| 1211 | ||
| 1212 | std.tar.pipeToFileSystem(out_dir, reader, .{ | |
| 1213 | .diagnostics = &diagnostics, | |
| 1214 | .strip_components = 0, | |
| 1215 | .mode_mode = .ignore, | |
| 1216 | .exclude_empty_directories = true, | |
| 1217 | }) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1218 | "unable to unpack tarball to temporary directory: {s}", | |
| 1219 | .{@errorName(err)}, | |
| 1220 | )); | |
| 1221 | ||
| 1222 | var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; | |
| 1223 | if (diagnostics.errors.items.len > 0) { | |
| 1224 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); | |
| 1225 | for (diagnostics.errors.items) |item| { | |
| 1226 | switch (item) { | |
| 1227 | .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), | |
| 1228 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), | |
| 1229 | .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), | |
| 1230 | .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 | |
| 1231 | } | |
| 1232 | } | |
| 1233 | } | |
| 1234 | return res; | |
| 1235 | } | |
| 1236 | ||
| 1237 | fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult { | |
| 1238 | // We write the entire contents to a file first because zip files | |
| 1239 | // must be processed back to front and they could be too large to | |
| 1240 | // load into memory. | |
| 1241 | ||
| 1242 | const cache_root = f.job_queue.global_cache; | |
| 1243 | ||
| 1244 | // TODO: the downside of this solution is if we get a failure/crash/oom/power out | |
| 1245 | // during this process, we leave behind a zip file that would be | |
| 1246 | // difficult to know if/when it can be cleaned up. | |
| 1247 | // Might be worth it to use a mechanism that enables other processes | |
| 1248 | // to see if the owning process of a file is still alive (on linux this | |
| 1249 | // can be done with file locks). | |
| 1250 | // Coupled with this mechansism, we could also use slots (i.e. zig-cache/tmp/0, | |
| 1251 | // zig-cache/tmp/1, etc) which would mean that subsequent runs would | |
| 1252 | // automatically clean up old dead files. | |
| 1253 | // This could all be done with a simple TmpFile abstraction. | |
| 1254 | const prefix = "tmp/"; | |
| 1255 | const suffix = ".zip"; | |
| 1256 | ||
| 1257 | const random_bytes_count = 20; | |
| 1258 | const random_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count); | |
| 1259 | var zip_path: [prefix.len + random_path_len + suffix.len]u8 = undefined; | |
| 1260 | @memcpy(zip_path[0..prefix.len], prefix); | |
| 1261 | @memcpy(zip_path[prefix.len + random_path_len ..], suffix); | |
| 1262 | { | |
| 1263 | var random_bytes: [random_bytes_count]u8 = undefined; | |
| 1264 | std.crypto.random.bytes(&random_bytes); | |
| 1265 | _ = std.fs.base64_encoder.encode( | |
| 1266 | zip_path[prefix.len..][0..random_path_len], | |
| 1267 | &random_bytes, | |
| 1268 | ); | |
| 1269 | } | |
| 1270 | ||
| 1271 | defer cache_root.handle.deleteFile(&zip_path) catch {}; | |
| 1272 | ||
| 1273 | const eb = &f.error_bundle; | |
| 1274 | ||
| 1275 | { | |
| 1276 | var zip_file = cache_root.handle.createFile( | |
| 1277 | &zip_path, | |
| 1278 | .{}, | |
| 1279 | ) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1280 | "failed to create tmp zip file: {s}", | |
| 1281 | .{@errorName(err)}, | |
| 1282 | )); | |
| 1283 | defer zip_file.close(); | |
| 1284 | var buf: [4096]u8 = undefined; | |
| 1285 | while (true) { | |
| 1286 | const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1287 | "read zip stream failed: {s}", | |
| 1288 | .{@errorName(err)}, | |
| 1289 | )); | |
| 1290 | if (len == 0) break; | |
| 1291 | zip_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1292 | "write temporary zip file failed: {s}", | |
| 1293 | .{@errorName(err)}, | |
| 1294 | )); | |
| 1295 | } | |
| 1296 | } | |
| 1297 | ||
| 1298 | var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; | |
| 1299 | // no need to deinit since we are using an arena allocator | |
| 1300 | ||
| 1301 | { | |
| 1302 | var zip_file = cache_root.handle.openFile( | |
| 1303 | &zip_path, | |
| 1304 | .{}, | |
| 1305 | ) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1306 | "failed to open temporary zip file: {s}", | |
| 1307 | .{@errorName(err)}, | |
| 1308 | )); | |
| 1309 | defer zip_file.close(); | |
| 1310 | ||
| 1311 | std.zip.extract(out_dir, zip_file.seekableStream(), .{ | |
| 1312 | .allow_backslashes = true, | |
| 1313 | .diagnostics = &diagnostics, | |
| 1314 | }) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1315 | "zip extract failed: {s}", | |
| 1316 | .{@errorName(err)}, | |
| 1317 | )); | |
| 1318 | } | |
| 1319 | ||
| 1320 | cache_root.handle.deleteFile(&zip_path) catch |err| return f.fail(f.location_tok, try eb.printString( | |
| 1321 | "delete temporary zip failed: {s}", | |
| 1322 | .{@errorName(err)}, | |
| 1323 | )); | |
| 1324 | ||
| 1325 | const res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; | |
| 1326 | return res; | |
| 1327 | } | |
| 1328 | ||
| 1329 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult { | |
| 1330 | const arena = f.arena.allocator(); | |
| 1331 | const gpa = f.arena.child_allocator; | |
| 1332 | const object_format: git.Oid.Format = resource.want_oid; | |
| 1333 | ||
| 1334 | var res: UnpackResult = .{}; | |
| 1335 | // The .git directory is used to store the packfile and associated index, but | |
| 1336 | // we do not attempt to replicate the exact structure of a real .git | |
| 1337 | // directory, since that isn't relevant for fetching a package. | |
| 1338 | { | |
| 1339 | var pack_dir = try out_dir.makeOpenPath(".git", .{}); | |
| 1340 | defer pack_dir.close(); | |
| 1341 | var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); | |
| 1342 | defer pack_file.close(); | |
| 1343 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1344 | try fifo.pump(resource.fetch_stream.reader(), pack_file.writer()); | |
| 1345 | try pack_file.sync(); | |
| 1346 | ||
| 1347 | var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); | |
| 1348 | defer index_file.close(); | |
| 1349 | { | |
| 1350 | const index_prog_node = f.prog_node.start("Index pack", 0); | |
| 1351 | defer index_prog_node.end(); | |
| 1352 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); | |
| 1353 | try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer()); | |
| 1354 | try index_buffered_writer.flush(); | |
| 1355 | try index_file.sync(); | |
| 1356 | } | |
| 1357 | ||
| 1358 | { | |
| 1359 | const checkout_prog_node = f.prog_node.start("Checkout", 0); | |
| 1360 | defer checkout_prog_node.end(); | |
| 1361 | var repository = try git.Repository.init(gpa, object_format, pack_file, index_file); | |
| 1362 | defer repository.deinit(); | |
| 1363 | var diagnostics: git.Diagnostics = .{ .allocator = arena }; | |
| 1364 | try repository.checkout(out_dir, resource.want_oid, &diagnostics); | |
| 1365 | ||
| 1366 | if (diagnostics.errors.items.len > 0) { | |
| 1367 | try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); | |
| 1368 | for (diagnostics.errors.items) |item| { | |
| 1369 | switch (item) { | |
| 1370 | .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), | |
| 1371 | .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), | |
| 1372 | } | |
| 1373 | } | |
| 1374 | } | |
| 1375 | } | |
| 1376 | } | |
| 1377 | ||
| 1378 | try out_dir.deleteTree(".git"); | |
| 1379 | return res; | |
| 1380 | } | |
| 1381 | ||
| 1382 | fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void { | |
| 1383 | const gpa = f.arena.child_allocator; | |
| 1384 | // Recursive directory copy. | |
| 1385 | var it = try dir.walk(gpa); | |
| 1386 | defer it.deinit(); | |
| 1387 | while (try it.next()) |entry| { | |
| 1388 | switch (entry.kind) { | |
| 1389 | .directory => {}, // omit empty directories | |
| 1390 | .file => { | |
| 1391 | dir.copyFile( | |
| 1392 | entry.path, | |
| 1393 | tmp_dir, | |
| 1394 | entry.path, | |
| 1395 | .{}, | |
| 1396 | ) catch |err| switch (err) { | |
| 1397 | error.FileNotFound => { | |
| 1398 | if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); | |
| 1399 | try dir.copyFile(entry.path, tmp_dir, entry.path, .{}); | |
| 1400 | }, | |
| 1401 | else => |e| return e, | |
| 1402 | }; | |
| 1403 | }, | |
| 1404 | .sym_link => { | |
| 1405 | var buf: [fs.max_path_bytes]u8 = undefined; | |
| 1406 | const link_name = try dir.readLink(entry.path, &buf); | |
| 1407 | // TODO: if this would create a symlink to outside | |
| 1408 | // the destination directory, fail with an error instead. | |
| 1409 | tmp_dir.symLink(link_name, entry.path, .{}) catch |err| switch (err) { | |
| 1410 | error.FileNotFound => { | |
| 1411 | if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); | |
| 1412 | try tmp_dir.symLink(link_name, entry.path, .{}); | |
| 1413 | }, | |
| 1414 | else => |e| return e, | |
| 1415 | }; | |
| 1416 | }, | |
| 1417 | else => return error.IllegalFileTypeInPackage, | |
| 1418 | } | |
| 1419 | } | |
| 1420 | } | |
| 1421 | ||
| 1422 | pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { | |
| 1423 | assert(dest_dir_sub_path[1] == fs.path.sep); | |
| 1424 | var handled_missing_dir = false; | |
| 1425 | while (true) { | |
| 1426 | cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { | |
| 1427 | error.FileNotFound => { | |
| 1428 | if (handled_missing_dir) return err; | |
| 1429 | cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { | |
| 1430 | error.PathAlreadyExists => handled_missing_dir = true, | |
| 1431 | else => |e| return e, | |
| 1432 | }; | |
| 1433 | continue; | |
| 1434 | }, | |
| 1435 | error.PathAlreadyExists, error.AccessDenied => { | |
| 1436 | // Package has been already downloaded and may already be in use on the system. | |
| 1437 | cache_dir.deleteTree(tmp_dir_sub_path) catch { | |
| 1438 | // Garbage files leftover in zig-cache/tmp/ is, as they say | |
| 1439 | // on Star Trek, "operating within normal parameters". | |
| 1440 | }; | |
| 1441 | }, | |
| 1442 | else => |e| return e, | |
| 1443 | }; | |
| 1444 | break; | |
| 1445 | } | |
| 1446 | } | |
| 1447 | ||
| 1448 | const ComputedHash = struct { | |
| 1449 | digest: Package.Hash.Digest, | |
| 1450 | total_size: u64, | |
| 1451 | }; | |
| 1452 | ||
| 1453 | /// Assumes that files not included in the package have already been filtered | |
| 1454 | /// prior to calling this function. This ensures that files not protected by | |
| 1455 | /// the hash are not present on the file system. Empty directories are *not | |
| 1456 | /// hashed* and must not be present on the file system when calling this | |
| 1457 | /// function. | |
| 1458 | fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { | |
| 1459 | // All the path name strings need to be in memory for sorting. | |
| 1460 | const arena = f.arena.allocator(); | |
| 1461 | const gpa = f.arena.child_allocator; | |
| 1462 | const eb = &f.error_bundle; | |
| 1463 | const thread_pool = f.job_queue.thread_pool; | |
| 1464 | const root_dir = pkg_path.root_dir.handle; | |
| 1465 | ||
| 1466 | // Collect all files, recursively, then sort. | |
| 1467 | var all_files = std.ArrayList(*HashedFile).init(gpa); | |
| 1468 | defer all_files.deinit(); | |
| 1469 | ||
| 1470 | var deleted_files = std.ArrayList(*DeletedFile).init(gpa); | |
| 1471 | defer deleted_files.deinit(); | |
| 1472 | ||
| 1473 | // Track directories which had any files deleted from them so that empty directories | |
| 1474 | // can be deleted. | |
| 1475 | var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .empty; | |
| 1476 | defer sus_dirs.deinit(gpa); | |
| 1477 | ||
| 1478 | var walker = try root_dir.walk(gpa); | |
| 1479 | defer walker.deinit(); | |
| 1480 | ||
| 1481 | // Total number of bytes of file contents included in the package. | |
| 1482 | var total_size: u64 = 0; | |
| 1483 | ||
| 1484 | { | |
| 1485 | // The final hash will be a hash of each file hashed independently. This | |
| 1486 | // allows hashing in parallel. | |
| 1487 | var wait_group: WaitGroup = .{}; | |
| 1488 | // `computeHash` is called from a worker thread so there must not be | |
| 1489 | // any waiting without working or a deadlock could occur. | |
| 1490 | defer thread_pool.waitAndWork(&wait_group); | |
| 1491 | ||
| 1492 | while (walker.next() catch |err| { | |
| 1493 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | |
| 1494 | "unable to walk temporary directory '{}': {s}", | |
| 1495 | .{ pkg_path, @errorName(err) }, | |
| 1496 | ) }); | |
| 1497 | return error.FetchFailed; | |
| 1498 | }) |entry| { | |
| 1499 | if (entry.kind == .directory) continue; | |
| 1500 | ||
| 1501 | const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); | |
| 1502 | if (!filter.includePath(entry_pkg_path)) { | |
| 1503 | // Delete instead of including in hash calculation. | |
| 1504 | const fs_path = try arena.dupe(u8, entry.path); | |
| 1505 | ||
| 1506 | // Also track the parent directory in case it becomes empty. | |
| 1507 | if (fs.path.dirname(fs_path)) |parent| | |
| 1508 | try sus_dirs.put(gpa, parent, {}); | |
| 1509 | ||
| 1510 | const deleted_file = try arena.create(DeletedFile); | |
| 1511 | deleted_file.* = .{ | |
| 1512 | .fs_path = fs_path, | |
| 1513 | .failure = undefined, // to be populated by the worker | |
| 1514 | }; | |
| 1515 | thread_pool.spawnWg(&wait_group, workerDeleteFile, .{ root_dir, deleted_file }); | |
| 1516 | try deleted_files.append(deleted_file); | |
| 1517 | continue; | |
| 1518 | } | |
| 1519 | ||
| 1520 | const kind: HashedFile.Kind = switch (entry.kind) { | |
| 1521 | .directory => unreachable, | |
| 1522 | .file => .file, | |
| 1523 | .sym_link => .link, | |
| 1524 | else => return f.fail(f.location_tok, try eb.printString( | |
| 1525 | "package contains '{s}' which has illegal file type '{s}'", | |
| 1526 | .{ entry.path, @tagName(entry.kind) }, | |
| 1527 | )), | |
| 1528 | }; | |
| 1529 | ||
| 1530 | if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) | |
| 1531 | f.has_build_zig = true; | |
| 1532 | ||
| 1533 | const fs_path = try arena.dupe(u8, entry.path); | |
| 1534 | const hashed_file = try arena.create(HashedFile); | |
| 1535 | hashed_file.* = .{ | |
| 1536 | .fs_path = fs_path, | |
| 1537 | .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), | |
| 1538 | .kind = kind, | |
| 1539 | .hash = undefined, // to be populated by the worker | |
| 1540 | .failure = undefined, // to be populated by the worker | |
| 1541 | .size = undefined, // to be populated by the worker | |
| 1542 | }; | |
| 1543 | thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file }); | |
| 1544 | try all_files.append(hashed_file); | |
| 1545 | } | |
| 1546 | } | |
| 1547 | ||
| 1548 | { | |
| 1549 | // Sort by length, descending, so that child directories get removed first. | |
| 1550 | sus_dirs.sortUnstable(@as(struct { | |
| 1551 | keys: []const []const u8, | |
| 1552 | pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { | |
| 1553 | return ctx.keys[b_index].len < ctx.keys[a_index].len; | |
| 1554 | } | |
| 1555 | }, .{ .keys = sus_dirs.keys() })); | |
| 1556 | ||
| 1557 | // During this loop, more entries will be added, so we must loop by index. | |
| 1558 | var i: usize = 0; | |
| 1559 | while (i < sus_dirs.count()) : (i += 1) { | |
| 1560 | const sus_dir = sus_dirs.keys()[i]; | |
| 1561 | root_dir.deleteDir(sus_dir) catch |err| switch (err) { | |
| 1562 | error.DirNotEmpty => continue, | |
| 1563 | error.FileNotFound => continue, | |
| 1564 | else => |e| { | |
| 1565 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | |
| 1566 | "unable to delete empty directory '{s}': {s}", | |
| 1567 | .{ sus_dir, @errorName(e) }, | |
| 1568 | ) }); | |
| 1569 | return error.FetchFailed; | |
| 1570 | }, | |
| 1571 | }; | |
| 1572 | if (fs.path.dirname(sus_dir)) |parent| { | |
| 1573 | try sus_dirs.put(gpa, parent, {}); | |
| 1574 | } | |
| 1575 | } | |
| 1576 | } | |
| 1577 | ||
| 1578 | std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); | |
| 1579 | ||
| 1580 | var hasher = Package.Hash.Algo.init(.{}); | |
| 1581 | var any_failures = false; | |
| 1582 | for (all_files.items) |hashed_file| { | |
| 1583 | hashed_file.failure catch |err| { | |
| 1584 | any_failures = true; | |
| 1585 | try eb.addRootErrorMessage(.{ | |
| 1586 | .msg = try eb.printString("unable to hash '{s}': {s}", .{ | |
| 1587 | hashed_file.fs_path, @errorName(err), | |
| 1588 | }), | |
| 1589 | }); | |
| 1590 | }; | |
| 1591 | hasher.update(&hashed_file.hash); | |
| 1592 | total_size += hashed_file.size; | |
| 1593 | } | |
| 1594 | for (deleted_files.items) |deleted_file| { | |
| 1595 | deleted_file.failure catch |err| { | |
| 1596 | any_failures = true; | |
| 1597 | try eb.addRootErrorMessage(.{ | |
| 1598 | .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ | |
| 1599 | deleted_file.fs_path, @errorName(err), | |
| 1600 | }), | |
| 1601 | }); | |
| 1602 | }; | |
| 1603 | } | |
| 1604 | ||
| 1605 | if (any_failures) return error.FetchFailed; | |
| 1606 | ||
| 1607 | if (f.job_queue.debug_hash) { | |
| 1608 | assert(!f.job_queue.recursive); | |
| 1609 | // Print something to stdout that can be text diffed to figure out why | |
| 1610 | // the package hash is different. | |
| 1611 | dumpHashInfo(all_files.items) catch |err| { | |
| 1612 | std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)}); | |
| 1613 | std.process.exit(1); | |
| 1614 | }; | |
| 1615 | } | |
| 1616 | ||
| 1617 | return .{ | |
| 1618 | .digest = hasher.finalResult(), | |
| 1619 | .total_size = total_size, | |
| 1620 | }; | |
| 1621 | } | |
| 1622 | ||
| 1623 | fn dumpHashInfo(all_files: []const *const HashedFile) !void { | |
| 1624 | const stdout = std.io.getStdOut(); | |
| 1625 | var bw = std.io.bufferedWriter(stdout.writer()); | |
| 1626 | const w = bw.writer(); | |
| 1627 | ||
| 1628 | for (all_files) |hashed_file| { | |
| 1629 | try w.print("{s}: {s}: {s}\n", .{ | |
| 1630 | @tagName(hashed_file.kind), | |
| 1631 | std.fmt.fmtSliceHexLower(&hashed_file.hash), | |
| 1632 | hashed_file.normalized_path, | |
| 1633 | }); | |
| 1634 | } | |
| 1635 | ||
| 1636 | try bw.flush(); | |
| 1637 | } | |
| 1638 | ||
| 1639 | fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile) void { | |
| 1640 | hashed_file.failure = hashFileFallible(dir, hashed_file); | |
| 1641 | } | |
| 1642 | ||
| 1643 | fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void { | |
| 1644 | deleted_file.failure = deleteFileFallible(dir, deleted_file); | |
| 1645 | } | |
| 1646 | ||
| 1647 | fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { | |
| 1648 | var buf: [8000]u8 = undefined; | |
| 1649 | var hasher = Package.Hash.Algo.init(.{}); | |
| 1650 | hasher.update(hashed_file.normalized_path); | |
| 1651 | var file_size: u64 = 0; | |
| 1652 | ||
| 1653 | switch (hashed_file.kind) { | |
| 1654 | .file => { | |
| 1655 | var file = try dir.openFile(hashed_file.fs_path, .{}); | |
| 1656 | defer file.close(); | |
| 1657 | // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 | |
| 1658 | hasher.update(&.{ 0, 0 }); | |
| 1659 | var file_header: FileHeader = .{}; | |
| 1660 | while (true) { | |
| 1661 | const bytes_read = try file.read(&buf); | |
| 1662 | if (bytes_read == 0) break; | |
| 1663 | file_size += bytes_read; | |
| 1664 | hasher.update(buf[0..bytes_read]); | |
| 1665 | file_header.update(buf[0..bytes_read]); | |
| 1666 | } | |
| 1667 | if (file_header.isExecutable()) { | |
| 1668 | try setExecutable(file); | |
| 1669 | } | |
| 1670 | }, | |
| 1671 | .link => { | |
| 1672 | const link_name = try dir.readLink(hashed_file.fs_path, &buf); | |
| 1673 | if (fs.path.sep != canonical_sep) { | |
| 1674 | // Package hashes are intended to be consistent across | |
| 1675 | // platforms which means we must normalize path separators | |
| 1676 | // inside symlinks. | |
| 1677 | normalizePath(link_name); | |
| 1678 | } | |
| 1679 | hasher.update(link_name); | |
| 1680 | }, | |
| 1681 | } | |
| 1682 | hasher.final(&hashed_file.hash); | |
| 1683 | hashed_file.size = file_size; | |
| 1684 | } | |
| 1685 | ||
| 1686 | fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { | |
| 1687 | try dir.deleteFile(deleted_file.fs_path); | |
| 1688 | } | |
| 1689 | ||
| 1690 | fn setExecutable(file: fs.File) !void { | |
| 1691 | if (!std.fs.has_executable_bit) return; | |
| 1692 | ||
| 1693 | const S = std.posix.S; | |
| 1694 | const mode = fs.File.default_mode | S.IXUSR | S.IXGRP | S.IXOTH; | |
| 1695 | try file.chmod(mode); | |
| 1696 | } | |
| 1697 | ||
| 1698 | const DeletedFile = struct { | |
| 1699 | fs_path: []const u8, | |
| 1700 | failure: Error!void, | |
| 1701 | ||
| 1702 | const Error = | |
| 1703 | fs.Dir.DeleteFileError || | |
| 1704 | fs.Dir.DeleteDirError; | |
| 1705 | }; | |
| 1706 | ||
| 1707 | const HashedFile = struct { | |
| 1708 | fs_path: []const u8, | |
| 1709 | normalized_path: []const u8, | |
| 1710 | hash: Package.Hash.Digest, | |
| 1711 | failure: Error!void, | |
| 1712 | kind: Kind, | |
| 1713 | size: u64, | |
| 1714 | ||
| 1715 | const Error = | |
| 1716 | fs.File.OpenError || | |
| 1717 | fs.File.ReadError || | |
| 1718 | fs.File.StatError || | |
| 1719 | fs.File.ChmodError || | |
| 1720 | fs.Dir.ReadLinkError; | |
| 1721 | ||
| 1722 | const Kind = enum { file, link }; | |
| 1723 | ||
| 1724 | fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { | |
| 1725 | _ = context; | |
| 1726 | return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); | |
| 1727 | } | |
| 1728 | }; | |
| 1729 | ||
| 1730 | /// Strips root directory name from file system path. | |
| 1731 | fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { | |
| 1732 | if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; | |
| 1733 | ||
| 1734 | if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { | |
| 1735 | return fs_path[root_dir.len + 1 ..]; | |
| 1736 | } | |
| 1737 | ||
| 1738 | return fs_path; | |
| 1739 | } | |
| 1740 | ||
| 1741 | /// Make a file system path identical independently of operating system path inconsistencies. | |
| 1742 | /// This converts backslashes into forward slashes. | |
| 1743 | fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { | |
| 1744 | const normalized = try arena.dupe(u8, pkg_path); | |
| 1745 | if (fs.path.sep == canonical_sep) return normalized; | |
| 1746 | normalizePath(normalized); | |
| 1747 | return normalized; | |
| 1748 | } | |
| 1749 | ||
| 1750 | const canonical_sep = fs.path.sep_posix; | |
| 1751 | ||
| 1752 | fn normalizePath(bytes: []u8) void { | |
| 1753 | assert(fs.path.sep != canonical_sep); | |
| 1754 | std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); | |
| 1755 | } | |
| 1756 | ||
| 1757 | const Filter = struct { | |
| 1758 | include_paths: std.StringArrayHashMapUnmanaged(void) = .empty, | |
| 1759 | ||
| 1760 | /// sub_path is relative to the package root. | |
| 1761 | pub fn includePath(self: Filter, sub_path: []const u8) bool { | |
| 1762 | if (self.include_paths.count() == 0) return true; | |
| 1763 | if (self.include_paths.contains("")) return true; | |
| 1764 | if (self.include_paths.contains(".")) return true; | |
| 1765 | if (self.include_paths.contains(sub_path)) return true; | |
| 1766 | ||
| 1767 | // Check if any included paths are parent directories of sub_path. | |
| 1768 | var dirname = sub_path; | |
| 1769 | while (std.fs.path.dirname(dirname)) |next_dirname| { | |
| 1770 | if (self.include_paths.contains(next_dirname)) return true; | |
| 1771 | dirname = next_dirname; | |
| 1772 | } | |
| 1773 | ||
| 1774 | return false; | |
| 1775 | } | |
| 1776 | ||
| 1777 | test includePath { | |
| 1778 | const gpa = std.testing.allocator; | |
| 1779 | var filter: Filter = .{}; | |
| 1780 | defer filter.include_paths.deinit(gpa); | |
| 1781 | ||
| 1782 | try filter.include_paths.put(gpa, "src", {}); | |
| 1783 | try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); | |
| 1784 | try std.testing.expect(!filter.includePath(".gitignore")); | |
| 1785 | } | |
| 1786 | }; | |
| 1787 | ||
| 1788 | pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { | |
| 1789 | if (dep.hash) |h| return .fromSlice(h); | |
| 1790 | ||
| 1791 | switch (dep.location) { | |
| 1792 | .url => return null, | |
| 1793 | .path => |rel_path| { | |
| 1794 | var buf: [fs.max_path_bytes]u8 = undefined; | |
| 1795 | var fba = std.heap.FixedBufferAllocator.init(&buf); | |
| 1796 | const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch | |
| 1797 | return null; | |
| 1798 | return relativePathDigest(new_root, cache_root); | |
| 1799 | }, | |
| 1800 | } | |
| 1801 | } | |
| 1802 | ||
| 1803 | const builtin = @import("builtin"); | |
| 1804 | const std = @import("std"); | |
| 1805 | const fs = std.fs; | |
| 1806 | const assert = std.debug.assert; | |
| 1807 | const ascii = std.ascii; | |
| 1808 | const Allocator = std.mem.Allocator; | |
| 1809 | const Cache = std.Build.Cache; | |
| 1810 | const ThreadPool = std.Thread.Pool; | |
| 1811 | const WaitGroup = std.Thread.WaitGroup; | |
| 1812 | const Fetch = @This(); | |
| 1813 | const git = @import("Fetch/git.zig"); | |
| 1814 | const Package = @import("../Package.zig"); | |
| 1815 | const Manifest = Package.Manifest; | |
| 1816 | const ErrorBundle = std.zig.ErrorBundle; | |
| 1817 | const native_os = builtin.os.tag; | |
| 1818 | ||
| 1819 | test { | |
| 1820 | _ = Filter; | |
| 1821 | _ = FileType; | |
| 1822 | _ = UnpackResult; | |
| 1823 | } | |
| 1824 | ||
| 1825 | // Detects executable header: ELF or Macho-O magic header or shebang line. | |
| 1826 | const FileHeader = struct { | |
| 1827 | header: [4]u8 = undefined, | |
| 1828 | bytes_read: usize = 0, | |
| 1829 | ||
| 1830 | pub fn update(self: *FileHeader, buf: []const u8) void { | |
| 1831 | if (self.bytes_read >= self.header.len) return; | |
| 1832 | const n = @min(self.header.len - self.bytes_read, buf.len); | |
| 1833 | @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); | |
| 1834 | self.bytes_read += n; | |
| 1835 | } | |
| 1836 | ||
| 1837 | fn isScript(self: *FileHeader) bool { | |
| 1838 | const shebang = "#!"; | |
| 1839 | return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); | |
| 1840 | } | |
| 1841 | ||
| 1842 | fn isElf(self: *FileHeader) bool { | |
| 1843 | const elf_magic = std.elf.MAGIC; | |
| 1844 | return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); | |
| 1845 | } | |
| 1846 | ||
| 1847 | fn isMachO(self: *FileHeader) bool { | |
| 1848 | if (self.bytes_read < 4) return false; | |
| 1849 | const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); | |
| 1850 | return magic_number == std.macho.MH_MAGIC or | |
| 1851 | magic_number == std.macho.MH_MAGIC_64 or | |
| 1852 | magic_number == std.macho.FAT_MAGIC or | |
| 1853 | magic_number == std.macho.FAT_MAGIC_64 or | |
| 1854 | magic_number == std.macho.MH_CIGAM or | |
| 1855 | magic_number == std.macho.MH_CIGAM_64 or | |
| 1856 | magic_number == std.macho.FAT_CIGAM or | |
| 1857 | magic_number == std.macho.FAT_CIGAM_64; | |
| 1858 | } | |
| 1859 | ||
| 1860 | pub fn isExecutable(self: *FileHeader) bool { | |
| 1861 | return self.isScript() or self.isElf() or self.isMachO(); | |
| 1862 | } | |
| 1863 | }; | |
| 1864 | ||
| 1865 | test FileHeader { | |
| 1866 | var h: FileHeader = .{}; | |
| 1867 | try std.testing.expect(!h.isExecutable()); | |
| 1868 | ||
| 1869 | const elf_magic = std.elf.MAGIC; | |
| 1870 | h.update(elf_magic[0..2]); | |
| 1871 | try std.testing.expect(!h.isExecutable()); | |
| 1872 | h.update(elf_magic[2..4]); | |
| 1873 | try std.testing.expect(h.isExecutable()); | |
| 1874 | ||
| 1875 | h.update(elf_magic[2..4]); | |
| 1876 | try std.testing.expect(h.isExecutable()); | |
| 1877 | ||
| 1878 | const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; | |
| 1879 | h.bytes_read = 0; | |
| 1880 | h.update(&macho64_magic_bytes); | |
| 1881 | try std.testing.expect(h.isExecutable()); | |
| 1882 | ||
| 1883 | const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; | |
| 1884 | h.bytes_read = 0; | |
| 1885 | h.update(&macho64_cigam_bytes); | |
| 1886 | try std.testing.expect(h.isExecutable()); | |
| 1887 | } | |
| 1888 | ||
| 1889 | // Result of the `unpackResource` operation. Enables collecting errors from | |
| 1890 | // tar/git diagnostic, filtering that errors by manifest inclusion rules and | |
| 1891 | // emitting remaining errors to an `ErrorBundle`. | |
| 1892 | const UnpackResult = struct { | |
| 1893 | errors: []Error = undefined, | |
| 1894 | errors_count: usize = 0, | |
| 1895 | root_error_message: []const u8 = "", | |
| 1896 | ||
| 1897 | // A non empty value means that the package contents are inside a | |
| 1898 | // sub-directory indicated by the named path. | |
| 1899 | root_dir: []const u8 = "", | |
| 1900 | ||
| 1901 | const Error = union(enum) { | |
| 1902 | unable_to_create_sym_link: struct { | |
| 1903 | code: anyerror, | |
| 1904 | file_name: []const u8, | |
| 1905 | link_name: []const u8, | |
| 1906 | }, | |
| 1907 | unable_to_create_file: struct { | |
| 1908 | code: anyerror, | |
| 1909 | file_name: []const u8, | |
| 1910 | }, | |
| 1911 | unsupported_file_type: struct { | |
| 1912 | file_name: []const u8, | |
| 1913 | file_type: u8, | |
| 1914 | }, | |
| 1915 | ||
| 1916 | fn excluded(self: Error, filter: Filter) bool { | |
| 1917 | const file_name = switch (self) { | |
| 1918 | .unable_to_create_file => |info| info.file_name, | |
| 1919 | .unable_to_create_sym_link => |info| info.file_name, | |
| 1920 | .unsupported_file_type => |info| info.file_name, | |
| 1921 | }; | |
| 1922 | return !filter.includePath(file_name); | |
| 1923 | } | |
| 1924 | }; | |
| 1925 | ||
| 1926 | fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { | |
| 1927 | self.root_error_message = try arena.dupe(u8, root_error_message); | |
| 1928 | self.errors = try arena.alloc(UnpackResult.Error, n); | |
| 1929 | } | |
| 1930 | ||
| 1931 | fn hasErrors(self: *UnpackResult) bool { | |
| 1932 | return self.errors_count > 0; | |
| 1933 | } | |
| 1934 | ||
| 1935 | fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { | |
| 1936 | self.errors[self.errors_count] = .{ .unable_to_create_file = .{ | |
| 1937 | .code = err, | |
| 1938 | .file_name = file_name, | |
| 1939 | } }; | |
| 1940 | self.errors_count += 1; | |
| 1941 | } | |
| 1942 | ||
| 1943 | fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { | |
| 1944 | self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ | |
| 1945 | .code = err, | |
| 1946 | .file_name = file_name, | |
| 1947 | .link_name = link_name, | |
| 1948 | } }; | |
| 1949 | self.errors_count += 1; | |
| 1950 | } | |
| 1951 | ||
| 1952 | fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { | |
| 1953 | self.errors[self.errors_count] = .{ .unsupported_file_type = .{ | |
| 1954 | .file_name = file_name, | |
| 1955 | .file_type = file_type, | |
| 1956 | } }; | |
| 1957 | self.errors_count += 1; | |
| 1958 | } | |
| 1959 | ||
| 1960 | fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { | |
| 1961 | if (self.errors_count == 0) return; | |
| 1962 | ||
| 1963 | var unfiltered_errors: u32 = 0; | |
| 1964 | for (self.errors) |item| { | |
| 1965 | if (item.excluded(filter)) continue; | |
| 1966 | unfiltered_errors += 1; | |
| 1967 | } | |
| 1968 | if (unfiltered_errors == 0) return; | |
| 1969 | ||
| 1970 | // Emmit errors to an `ErrorBundle`. | |
| 1971 | const eb = &f.error_bundle; | |
| 1972 | try eb.addRootErrorMessage(.{ | |
| 1973 | .msg = try eb.addString(self.root_error_message), | |
| 1974 | .src_loc = try f.srcLoc(f.location_tok), | |
| 1975 | .notes_len = unfiltered_errors, | |
| 1976 | }); | |
| 1977 | var note_i: u32 = try eb.reserveNotes(unfiltered_errors); | |
| 1978 | for (self.errors) |item| { | |
| 1979 | if (item.excluded(filter)) continue; | |
| 1980 | switch (item) { | |
| 1981 | .unable_to_create_sym_link => |info| { | |
| 1982 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1983 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | |
| 1984 | info.file_name, info.link_name, @errorName(info.code), | |
| 1985 | }), | |
| 1986 | })); | |
| 1987 | }, | |
| 1988 | .unable_to_create_file => |info| { | |
| 1989 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1990 | .msg = try eb.printString("unable to create file '{s}': {s}", .{ | |
| 1991 | info.file_name, @errorName(info.code), | |
| 1992 | }), | |
| 1993 | })); | |
| 1994 | }, | |
| 1995 | .unsupported_file_type => |info| { | |
| 1996 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | |
| 1997 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ | |
| 1998 | info.file_name, info.file_type, | |
| 1999 | }), | |
| 2000 | })); | |
| 2001 | }, | |
| 2002 | } | |
| 2003 | note_i += 1; | |
| 2004 | } | |
| 2005 | ||
| 2006 | return error.FetchFailed; | |
| 2007 | } | |
| 2008 | ||
| 2009 | test validate { | |
| 2010 | const gpa = std.testing.allocator; | |
| 2011 | var arena_instance = std.heap.ArenaAllocator.init(gpa); | |
| 2012 | defer arena_instance.deinit(); | |
| 2013 | const arena = arena_instance.allocator(); | |
| 2014 | ||
| 2015 | // fill UnpackResult with errors | |
| 2016 | var res: UnpackResult = .{}; | |
| 2017 | try res.allocErrors(arena, 4, "unable to unpack"); | |
| 2018 | try std.testing.expectEqual(0, res.errors_count); | |
| 2019 | res.unableToCreateFile("dir1/file1", error.File1); | |
| 2020 | res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); | |
| 2021 | res.unableToCreateFile("dir1/file3", error.File3); | |
| 2022 | res.unsupportedFileType("dir2/file4", 'x'); | |
| 2023 | try std.testing.expectEqual(4, res.errors_count); | |
| 2024 | ||
| 2025 | // create filter, includes dir2, excludes dir1 | |
| 2026 | var filter: Filter = .{}; | |
| 2027 | try filter.include_paths.put(arena, "dir2", {}); | |
| 2028 | ||
| 2029 | // init Fetch | |
| 2030 | var fetch: Fetch = undefined; | |
| 2031 | fetch.parent_manifest_ast = null; | |
| 2032 | fetch.location_tok = 0; | |
| 2033 | try fetch.error_bundle.init(gpa); | |
| 2034 | defer fetch.error_bundle.deinit(); | |
| 2035 | ||
| 2036 | // validate errors with filter | |
| 2037 | try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); | |
| 2038 | ||
| 2039 | // output errors to string | |
| 2040 | var errors = try fetch.error_bundle.toOwnedBundle(""); | |
| 2041 | defer errors.deinit(gpa); | |
| 2042 | var out = std.ArrayList(u8).init(gpa); | |
| 2043 | defer out.deinit(); | |
| 2044 | try errors.renderToWriter(.{ .ttyconf = .no_color }, out.writer()); | |
| 2045 | try std.testing.expectEqualStrings( | |
| 2046 | \\error: unable to unpack | |
| 2047 | \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError | |
| 2048 | \\ note: file 'dir2/file4' has unsupported type 'x' | |
| 2049 | \\ | |
| 2050 | , out.items); | |
| 2051 | } | |
| 2052 | }; | |
| 2053 | ||
| 2054 | test "zip" { | |
| 2055 | const gpa = std.testing.allocator; | |
| 2056 | var tmp = std.testing.tmpDir(.{}); | |
| 2057 | defer tmp.cleanup(); | |
| 2058 | ||
| 2059 | const test_files = [_]std.zip.testutil.File{ | |
| 2060 | .{ .name = "foo", .content = "this is just foo\n", .compression = .store }, | |
| 2061 | .{ .name = "bar", .content = "another file\n", .compression = .deflate }, | |
| 2062 | }; | |
| 2063 | { | |
| 2064 | var zip_file = try tmp.dir.createFile("test.zip", .{}); | |
| 2065 | defer zip_file.close(); | |
| 2066 | var bw = std.io.bufferedWriter(zip_file.writer()); | |
| 2067 | var store: [test_files.len]std.zip.testutil.FileStore = undefined; | |
| 2068 | try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); | |
| 2069 | try bw.flush(); | |
| 2070 | } | |
| 2071 | ||
| 2072 | const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); | |
| 2073 | defer gpa.free(zip_path); | |
| 2074 | ||
| 2075 | var fb: TestFetchBuilder = undefined; | |
| 2076 | var fetch = try fb.build(gpa, tmp.dir, zip_path); | |
| 2077 | defer fb.deinit(); | |
| 2078 | ||
| 2079 | try fetch.run(); | |
| 2080 | ||
| 2081 | var out = try fb.packageDir(); | |
| 2082 | defer out.close(); | |
| 2083 | ||
| 2084 | try std.zip.testutil.expectFiles(&test_files, out, .{}); | |
| 2085 | } | |
| 2086 | ||
| 2087 | test "zip with one root folder" { | |
| 2088 | const gpa = std.testing.allocator; | |
| 2089 | var tmp = std.testing.tmpDir(.{}); | |
| 2090 | defer tmp.cleanup(); | |
| 2091 | ||
| 2092 | const test_files = [_]std.zip.testutil.File{ | |
| 2093 | .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store }, | |
| 2094 | .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store }, | |
| 2095 | }; | |
| 2096 | { | |
| 2097 | var zip_file = try tmp.dir.createFile("test.zip", .{}); | |
| 2098 | defer zip_file.close(); | |
| 2099 | var bw = std.io.bufferedWriter(zip_file.writer()); | |
| 2100 | var store: [test_files.len]std.zip.testutil.FileStore = undefined; | |
| 2101 | try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{}); | |
| 2102 | try bw.flush(); | |
| 2103 | } | |
| 2104 | ||
| 2105 | const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path}); | |
| 2106 | defer gpa.free(zip_path); | |
| 2107 | ||
| 2108 | var fb: TestFetchBuilder = undefined; | |
| 2109 | var fetch = try fb.build(gpa, tmp.dir, zip_path); | |
| 2110 | defer fb.deinit(); | |
| 2111 | ||
| 2112 | try fetch.run(); | |
| 2113 | ||
| 2114 | var out = try fb.packageDir(); | |
| 2115 | defer out.close(); | |
| 2116 | ||
| 2117 | try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" }); | |
| 2118 | } | |
| 2119 | ||
| 2120 | test "tarball with duplicate paths" { | |
| 2121 | // This tarball has duplicate path 'dir1/file1' to simulate case sensitve | |
| 2122 | // file system on any file sytstem. | |
| 2123 | // | |
| 2124 | // duplicate_paths/ | |
| 2125 | // duplicate_paths/dir1/ | |
| 2126 | // duplicate_paths/dir1/file1 | |
| 2127 | // duplicate_paths/dir1/file1 | |
| 2128 | // duplicate_paths/build.zig.zon | |
| 2129 | // duplicate_paths/src/ | |
| 2130 | // duplicate_paths/src/main.zig | |
| 2131 | // duplicate_paths/src/root.zig | |
| 2132 | // duplicate_paths/build.zig | |
| 2133 | // | |
| 2134 | ||
| 2135 | const gpa = std.testing.allocator; | |
| 2136 | var tmp = std.testing.tmpDir(.{}); | |
| 2137 | defer tmp.cleanup(); | |
| 2138 | ||
| 2139 | const tarball_name = "duplicate_paths.tar.gz"; | |
| 2140 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2141 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2142 | defer gpa.free(tarball_path); | |
| 2143 | ||
| 2144 | // Run tarball fetch, expect to fail | |
| 2145 | var fb: TestFetchBuilder = undefined; | |
| 2146 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2147 | defer fb.deinit(); | |
| 2148 | try std.testing.expectError(error.FetchFailed, fetch.run()); | |
| 2149 | ||
| 2150 | try fb.expectFetchErrors(1, | |
| 2151 | \\error: unable to unpack tarball | |
| 2152 | \\ note: unable to create file 'dir1/file1': PathAlreadyExists | |
| 2153 | \\ | |
| 2154 | ); | |
| 2155 | } | |
| 2156 | ||
| 2157 | test "tarball with excluded duplicate paths" { | |
| 2158 | // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. | |
| 2159 | // | |
| 2160 | // .paths = .{ | |
| 2161 | // "build.zig", | |
| 2162 | // "build.zig.zon", | |
| 2163 | // "src", | |
| 2164 | // } | |
| 2165 | // | |
| 2166 | ||
| 2167 | const gpa = std.testing.allocator; | |
| 2168 | var tmp = std.testing.tmpDir(.{}); | |
| 2169 | defer tmp.cleanup(); | |
| 2170 | ||
| 2171 | const tarball_name = "duplicate_paths_excluded.tar.gz"; | |
| 2172 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2173 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2174 | defer gpa.free(tarball_path); | |
| 2175 | ||
| 2176 | // Run tarball fetch, should succeed | |
| 2177 | var fb: TestFetchBuilder = undefined; | |
| 2178 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2179 | defer fb.deinit(); | |
| 2180 | try fetch.run(); | |
| 2181 | ||
| 2182 | const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); | |
| 2183 | try std.testing.expectEqualStrings( | |
| 2184 | "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", | |
| 2185 | &hex_digest, | |
| 2186 | ); | |
| 2187 | ||
| 2188 | const expected_files: []const []const u8 = &.{ | |
| 2189 | "build.zig", | |
| 2190 | "build.zig.zon", | |
| 2191 | "src/main.zig", | |
| 2192 | "src/root.zig", | |
| 2193 | }; | |
| 2194 | try fb.expectPackageFiles(expected_files); | |
| 2195 | } | |
| 2196 | ||
| 2197 | test "tarball without root folder" { | |
| 2198 | // Tarball with root folder. Manifest excludes dir1 and dir2. | |
| 2199 | // | |
| 2200 | // build.zig | |
| 2201 | // build.zig.zon | |
| 2202 | // dir1/ | |
| 2203 | // dir1/file2 | |
| 2204 | // dir1/file1 | |
| 2205 | // dir2/ | |
| 2206 | // dir2/file2 | |
| 2207 | // src/ | |
| 2208 | // src/main.zig | |
| 2209 | // | |
| 2210 | ||
| 2211 | const gpa = std.testing.allocator; | |
| 2212 | var tmp = std.testing.tmpDir(.{}); | |
| 2213 | defer tmp.cleanup(); | |
| 2214 | ||
| 2215 | const tarball_name = "no_root.tar.gz"; | |
| 2216 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2217 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2218 | defer gpa.free(tarball_path); | |
| 2219 | ||
| 2220 | // Run tarball fetch, should succeed | |
| 2221 | var fb: TestFetchBuilder = undefined; | |
| 2222 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2223 | defer fb.deinit(); | |
| 2224 | try fetch.run(); | |
| 2225 | ||
| 2226 | const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); | |
| 2227 | try std.testing.expectEqualStrings( | |
| 2228 | "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", | |
| 2229 | &hex_digest, | |
| 2230 | ); | |
| 2231 | ||
| 2232 | const expected_files: []const []const u8 = &.{ | |
| 2233 | "build.zig", | |
| 2234 | "build.zig.zon", | |
| 2235 | "src/main.zig", | |
| 2236 | }; | |
| 2237 | try fb.expectPackageFiles(expected_files); | |
| 2238 | } | |
| 2239 | ||
| 2240 | test "set executable bit based on file content" { | |
| 2241 | if (!std.fs.has_executable_bit) return error.SkipZigTest; | |
| 2242 | const gpa = std.testing.allocator; | |
| 2243 | var tmp = std.testing.tmpDir(.{}); | |
| 2244 | defer tmp.cleanup(); | |
| 2245 | ||
| 2246 | const tarball_name = "executables.tar.gz"; | |
| 2247 | try saveEmbedFile(tarball_name, tmp.dir); | |
| 2248 | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); | |
| 2249 | defer gpa.free(tarball_path); | |
| 2250 | ||
| 2251 | // $ tar -tvf executables.tar.gz | |
| 2252 | // drwxrwxr-x 0 executables/ | |
| 2253 | // -rwxrwxr-x 170 executables/hello | |
| 2254 | // lrwxrwxrwx 0 executables/hello_ln -> hello | |
| 2255 | // -rw-rw-r-- 0 executables/file1 | |
| 2256 | // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit | |
| 2257 | // -rwxrwxr-x 7 executables/script_without_shebang | |
| 2258 | // -rwxrwxr-x 17 executables/script | |
| 2259 | ||
| 2260 | var fb: TestFetchBuilder = undefined; | |
| 2261 | var fetch = try fb.build(gpa, tmp.dir, tarball_path); | |
| 2262 | defer fb.deinit(); | |
| 2263 | ||
| 2264 | try fetch.run(); | |
| 2265 | try std.testing.expectEqualStrings( | |
| 2266 | "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", | |
| 2267 | &Package.multiHashHexDigest(fetch.computed_hash.digest), | |
| 2268 | ); | |
| 2269 | ||
| 2270 | var out = try fb.packageDir(); | |
| 2271 | defer out.close(); | |
| 2272 | const S = std.posix.S; | |
| 2273 | // expect executable bit not set | |
| 2274 | try std.testing.expect((try out.statFile("file1")).mode & S.IXUSR == 0); | |
| 2275 | try std.testing.expect((try out.statFile("script_without_shebang")).mode & S.IXUSR == 0); | |
| 2276 | // expect executable bit set | |
| 2277 | try std.testing.expect((try out.statFile("hello")).mode & S.IXUSR != 0); | |
| 2278 | try std.testing.expect((try out.statFile("script")).mode & S.IXUSR != 0); | |
| 2279 | try std.testing.expect((try out.statFile("script_with_shebang_without_exec_bit")).mode & S.IXUSR != 0); | |
| 2280 | try std.testing.expect((try out.statFile("hello_ln")).mode & S.IXUSR != 0); | |
| 2281 | ||
| 2282 | // | |
| 2283 | // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3 | |
| 2284 | // -rw-rw-r-- 1 0 Apr file1 | |
| 2285 | // -rwxrwxr-x 1 170 Apr hello | |
| 2286 | // lrwxrwxrwx 1 5 Apr hello_ln -> hello | |
| 2287 | // -rwxrwxr-x 1 17 Apr script | |
| 2288 | // -rw-rw-r-- 1 7 Apr script_without_shebang | |
| 2289 | // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit | |
| 2290 | } | |
| 2291 | ||
| 2292 | fn saveEmbedFile(comptime tarball_name: []const u8, dir: fs.Dir) !void { | |
| 2293 | //const tarball_name = "duplicate_paths_excluded.tar.gz"; | |
| 2294 | const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name); | |
| 2295 | var tmp_file = try dir.createFile(tarball_name, .{}); | |
| 2296 | defer tmp_file.close(); | |
| 2297 | try tmp_file.writeAll(tarball_content); | |
| 2298 | } | |
| 2299 | ||
| 2300 | // Builds Fetch with required dependencies, clears dependencies on deinit(). | |
| 2301 | const TestFetchBuilder = struct { | |
| 2302 | thread_pool: ThreadPool, | |
| 2303 | http_client: std.http.Client, | |
| 2304 | global_cache_directory: Cache.Directory, | |
| 2305 | job_queue: Fetch.JobQueue, | |
| 2306 | fetch: Fetch, | |
| 2307 | ||
| 2308 | fn build( | |
| 2309 | self: *TestFetchBuilder, | |
| 2310 | allocator: std.mem.Allocator, | |
| 2311 | cache_parent_dir: std.fs.Dir, | |
| 2312 | path_or_url: []const u8, | |
| 2313 | ) !*Fetch { | |
| 2314 | const cache_dir = try cache_parent_dir.makeOpenPath("zig-global-cache", .{}); | |
| 2315 | ||
| 2316 | try self.thread_pool.init(.{ .allocator = allocator }); | |
| 2317 | self.http_client = .{ .allocator = allocator }; | |
| 2318 | self.global_cache_directory = .{ .handle = cache_dir, .path = null }; | |
| 2319 | ||
| 2320 | self.job_queue = .{ | |
| 2321 | .http_client = &self.http_client, | |
| 2322 | .thread_pool = &self.thread_pool, | |
| 2323 | .global_cache = self.global_cache_directory, | |
| 2324 | .recursive = false, | |
| 2325 | .read_only = false, | |
| 2326 | .debug_hash = false, | |
| 2327 | .work_around_btrfs_bug = false, | |
| 2328 | }; | |
| 2329 | ||
| 2330 | self.fetch = .{ | |
| 2331 | .arena = std.heap.ArenaAllocator.init(allocator), | |
| 2332 | .location = .{ .path_or_url = path_or_url }, | |
| 2333 | .location_tok = 0, | |
| 2334 | .hash_tok = .none, | |
| 2335 | .name_tok = 0, | |
| 2336 | .lazy_status = .eager, | |
| 2337 | .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, | |
| 2338 | .parent_manifest_ast = null, | |
| 2339 | .prog_node = std.Progress.Node.none, | |
| 2340 | .job_queue = &self.job_queue, | |
| 2341 | .omit_missing_hash_error = true, | |
| 2342 | .allow_missing_paths_field = false, | |
| 2343 | .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz | |
| 2344 | .allow_name_string = true, // so we can keep using the old testdata .tar.gz | |
| 2345 | .use_latest_commit = true, | |
| 2346 | ||
| 2347 | .package_root = undefined, | |
| 2348 | .error_bundle = undefined, | |
| 2349 | .manifest = null, | |
| 2350 | .manifest_ast = undefined, | |
| 2351 | .computed_hash = undefined, | |
| 2352 | .has_build_zig = false, | |
| 2353 | .oom_flag = false, | |
| 2354 | .latest_commit = null, | |
| 2355 | ||
| 2356 | .module = null, | |
| 2357 | }; | |
| 2358 | return &self.fetch; | |
| 2359 | } | |
| 2360 | ||
| 2361 | fn deinit(self: *TestFetchBuilder) void { | |
| 2362 | self.fetch.deinit(); | |
| 2363 | self.job_queue.deinit(); | |
| 2364 | self.fetch.prog_node.end(); | |
| 2365 | self.global_cache_directory.handle.close(); | |
| 2366 | self.http_client.deinit(); | |
| 2367 | self.thread_pool.deinit(); | |
| 2368 | } | |
| 2369 | ||
| 2370 | fn packageDir(self: *TestFetchBuilder) !fs.Dir { | |
| 2371 | const root = self.fetch.package_root; | |
| 2372 | return try root.root_dir.handle.openDir(root.sub_path, .{ .iterate = true }); | |
| 2373 | } | |
| 2374 | ||
| 2375 | // Test helper, asserts thet package dir constains expected_files. | |
| 2376 | // expected_files must be sorted. | |
| 2377 | fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void { | |
| 2378 | var package_dir = try self.packageDir(); | |
| 2379 | defer package_dir.close(); | |
| 2380 | ||
| 2381 | var actual_files: std.ArrayListUnmanaged([]u8) = .empty; | |
| 2382 | defer actual_files.deinit(std.testing.allocator); | |
| 2383 | defer for (actual_files.items) |file| std.testing.allocator.free(file); | |
| 2384 | var walker = try package_dir.walk(std.testing.allocator); | |
| 2385 | defer walker.deinit(); | |
| 2386 | while (try walker.next()) |entry| { | |
| 2387 | if (entry.kind != .file) continue; | |
| 2388 | const path = try std.testing.allocator.dupe(u8, entry.path); | |
| 2389 | errdefer std.testing.allocator.free(path); | |
| 2390 | std.mem.replaceScalar(u8, path, std.fs.path.sep, '/'); | |
| 2391 | try actual_files.append(std.testing.allocator, path); | |
| 2392 | } | |
| 2393 | std.mem.sortUnstable([]u8, actual_files.items, {}, struct { | |
| 2394 | fn lessThan(_: void, a: []u8, b: []u8) bool { | |
| 2395 | return std.mem.lessThan(u8, a, b); | |
| 2396 | } | |
| 2397 | }.lessThan); | |
| 2398 | ||
| 2399 | try std.testing.expectEqual(expected_files.len, actual_files.items.len); | |
| 2400 | for (expected_files, 0..) |file_name, i| { | |
| 2401 | try std.testing.expectEqualStrings(file_name, actual_files.items[i]); | |
| 2402 | } | |
| 2403 | try std.testing.expectEqualDeep(expected_files, actual_files.items); | |
| 2404 | } | |
| 2405 | ||
| 2406 | // Test helper, asserts that fetch has failed with `msg` error message. | |
| 2407 | fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void { | |
| 2408 | var errors = try self.fetch.error_bundle.toOwnedBundle(""); | |
| 2409 | defer errors.deinit(std.testing.allocator); | |
| 2410 | ||
| 2411 | const em = errors.getErrorMessage(errors.getMessages()[0]); | |
| 2412 | try std.testing.expectEqual(1, em.count); | |
| 2413 | if (notes_len > 0) { | |
| 2414 | try std.testing.expectEqual(notes_len, em.notes_len); | |
| 2415 | } | |
| 2416 | var al = std.ArrayList(u8).init(std.testing.allocator); | |
| 2417 | defer al.deinit(); | |
| 2418 | try errors.renderToWriter(.{ .ttyconf = .no_color }, al.writer()); | |
| 2419 | try std.testing.expectEqualStrings(msg, al.items); | |
| 2420 | } | |
| 2421 | }; |
src/Package/Fetch/git.zig deleted-1689
| ... | ... | @@ -1,1689 +0,0 @@ |
| 1 | //! Git support for package fetching. | |
| 2 | //! | |
| 3 | //! This is not intended to support all features of Git: it is limited to the | |
| 4 | //! basic functionality needed to clone a repository for the purpose of fetching | |
| 5 | //! a package. | |
| 6 | ||
| 7 | const std = @import("std"); | |
| 8 | const mem = std.mem; | |
| 9 | const testing = std.testing; | |
| 10 | const Allocator = mem.Allocator; | |
| 11 | const Sha1 = std.crypto.hash.Sha1; | |
| 12 | const Sha256 = std.crypto.hash.sha2.Sha256; | |
| 13 | const assert = std.debug.assert; | |
| 14 | ||
| 15 | /// The ID of a Git object. | |
| 16 | pub const Oid = union(Format) { | |
| 17 | sha1: [Sha1.digest_length]u8, | |
| 18 | sha256: [Sha256.digest_length]u8, | |
| 19 | ||
| 20 | pub const max_formatted_length = len: { | |
| 21 | var max: usize = 0; | |
| 22 | for (std.enums.values(Format)) |f| { | |
| 23 | max = @max(max, f.formattedLength()); | |
| 24 | } | |
| 25 | break :len max; | |
| 26 | }; | |
| 27 | ||
| 28 | pub const Format = enum { | |
| 29 | sha1, | |
| 30 | sha256, | |
| 31 | ||
| 32 | pub fn byteLength(f: Format) usize { | |
| 33 | return switch (f) { | |
| 34 | .sha1 => Sha1.digest_length, | |
| 35 | .sha256 => Sha256.digest_length, | |
| 36 | }; | |
| 37 | } | |
| 38 | ||
| 39 | pub fn formattedLength(f: Format) usize { | |
| 40 | return 2 * f.byteLength(); | |
| 41 | } | |
| 42 | }; | |
| 43 | ||
| 44 | const Hasher = union(Format) { | |
| 45 | sha1: Sha1, | |
| 46 | sha256: Sha256, | |
| 47 | ||
| 48 | fn init(oid_format: Format) Hasher { | |
| 49 | return switch (oid_format) { | |
| 50 | .sha1 => .{ .sha1 = Sha1.init(.{}) }, | |
| 51 | .sha256 => .{ .sha256 = Sha256.init(.{}) }, | |
| 52 | }; | |
| 53 | } | |
| 54 | ||
| 55 | // Must be public for use from HashedReader and HashedWriter. | |
| 56 | pub fn update(hasher: *Hasher, b: []const u8) void { | |
| 57 | switch (hasher.*) { | |
| 58 | inline else => |*inner| inner.update(b), | |
| 59 | } | |
| 60 | } | |
| 61 | ||
| 62 | fn finalResult(hasher: *Hasher) Oid { | |
| 63 | return switch (hasher.*) { | |
| 64 | inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), | |
| 65 | }; | |
| 66 | } | |
| 67 | }; | |
| 68 | ||
| 69 | pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { | |
| 70 | assert(bytes.len == oid_format.byteLength()); | |
| 71 | return switch (oid_format) { | |
| 72 | inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), | |
| 73 | }; | |
| 74 | } | |
| 75 | ||
| 76 | pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { | |
| 77 | return switch (oid_format) { | |
| 78 | inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), | |
| 79 | }; | |
| 80 | } | |
| 81 | ||
| 82 | pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { | |
| 83 | switch (oid_format) { | |
| 84 | inline else => |tag| { | |
| 85 | if (s.len != tag.formattedLength()) return error.InvalidOid; | |
| 86 | var bytes: [tag.byteLength()]u8 = undefined; | |
| 87 | for (&bytes, 0..) |*b, i| { | |
| 88 | b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; | |
| 89 | } | |
| 90 | return @unionInit(Oid, @tagName(tag), bytes); | |
| 91 | }, | |
| 92 | } | |
| 93 | } | |
| 94 | ||
| 95 | test parse { | |
| 96 | try testing.expectEqualSlices( | |
| 97 | u8, | |
| 98 | &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, | |
| 99 | &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, | |
| 100 | ); | |
| 101 | try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); | |
| 102 | try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); | |
| 103 | try testing.expectEqualSlices( | |
| 104 | u8, | |
| 105 | &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, | |
| 106 | &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, | |
| 107 | ); | |
| 108 | try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); | |
| 109 | try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); | |
| 110 | try testing.expectError(error.InvalidOid, parse(.sha1, "master")); | |
| 111 | try testing.expectError(error.InvalidOid, parse(.sha256, "master")); | |
| 112 | try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); | |
| 113 | try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); | |
| 114 | } | |
| 115 | ||
| 116 | pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { | |
| 117 | return for (std.enums.values(Format)) |f| { | |
| 118 | if (s.len == f.formattedLength()) break parse(f, s); | |
| 119 | } else error.InvalidOid; | |
| 120 | } | |
| 121 | ||
| 122 | pub fn format( | |
| 123 | oid: Oid, | |
| 124 | comptime fmt: []const u8, | |
| 125 | options: std.fmt.FormatOptions, | |
| 126 | writer: anytype, | |
| 127 | ) @TypeOf(writer).Error!void { | |
| 128 | _ = fmt; | |
| 129 | _ = options; | |
| 130 | try writer.print("{}", .{std.fmt.fmtSliceHexLower(oid.slice())}); | |
| 131 | } | |
| 132 | ||
| 133 | pub fn slice(oid: *const Oid) []const u8 { | |
| 134 | return switch (oid.*) { | |
| 135 | inline else => |*bytes| bytes, | |
| 136 | }; | |
| 137 | } | |
| 138 | }; | |
| 139 | ||
| 140 | pub const Diagnostics = struct { | |
| 141 | allocator: Allocator, | |
| 142 | errors: std.ArrayListUnmanaged(Error) = .empty, | |
| 143 | ||
| 144 | pub const Error = union(enum) { | |
| 145 | unable_to_create_sym_link: struct { | |
| 146 | code: anyerror, | |
| 147 | file_name: []const u8, | |
| 148 | link_name: []const u8, | |
| 149 | }, | |
| 150 | unable_to_create_file: struct { | |
| 151 | code: anyerror, | |
| 152 | file_name: []const u8, | |
| 153 | }, | |
| 154 | }; | |
| 155 | ||
| 156 | pub fn deinit(d: *Diagnostics) void { | |
| 157 | for (d.errors.items) |item| { | |
| 158 | switch (item) { | |
| 159 | .unable_to_create_sym_link => |info| { | |
| 160 | d.allocator.free(info.file_name); | |
| 161 | d.allocator.free(info.link_name); | |
| 162 | }, | |
| 163 | .unable_to_create_file => |info| { | |
| 164 | d.allocator.free(info.file_name); | |
| 165 | }, | |
| 166 | } | |
| 167 | } | |
| 168 | d.errors.deinit(d.allocator); | |
| 169 | d.* = undefined; | |
| 170 | } | |
| 171 | }; | |
| 172 | ||
| 173 | pub const Repository = struct { | |
| 174 | odb: Odb, | |
| 175 | ||
| 176 | pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { | |
| 177 | return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; | |
| 178 | } | |
| 179 | ||
| 180 | pub fn deinit(repository: *Repository) void { | |
| 181 | repository.odb.deinit(); | |
| 182 | repository.* = undefined; | |
| 183 | } | |
| 184 | ||
| 185 | /// Checks out the repository at `commit_oid` to `worktree`. | |
| 186 | pub fn checkout( | |
| 187 | repository: *Repository, | |
| 188 | worktree: std.fs.Dir, | |
| 189 | commit_oid: Oid, | |
| 190 | diagnostics: *Diagnostics, | |
| 191 | ) !void { | |
| 192 | try repository.odb.seekOid(commit_oid); | |
| 193 | const tree_oid = tree_oid: { | |
| 194 | const commit_object = try repository.odb.readObject(); | |
| 195 | if (commit_object.type != .commit) return error.NotACommit; | |
| 196 | break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); | |
| 197 | }; | |
| 198 | try repository.checkoutTree(worktree, tree_oid, "", diagnostics); | |
| 199 | } | |
| 200 | ||
| 201 | /// Checks out the tree at `tree_oid` to `worktree`. | |
| 202 | fn checkoutTree( | |
| 203 | repository: *Repository, | |
| 204 | dir: std.fs.Dir, | |
| 205 | tree_oid: Oid, | |
| 206 | current_path: []const u8, | |
| 207 | diagnostics: *Diagnostics, | |
| 208 | ) !void { | |
| 209 | try repository.odb.seekOid(tree_oid); | |
| 210 | const tree_object = try repository.odb.readObject(); | |
| 211 | if (tree_object.type != .tree) return error.NotATree; | |
| 212 | // The tree object may be evicted from the object cache while we're | |
| 213 | // iterating over it, so we can make a defensive copy here to make sure | |
| 214 | // it remains valid until we're done with it | |
| 215 | const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); | |
| 216 | defer repository.odb.allocator.free(tree_data); | |
| 217 | ||
| 218 | var tree_iter: TreeIterator = .{ | |
| 219 | .format = repository.odb.format, | |
| 220 | .data = tree_data, | |
| 221 | .pos = 0, | |
| 222 | }; | |
| 223 | while (try tree_iter.next()) |entry| { | |
| 224 | switch (entry.type) { | |
| 225 | .directory => { | |
| 226 | try dir.makeDir(entry.name); | |
| 227 | var subdir = try dir.openDir(entry.name, .{}); | |
| 228 | defer subdir.close(); | |
| 229 | const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); | |
| 230 | defer repository.odb.allocator.free(sub_path); | |
| 231 | try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics); | |
| 232 | }, | |
| 233 | .file => { | |
| 234 | try repository.odb.seekOid(entry.oid); | |
| 235 | const file_object = try repository.odb.readObject(); | |
| 236 | if (file_object.type != .blob) return error.InvalidFile; | |
| 237 | var file = dir.createFile(entry.name, .{ .exclusive = true }) catch |e| { | |
| 238 | const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); | |
| 239 | errdefer diagnostics.allocator.free(file_name); | |
| 240 | try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ | |
| 241 | .code = e, | |
| 242 | .file_name = file_name, | |
| 243 | } }); | |
| 244 | continue; | |
| 245 | }; | |
| 246 | defer file.close(); | |
| 247 | try file.writeAll(file_object.data); | |
| 248 | try file.sync(); | |
| 249 | }, | |
| 250 | .symlink => { | |
| 251 | try repository.odb.seekOid(entry.oid); | |
| 252 | const symlink_object = try repository.odb.readObject(); | |
| 253 | if (symlink_object.type != .blob) return error.InvalidFile; | |
| 254 | const link_name = symlink_object.data; | |
| 255 | dir.symLink(link_name, entry.name, .{}) catch |e| { | |
| 256 | const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); | |
| 257 | errdefer diagnostics.allocator.free(file_name); | |
| 258 | const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); | |
| 259 | errdefer diagnostics.allocator.free(link_name_dup); | |
| 260 | try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ | |
| 261 | .code = e, | |
| 262 | .file_name = file_name, | |
| 263 | .link_name = link_name_dup, | |
| 264 | } }); | |
| 265 | }; | |
| 266 | }, | |
| 267 | .gitlink => { | |
| 268 | // Consistent with git archive behavior, create the directory but | |
| 269 | // do nothing else | |
| 270 | try dir.makeDir(entry.name); | |
| 271 | }, | |
| 272 | } | |
| 273 | } | |
| 274 | } | |
| 275 | ||
| 276 | /// Returns the ID of the tree associated with the given commit (provided as | |
| 277 | /// raw object data). | |
| 278 | fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { | |
| 279 | if (!mem.startsWith(u8, commit_data, "tree ") or | |
| 280 | commit_data.len < "tree ".len + format.formattedLength() + "\n".len or | |
| 281 | commit_data["tree ".len + format.formattedLength()] != '\n') | |
| 282 | { | |
| 283 | return error.InvalidCommit; | |
| 284 | } | |
| 285 | return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); | |
| 286 | } | |
| 287 | ||
| 288 | const TreeIterator = struct { | |
| 289 | format: Oid.Format, | |
| 290 | data: []const u8, | |
| 291 | pos: usize, | |
| 292 | ||
| 293 | const Entry = struct { | |
| 294 | type: Type, | |
| 295 | executable: bool, | |
| 296 | name: [:0]const u8, | |
| 297 | oid: Oid, | |
| 298 | ||
| 299 | const Type = enum(u4) { | |
| 300 | directory = 0o4, | |
| 301 | file = 0o10, | |
| 302 | symlink = 0o12, | |
| 303 | gitlink = 0o16, | |
| 304 | }; | |
| 305 | }; | |
| 306 | ||
| 307 | fn next(iterator: *TreeIterator) !?Entry { | |
| 308 | if (iterator.pos == iterator.data.len) return null; | |
| 309 | ||
| 310 | const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; | |
| 311 | const mode: packed struct { | |
| 312 | permission: u9, | |
| 313 | unused: u3, | |
| 314 | type: u4, | |
| 315 | } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); | |
| 316 | const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; | |
| 317 | const executable = switch (mode.permission) { | |
| 318 | 0 => if (@"type" == .file) return error.InvalidTree else false, | |
| 319 | 0o644 => if (@"type" != .file) return error.InvalidTree else false, | |
| 320 | 0o755 => if (@"type" != .file) return error.InvalidTree else true, | |
| 321 | else => return error.InvalidTree, | |
| 322 | }; | |
| 323 | iterator.pos = mode_end + 1; | |
| 324 | ||
| 325 | const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; | |
| 326 | const name = iterator.data[iterator.pos..name_end :0]; | |
| 327 | iterator.pos = name_end + 1; | |
| 328 | ||
| 329 | const oid_length = iterator.format.byteLength(); | |
| 330 | if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; | |
| 331 | const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); | |
| 332 | iterator.pos += oid_length; | |
| 333 | ||
| 334 | return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; | |
| 335 | } | |
| 336 | }; | |
| 337 | }; | |
| 338 | ||
| 339 | /// A Git object database backed by a packfile. A packfile index is also used | |
| 340 | /// for efficient access to objects in the packfile. | |
| 341 | /// | |
| 342 | /// The format of the packfile and its associated index are documented in | |
| 343 | /// [pack-format](https://git-scm.com/docs/pack-format). | |
| 344 | const Odb = struct { | |
| 345 | format: Oid.Format, | |
| 346 | pack_file: std.fs.File, | |
| 347 | index_header: IndexHeader, | |
| 348 | index_file: std.fs.File, | |
| 349 | cache: ObjectCache = .{}, | |
| 350 | allocator: Allocator, | |
| 351 | ||
| 352 | /// Initializes the database from open pack and index files. | |
| 353 | fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { | |
| 354 | try pack_file.seekTo(0); | |
| 355 | try index_file.seekTo(0); | |
| 356 | const index_header = try IndexHeader.read(index_file.reader()); | |
| 357 | return .{ | |
| 358 | .format = format, | |
| 359 | .pack_file = pack_file, | |
| 360 | .index_header = index_header, | |
| 361 | .index_file = index_file, | |
| 362 | .allocator = allocator, | |
| 363 | }; | |
| 364 | } | |
| 365 | ||
| 366 | fn deinit(odb: *Odb) void { | |
| 367 | odb.cache.deinit(odb.allocator); | |
| 368 | odb.* = undefined; | |
| 369 | } | |
| 370 | ||
| 371 | /// Reads the object at the current position in the database. | |
| 372 | fn readObject(odb: *Odb) !Object { | |
| 373 | var base_offset = try odb.pack_file.getPos(); | |
| 374 | var base_header: EntryHeader = undefined; | |
| 375 | var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; | |
| 376 | defer delta_offsets.deinit(odb.allocator); | |
| 377 | const base_object = while (true) { | |
| 378 | if (odb.cache.get(base_offset)) |base_object| break base_object; | |
| 379 | ||
| 380 | base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); | |
| 381 | switch (base_header) { | |
| 382 | .ofs_delta => |ofs_delta| { | |
| 383 | try delta_offsets.append(odb.allocator, base_offset); | |
| 384 | base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; | |
| 385 | try odb.pack_file.seekTo(base_offset); | |
| 386 | }, | |
| 387 | .ref_delta => |ref_delta| { | |
| 388 | try delta_offsets.append(odb.allocator, base_offset); | |
| 389 | try odb.seekOid(ref_delta.base_object); | |
| 390 | base_offset = try odb.pack_file.getPos(); | |
| 391 | }, | |
| 392 | else => { | |
| 393 | const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); | |
| 394 | errdefer odb.allocator.free(base_data); | |
| 395 | const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; | |
| 396 | try odb.cache.put(odb.allocator, base_offset, base_object); | |
| 397 | break base_object; | |
| 398 | }, | |
| 399 | } | |
| 400 | }; | |
| 401 | ||
| 402 | const base_data = try resolveDeltaChain( | |
| 403 | odb.allocator, | |
| 404 | odb.format, | |
| 405 | odb.pack_file, | |
| 406 | base_object, | |
| 407 | delta_offsets.items, | |
| 408 | &odb.cache, | |
| 409 | ); | |
| 410 | ||
| 411 | return .{ .type = base_object.type, .data = base_data }; | |
| 412 | } | |
| 413 | ||
| 414 | /// Seeks to the beginning of the object with the given ID. | |
| 415 | fn seekOid(odb: *Odb, oid: Oid) !void { | |
| 416 | const oid_length = odb.format.byteLength(); | |
| 417 | const key = oid.slice()[0]; | |
| 418 | var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; | |
| 419 | var end_index = odb.index_header.fan_out_table[key]; | |
| 420 | const found_index = while (start_index < end_index) { | |
| 421 | const mid_index = start_index + (end_index - start_index) / 2; | |
| 422 | try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); | |
| 423 | const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); | |
| 424 | switch (mem.order(u8, mid_oid.slice(), oid.slice())) { | |
| 425 | .lt => start_index = mid_index + 1, | |
| 426 | .gt => end_index = mid_index, | |
| 427 | .eq => break mid_index, | |
| 428 | } | |
| 429 | } else return error.ObjectNotFound; | |
| 430 | ||
| 431 | const n_objects = odb.index_header.fan_out_table[255]; | |
| 432 | const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); | |
| 433 | try odb.index_file.seekTo(offset_values_start + found_index * 4); | |
| 434 | const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big)); | |
| 435 | const pack_offset = pack_offset: { | |
| 436 | if (l1_offset.big) { | |
| 437 | const l2_offset_values_start = offset_values_start + n_objects * 4; | |
| 438 | try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); | |
| 439 | break :pack_offset try odb.index_file.reader().readInt(u64, .big); | |
| 440 | } else { | |
| 441 | break :pack_offset l1_offset.value; | |
| 442 | } | |
| 443 | }; | |
| 444 | ||
| 445 | try odb.pack_file.seekTo(pack_offset); | |
| 446 | } | |
| 447 | }; | |
| 448 | ||
| 449 | const Object = struct { | |
| 450 | type: Type, | |
| 451 | data: []const u8, | |
| 452 | ||
| 453 | const Type = enum { | |
| 454 | commit, | |
| 455 | tree, | |
| 456 | blob, | |
| 457 | tag, | |
| 458 | }; | |
| 459 | }; | |
| 460 | ||
| 461 | /// A cache for object data. | |
| 462 | /// | |
| 463 | /// The purpose of this cache is to speed up resolution of deltas by caching the | |
| 464 | /// results of resolving delta objects, while maintaining a maximum cache size | |
| 465 | /// to avoid excessive memory usage. If the total size of the objects in the | |
| 466 | /// cache exceeds the maximum, the cache will begin evicting the least recently | |
| 467 | /// used objects: when resolving delta chains, the most recently used objects | |
| 468 | /// will likely be more helpful as they will be further along in the chain | |
| 469 | /// (skipping earlier reconstruction steps). | |
| 470 | /// | |
| 471 | /// Object data stored in the cache is managed by the cache. It should not be | |
| 472 | /// freed by the caller at any point after inserting it into the cache. Any | |
| 473 | /// objects remaining in the cache will be freed when the cache itself is freed. | |
| 474 | const ObjectCache = struct { | |
| 475 | objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, | |
| 476 | lru_nodes: LruList = .{}, | |
| 477 | byte_size: usize = 0, | |
| 478 | ||
| 479 | const max_byte_size = 128 * 1024 * 1024; // 128MiB | |
| 480 | /// A list of offsets stored in the cache, with the most recently used | |
| 481 | /// entries at the end. | |
| 482 | const LruList = std.DoublyLinkedList(u64); | |
| 483 | const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; | |
| 484 | ||
| 485 | fn deinit(cache: *ObjectCache, allocator: Allocator) void { | |
| 486 | var object_iterator = cache.objects.iterator(); | |
| 487 | while (object_iterator.next()) |object| { | |
| 488 | allocator.free(object.value_ptr.object.data); | |
| 489 | allocator.destroy(object.value_ptr.lru_node); | |
| 490 | } | |
| 491 | cache.objects.deinit(allocator); | |
| 492 | cache.* = undefined; | |
| 493 | } | |
| 494 | ||
| 495 | /// Gets an object from the cache, moving it to the most recently used | |
| 496 | /// position if it is present. | |
| 497 | fn get(cache: *ObjectCache, offset: u64) ?Object { | |
| 498 | if (cache.objects.get(offset)) |entry| { | |
| 499 | cache.lru_nodes.remove(entry.lru_node); | |
| 500 | cache.lru_nodes.append(entry.lru_node); | |
| 501 | return entry.object; | |
| 502 | } else { | |
| 503 | return null; | |
| 504 | } | |
| 505 | } | |
| 506 | ||
| 507 | /// Puts an object in the cache, possibly evicting older entries if the | |
| 508 | /// cache exceeds its maximum size. Note that, although old objects may | |
| 509 | /// be evicted, the object just added to the cache with this function | |
| 510 | /// will not be evicted before the next call to `put` or `deinit` even if | |
| 511 | /// it exceeds the maximum cache size. | |
| 512 | fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { | |
| 513 | const lru_node = try allocator.create(LruList.Node); | |
| 514 | errdefer allocator.destroy(lru_node); | |
| 515 | lru_node.data = offset; | |
| 516 | ||
| 517 | const gop = try cache.objects.getOrPut(allocator, offset); | |
| 518 | if (gop.found_existing) { | |
| 519 | cache.byte_size -= gop.value_ptr.object.data.len; | |
| 520 | cache.lru_nodes.remove(gop.value_ptr.lru_node); | |
| 521 | allocator.destroy(gop.value_ptr.lru_node); | |
| 522 | allocator.free(gop.value_ptr.object.data); | |
| 523 | } | |
| 524 | gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; | |
| 525 | cache.byte_size += object.data.len; | |
| 526 | cache.lru_nodes.append(lru_node); | |
| 527 | ||
| 528 | while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { | |
| 529 | // The > 1 check is to make sure that we don't evict the most | |
| 530 | // recently added node, even if it by itself happens to exceed the | |
| 531 | // maximum size of the cache. | |
| 532 | const evict_node = cache.lru_nodes.popFirst().?; | |
| 533 | const evict_offset = evict_node.data; | |
| 534 | allocator.destroy(evict_node); | |
| 535 | const evict_object = cache.objects.get(evict_offset).?.object; | |
| 536 | cache.byte_size -= evict_object.data.len; | |
| 537 | allocator.free(evict_object.data); | |
| 538 | _ = cache.objects.remove(evict_offset); | |
| 539 | } | |
| 540 | } | |
| 541 | }; | |
| 542 | ||
| 543 | /// A single pkt-line in the Git protocol. | |
| 544 | /// | |
| 545 | /// The format of a pkt-line is documented in | |
| 546 | /// [protocol-common](https://git-scm.com/docs/protocol-common). The special | |
| 547 | /// meanings of the delimiter and response-end packets are documented in | |
| 548 | /// [protocol-v2](https://git-scm.com/docs/protocol-v2). | |
| 549 | const Packet = union(enum) { | |
| 550 | flush, | |
| 551 | delimiter, | |
| 552 | response_end, | |
| 553 | data: []const u8, | |
| 554 | ||
| 555 | const max_data_length = 65516; | |
| 556 | ||
| 557 | /// Reads a packet in pkt-line format. | |
| 558 | fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { | |
| 559 | const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; | |
| 560 | switch (length) { | |
| 561 | 0 => return .flush, | |
| 562 | 1 => return .delimiter, | |
| 563 | 2 => return .response_end, | |
| 564 | 3 => return error.InvalidPacket, | |
| 565 | else => if (length - 4 > max_data_length) return error.InvalidPacket, | |
| 566 | } | |
| 567 | const data = buf[0 .. length - 4]; | |
| 568 | try reader.readNoEof(data); | |
| 569 | return .{ .data = data }; | |
| 570 | } | |
| 571 | ||
| 572 | /// Writes a packet in pkt-line format. | |
| 573 | fn write(packet: Packet, writer: anytype) !void { | |
| 574 | switch (packet) { | |
| 575 | .flush => try writer.writeAll("0000"), | |
| 576 | .delimiter => try writer.writeAll("0001"), | |
| 577 | .response_end => try writer.writeAll("0002"), | |
| 578 | .data => |data| { | |
| 579 | assert(data.len <= max_data_length); | |
| 580 | try writer.print("{x:0>4}", .{data.len + 4}); | |
| 581 | try writer.writeAll(data); | |
| 582 | }, | |
| 583 | } | |
| 584 | } | |
| 585 | ||
| 586 | /// Returns the normalized form of textual packet data, stripping any | |
| 587 | /// trailing '\n'. | |
| 588 | /// | |
| 589 | /// As documented in | |
| 590 | /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), | |
| 591 | /// non-binary (textual) pkt-line data should contain a trailing '\n', but | |
| 592 | /// is not required to do so (implementations must support both forms). | |
| 593 | fn normalizeText(data: []const u8) []const u8 { | |
| 594 | return if (mem.endsWith(u8, data, "\n")) | |
| 595 | data[0 .. data.len - 1] | |
| 596 | else | |
| 597 | data; | |
| 598 | } | |
| 599 | }; | |
| 600 | ||
| 601 | /// A client session for the Git protocol, currently limited to an HTTP(S) | |
| 602 | /// transport. Only protocol version 2 is supported, as documented in | |
| 603 | /// [protocol-v2](https://git-scm.com/docs/protocol-v2). | |
| 604 | pub const Session = struct { | |
| 605 | transport: *std.http.Client, | |
| 606 | location: Location, | |
| 607 | supports_agent: bool, | |
| 608 | supports_shallow: bool, | |
| 609 | object_format: Oid.Format, | |
| 610 | allocator: Allocator, | |
| 611 | ||
| 612 | const agent = "zig/" ++ @import("builtin").zig_version_string; | |
| 613 | const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); | |
| 614 | ||
| 615 | /// Initializes a client session and discovers the capabilities of the | |
| 616 | /// server for optimal transport. | |
| 617 | pub fn init( | |
| 618 | allocator: Allocator, | |
| 619 | transport: *std.http.Client, | |
| 620 | uri: std.Uri, | |
| 621 | http_headers_buffer: []u8, | |
| 622 | ) !Session { | |
| 623 | var session: Session = .{ | |
| 624 | .transport = transport, | |
| 625 | .location = try .init(allocator, uri), | |
| 626 | .supports_agent = false, | |
| 627 | .supports_shallow = false, | |
| 628 | .object_format = .sha1, | |
| 629 | .allocator = allocator, | |
| 630 | }; | |
| 631 | errdefer session.deinit(); | |
| 632 | var capability_iterator = try session.getCapabilities(http_headers_buffer); | |
| 633 | defer capability_iterator.deinit(); | |
| 634 | while (try capability_iterator.next()) |capability| { | |
| 635 | if (mem.eql(u8, capability.key, "agent")) { | |
| 636 | session.supports_agent = true; | |
| 637 | } else if (mem.eql(u8, capability.key, "fetch")) { | |
| 638 | var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); | |
| 639 | while (feature_iterator.next()) |feature| { | |
| 640 | if (mem.eql(u8, feature, "shallow")) { | |
| 641 | session.supports_shallow = true; | |
| 642 | } | |
| 643 | } | |
| 644 | } else if (mem.eql(u8, capability.key, "object-format")) { | |
| 645 | if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { | |
| 646 | session.object_format = format; | |
| 647 | } | |
| 648 | } | |
| 649 | } | |
| 650 | return session; | |
| 651 | } | |
| 652 | ||
| 653 | pub fn deinit(session: *Session) void { | |
| 654 | session.location.deinit(session.allocator); | |
| 655 | session.* = undefined; | |
| 656 | } | |
| 657 | ||
| 658 | /// An owned `std.Uri` representing the location of the server (base URI). | |
| 659 | const Location = struct { | |
| 660 | uri: std.Uri, | |
| 661 | ||
| 662 | fn init(allocator: Allocator, uri: std.Uri) !Location { | |
| 663 | const scheme = try allocator.dupe(u8, uri.scheme); | |
| 664 | errdefer allocator.free(scheme); | |
| 665 | const user = if (uri.user) |user| try std.fmt.allocPrint(allocator, "{user}", .{user}) else null; | |
| 666 | errdefer if (user) |s| allocator.free(s); | |
| 667 | const password = if (uri.password) |password| try std.fmt.allocPrint(allocator, "{password}", .{password}) else null; | |
| 668 | errdefer if (password) |s| allocator.free(s); | |
| 669 | const host = if (uri.host) |host| try std.fmt.allocPrint(allocator, "{host}", .{host}) else null; | |
| 670 | errdefer if (host) |s| allocator.free(s); | |
| 671 | const path = try std.fmt.allocPrint(allocator, "{path}", .{uri.path}); | |
| 672 | errdefer allocator.free(path); | |
| 673 | // The query and fragment are not used as part of the base server URI. | |
| 674 | return .{ | |
| 675 | .uri = .{ | |
| 676 | .scheme = scheme, | |
| 677 | .user = if (user) |s| .{ .percent_encoded = s } else null, | |
| 678 | .password = if (password) |s| .{ .percent_encoded = s } else null, | |
| 679 | .host = if (host) |s| .{ .percent_encoded = s } else null, | |
| 680 | .port = uri.port, | |
| 681 | .path = .{ .percent_encoded = path }, | |
| 682 | }, | |
| 683 | }; | |
| 684 | } | |
| 685 | ||
| 686 | fn deinit(loc: *Location, allocator: Allocator) void { | |
| 687 | allocator.free(loc.uri.scheme); | |
| 688 | if (loc.uri.user) |user| allocator.free(user.percent_encoded); | |
| 689 | if (loc.uri.password) |password| allocator.free(password.percent_encoded); | |
| 690 | if (loc.uri.host) |host| allocator.free(host.percent_encoded); | |
| 691 | allocator.free(loc.uri.path.percent_encoded); | |
| 692 | } | |
| 693 | }; | |
| 694 | ||
| 695 | /// Returns an iterator over capabilities supported by the server. | |
| 696 | /// | |
| 697 | /// The `session.location` is updated if the server returns a redirect, so | |
| 698 | /// that subsequent session functions do not need to handle redirects. | |
| 699 | fn getCapabilities(session: *Session, http_headers_buffer: []u8) !CapabilityIterator { | |
| 700 | var info_refs_uri = session.location.uri; | |
| 701 | { | |
| 702 | const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); | |
| 703 | defer session.allocator.free(session_uri_path); | |
| 704 | info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "info/refs" }) }; | |
| 705 | } | |
| 706 | defer session.allocator.free(info_refs_uri.path.percent_encoded); | |
| 707 | info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; | |
| 708 | info_refs_uri.fragment = null; | |
| 709 | ||
| 710 | const max_redirects = 3; | |
| 711 | var request = try session.transport.open(.GET, info_refs_uri, .{ | |
| 712 | .redirect_behavior = @enumFromInt(max_redirects), | |
| 713 | .server_header_buffer = http_headers_buffer, | |
| 714 | .extra_headers = &.{ | |
| 715 | .{ .name = "Git-Protocol", .value = "version=2" }, | |
| 716 | }, | |
| 717 | }); | |
| 718 | errdefer request.deinit(); | |
| 719 | try request.send(); | |
| 720 | try request.finish(); | |
| 721 | ||
| 722 | try request.wait(); | |
| 723 | if (request.response.status != .ok) return error.ProtocolError; | |
| 724 | const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; | |
| 725 | if (any_redirects_occurred) { | |
| 726 | const request_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{request.uri.path}); | |
| 727 | defer session.allocator.free(request_uri_path); | |
| 728 | if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; | |
| 729 | var new_uri = request.uri; | |
| 730 | new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; | |
| 731 | const new_location: Location = try .init(session.allocator, new_uri); | |
| 732 | session.location.deinit(session.allocator); | |
| 733 | session.location = new_location; | |
| 734 | } | |
| 735 | ||
| 736 | const reader = request.reader(); | |
| 737 | var buf: [Packet.max_data_length]u8 = undefined; | |
| 738 | var state: enum { response_start, response_content } = .response_start; | |
| 739 | while (true) { | |
| 740 | // Some Git servers (at least GitHub) include an additional | |
| 741 | // '# service=git-upload-pack' informative response before sending | |
| 742 | // the expected 'version 2' packet and capability information. | |
| 743 | // This is not universal: SourceHut, for example, does not do this. | |
| 744 | // Thus, we need to skip any such useless additional responses | |
| 745 | // before we get the one we're actually looking for. The responses | |
| 746 | // will be delimited by flush packets. | |
| 747 | const packet = Packet.read(reader, &buf) catch |e| switch (e) { | |
| 748 | error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found | |
| 749 | else => |other| return other, | |
| 750 | }; | |
| 751 | switch (packet) { | |
| 752 | .flush => state = .response_start, | |
| 753 | .data => |data| switch (state) { | |
| 754 | .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { | |
| 755 | return .{ .request = request }; | |
| 756 | } else { | |
| 757 | state = .response_content; | |
| 758 | }, | |
| 759 | else => {}, | |
| 760 | }, | |
| 761 | else => return error.UnexpectedPacket, | |
| 762 | } | |
| 763 | } | |
| 764 | } | |
| 765 | ||
| 766 | const CapabilityIterator = struct { | |
| 767 | request: std.http.Client.Request, | |
| 768 | buf: [Packet.max_data_length]u8 = undefined, | |
| 769 | ||
| 770 | const Capability = struct { | |
| 771 | key: []const u8, | |
| 772 | value: ?[]const u8 = null, | |
| 773 | ||
| 774 | fn parse(data: []const u8) Capability { | |
| 775 | return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| | |
| 776 | .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } | |
| 777 | else | |
| 778 | .{ .key = data }; | |
| 779 | } | |
| 780 | }; | |
| 781 | ||
| 782 | fn deinit(iterator: *CapabilityIterator) void { | |
| 783 | iterator.request.deinit(); | |
| 784 | iterator.* = undefined; | |
| 785 | } | |
| 786 | ||
| 787 | fn next(iterator: *CapabilityIterator) !?Capability { | |
| 788 | switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { | |
| 789 | .flush => return null, | |
| 790 | .data => |data| return Capability.parse(Packet.normalizeText(data)), | |
| 791 | else => return error.UnexpectedPacket, | |
| 792 | } | |
| 793 | } | |
| 794 | }; | |
| 795 | ||
| 796 | const ListRefsOptions = struct { | |
| 797 | /// The ref prefixes (if any) to use to filter the refs available on the | |
| 798 | /// server. Note that the client must still check the returned refs | |
| 799 | /// against its desired filters itself: the server is not required to | |
| 800 | /// respect these prefix filters and may return other refs as well. | |
| 801 | ref_prefixes: []const []const u8 = &.{}, | |
| 802 | /// Whether to include symref targets for returned symbolic refs. | |
| 803 | include_symrefs: bool = false, | |
| 804 | /// Whether to include the peeled object ID for returned tag refs. | |
| 805 | include_peeled: bool = false, | |
| 806 | server_header_buffer: []u8, | |
| 807 | }; | |
| 808 | ||
| 809 | /// Returns an iterator over refs known to the server. | |
| 810 | pub fn listRefs(session: Session, options: ListRefsOptions) !RefIterator { | |
| 811 | var upload_pack_uri = session.location.uri; | |
| 812 | { | |
| 813 | const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); | |
| 814 | defer session.allocator.free(session_uri_path); | |
| 815 | upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; | |
| 816 | } | |
| 817 | defer session.allocator.free(upload_pack_uri.path.percent_encoded); | |
| 818 | upload_pack_uri.query = null; | |
| 819 | upload_pack_uri.fragment = null; | |
| 820 | ||
| 821 | var body: std.ArrayListUnmanaged(u8) = .empty; | |
| 822 | defer body.deinit(session.allocator); | |
| 823 | const body_writer = body.writer(session.allocator); | |
| 824 | try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); | |
| 825 | if (session.supports_agent) { | |
| 826 | try Packet.write(.{ .data = agent_capability }, body_writer); | |
| 827 | } | |
| 828 | { | |
| 829 | const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); | |
| 830 | defer session.allocator.free(object_format_packet); | |
| 831 | try Packet.write(.{ .data = object_format_packet }, body_writer); | |
| 832 | } | |
| 833 | try Packet.write(.delimiter, body_writer); | |
| 834 | for (options.ref_prefixes) |ref_prefix| { | |
| 835 | const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix}); | |
| 836 | defer session.allocator.free(ref_prefix_packet); | |
| 837 | try Packet.write(.{ .data = ref_prefix_packet }, body_writer); | |
| 838 | } | |
| 839 | if (options.include_symrefs) { | |
| 840 | try Packet.write(.{ .data = "symrefs\n" }, body_writer); | |
| 841 | } | |
| 842 | if (options.include_peeled) { | |
| 843 | try Packet.write(.{ .data = "peel\n" }, body_writer); | |
| 844 | } | |
| 845 | try Packet.write(.flush, body_writer); | |
| 846 | ||
| 847 | var request = try session.transport.open(.POST, upload_pack_uri, .{ | |
| 848 | .redirect_behavior = .unhandled, | |
| 849 | .server_header_buffer = options.server_header_buffer, | |
| 850 | .extra_headers = &.{ | |
| 851 | .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, | |
| 852 | .{ .name = "Git-Protocol", .value = "version=2" }, | |
| 853 | }, | |
| 854 | }); | |
| 855 | errdefer request.deinit(); | |
| 856 | request.transfer_encoding = .{ .content_length = body.items.len }; | |
| 857 | try request.send(); | |
| 858 | try request.writeAll(body.items); | |
| 859 | try request.finish(); | |
| 860 | ||
| 861 | try request.wait(); | |
| 862 | if (request.response.status != .ok) return error.ProtocolError; | |
| 863 | ||
| 864 | return .{ | |
| 865 | .format = session.object_format, | |
| 866 | .request = request, | |
| 867 | }; | |
| 868 | } | |
| 869 | ||
| 870 | pub const RefIterator = struct { | |
| 871 | format: Oid.Format, | |
| 872 | request: std.http.Client.Request, | |
| 873 | buf: [Packet.max_data_length]u8 = undefined, | |
| 874 | ||
| 875 | pub const Ref = struct { | |
| 876 | oid: Oid, | |
| 877 | name: []const u8, | |
| 878 | symref_target: ?[]const u8, | |
| 879 | peeled: ?Oid, | |
| 880 | }; | |
| 881 | ||
| 882 | pub fn deinit(iterator: *RefIterator) void { | |
| 883 | iterator.request.deinit(); | |
| 884 | iterator.* = undefined; | |
| 885 | } | |
| 886 | ||
| 887 | pub fn next(iterator: *RefIterator) !?Ref { | |
| 888 | switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { | |
| 889 | .flush => return null, | |
| 890 | .data => |data| { | |
| 891 | const ref_data = Packet.normalizeText(data); | |
| 892 | const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; | |
| 893 | const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; | |
| 894 | ||
| 895 | const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; | |
| 896 | const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; | |
| 897 | ||
| 898 | var symref_target: ?[]const u8 = null; | |
| 899 | var peeled: ?Oid = null; | |
| 900 | var last_sep_pos = name_sep_pos; | |
| 901 | while (last_sep_pos < ref_data.len) { | |
| 902 | const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; | |
| 903 | const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; | |
| 904 | if (mem.startsWith(u8, attribute, "symref-target:")) { | |
| 905 | symref_target = attribute["symref-target:".len..]; | |
| 906 | } else if (mem.startsWith(u8, attribute, "peeled:")) { | |
| 907 | peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; | |
| 908 | } | |
| 909 | last_sep_pos = next_sep_pos; | |
| 910 | } | |
| 911 | ||
| 912 | return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; | |
| 913 | }, | |
| 914 | else => return error.UnexpectedPacket, | |
| 915 | } | |
| 916 | } | |
| 917 | }; | |
| 918 | ||
| 919 | /// Fetches the given refs from the server. A shallow fetch (depth 1) is | |
| 920 | /// performed if the server supports it. | |
| 921 | pub fn fetch( | |
| 922 | session: Session, | |
| 923 | wants: []const []const u8, | |
| 924 | http_headers_buffer: []u8, | |
| 925 | ) !FetchStream { | |
| 926 | var upload_pack_uri = session.location.uri; | |
| 927 | { | |
| 928 | const session_uri_path = try std.fmt.allocPrint(session.allocator, "{path}", .{session.location.uri.path}); | |
| 929 | defer session.allocator.free(session_uri_path); | |
| 930 | upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) }; | |
| 931 | } | |
| 932 | defer session.allocator.free(upload_pack_uri.path.percent_encoded); | |
| 933 | upload_pack_uri.query = null; | |
| 934 | upload_pack_uri.fragment = null; | |
| 935 | ||
| 936 | var body: std.ArrayListUnmanaged(u8) = .empty; | |
| 937 | defer body.deinit(session.allocator); | |
| 938 | const body_writer = body.writer(session.allocator); | |
| 939 | try Packet.write(.{ .data = "command=fetch\n" }, body_writer); | |
| 940 | if (session.supports_agent) { | |
| 941 | try Packet.write(.{ .data = agent_capability }, body_writer); | |
| 942 | } | |
| 943 | { | |
| 944 | const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); | |
| 945 | defer session.allocator.free(object_format_packet); | |
| 946 | try Packet.write(.{ .data = object_format_packet }, body_writer); | |
| 947 | } | |
| 948 | try Packet.write(.delimiter, body_writer); | |
| 949 | // Our packfile parser supports the OFS_DELTA object type | |
| 950 | try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); | |
| 951 | // We do not currently convey server progress information to the user | |
| 952 | try Packet.write(.{ .data = "no-progress\n" }, body_writer); | |
| 953 | if (session.supports_shallow) { | |
| 954 | try Packet.write(.{ .data = "deepen 1\n" }, body_writer); | |
| 955 | } | |
| 956 | for (wants) |want| { | |
| 957 | var buf: [Packet.max_data_length]u8 = undefined; | |
| 958 | const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; | |
| 959 | try Packet.write(.{ .data = arg }, body_writer); | |
| 960 | } | |
| 961 | try Packet.write(.{ .data = "done\n" }, body_writer); | |
| 962 | try Packet.write(.flush, body_writer); | |
| 963 | ||
| 964 | var request = try session.transport.open(.POST, upload_pack_uri, .{ | |
| 965 | .redirect_behavior = .not_allowed, | |
| 966 | .server_header_buffer = http_headers_buffer, | |
| 967 | .extra_headers = &.{ | |
| 968 | .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, | |
| 969 | .{ .name = "Git-Protocol", .value = "version=2" }, | |
| 970 | }, | |
| 971 | }); | |
| 972 | errdefer request.deinit(); | |
| 973 | request.transfer_encoding = .{ .content_length = body.items.len }; | |
| 974 | try request.send(); | |
| 975 | try request.writeAll(body.items); | |
| 976 | try request.finish(); | |
| 977 | ||
| 978 | try request.wait(); | |
| 979 | if (request.response.status != .ok) return error.ProtocolError; | |
| 980 | ||
| 981 | const reader = request.reader(); | |
| 982 | // We are not interested in any of the sections of the returned fetch | |
| 983 | // data other than the packfile section, since we aren't doing anything | |
| 984 | // complex like ref negotiation (this is a fresh clone). | |
| 985 | var state: enum { section_start, section_content } = .section_start; | |
| 986 | while (true) { | |
| 987 | var buf: [Packet.max_data_length]u8 = undefined; | |
| 988 | const packet = try Packet.read(reader, &buf); | |
| 989 | switch (state) { | |
| 990 | .section_start => switch (packet) { | |
| 991 | .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { | |
| 992 | return .{ .request = request }; | |
| 993 | } else { | |
| 994 | state = .section_content; | |
| 995 | }, | |
| 996 | else => return error.UnexpectedPacket, | |
| 997 | }, | |
| 998 | .section_content => switch (packet) { | |
| 999 | .delimiter => state = .section_start, | |
| 1000 | .data => {}, | |
| 1001 | else => return error.UnexpectedPacket, | |
| 1002 | }, | |
| 1003 | } | |
| 1004 | } | |
| 1005 | } | |
| 1006 | ||
| 1007 | pub const FetchStream = struct { | |
| 1008 | request: std.http.Client.Request, | |
| 1009 | buf: [Packet.max_data_length]u8 = undefined, | |
| 1010 | pos: usize = 0, | |
| 1011 | len: usize = 0, | |
| 1012 | ||
| 1013 | pub fn deinit(stream: *FetchStream) void { | |
| 1014 | stream.request.deinit(); | |
| 1015 | } | |
| 1016 | ||
| 1017 | pub const ReadError = std.http.Client.Request.ReadError || error{ | |
| 1018 | InvalidPacket, | |
| 1019 | ProtocolError, | |
| 1020 | UnexpectedPacket, | |
| 1021 | }; | |
| 1022 | pub const Reader = std.io.Reader(*FetchStream, ReadError, read); | |
| 1023 | ||
| 1024 | const StreamCode = enum(u8) { | |
| 1025 | pack_data = 1, | |
| 1026 | progress = 2, | |
| 1027 | fatal_error = 3, | |
| 1028 | _, | |
| 1029 | }; | |
| 1030 | ||
| 1031 | pub fn reader(stream: *FetchStream) Reader { | |
| 1032 | return .{ .context = stream }; | |
| 1033 | } | |
| 1034 | ||
| 1035 | pub fn read(stream: *FetchStream, buf: []u8) !usize { | |
| 1036 | if (stream.pos == stream.len) { | |
| 1037 | while (true) { | |
| 1038 | switch (try Packet.read(stream.request.reader(), &stream.buf)) { | |
| 1039 | .flush => return 0, | |
| 1040 | .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { | |
| 1041 | .pack_data => { | |
| 1042 | stream.pos = 1; | |
| 1043 | stream.len = data.len; | |
| 1044 | break; | |
| 1045 | }, | |
| 1046 | .fatal_error => return error.ProtocolError, | |
| 1047 | else => {}, | |
| 1048 | }, | |
| 1049 | else => return error.UnexpectedPacket, | |
| 1050 | } | |
| 1051 | } | |
| 1052 | } | |
| 1053 | ||
| 1054 | const size = @min(buf.len, stream.len - stream.pos); | |
| 1055 | @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); | |
| 1056 | stream.pos += size; | |
| 1057 | return size; | |
| 1058 | } | |
| 1059 | }; | |
| 1060 | }; | |
| 1061 | ||
| 1062 | const PackHeader = struct { | |
| 1063 | total_objects: u32, | |
| 1064 | ||
| 1065 | const signature = "PACK"; | |
| 1066 | const supported_version = 2; | |
| 1067 | ||
| 1068 | fn read(reader: anytype) !PackHeader { | |
| 1069 | const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { | |
| 1070 | error.EndOfStream => return error.InvalidHeader, | |
| 1071 | else => |other| return other, | |
| 1072 | }; | |
| 1073 | if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; | |
| 1074 | const version = reader.readInt(u32, .big) catch |e| switch (e) { | |
| 1075 | error.EndOfStream => return error.InvalidHeader, | |
| 1076 | else => |other| return other, | |
| 1077 | }; | |
| 1078 | if (version != supported_version) return error.UnsupportedVersion; | |
| 1079 | const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { | |
| 1080 | error.EndOfStream => return error.InvalidHeader, | |
| 1081 | else => |other| return other, | |
| 1082 | }; | |
| 1083 | return .{ .total_objects = total_objects }; | |
| 1084 | } | |
| 1085 | }; | |
| 1086 | ||
| 1087 | const EntryHeader = union(Type) { | |
| 1088 | commit: Undeltified, | |
| 1089 | tree: Undeltified, | |
| 1090 | blob: Undeltified, | |
| 1091 | tag: Undeltified, | |
| 1092 | ofs_delta: OfsDelta, | |
| 1093 | ref_delta: RefDelta, | |
| 1094 | ||
| 1095 | const Type = enum(u3) { | |
| 1096 | commit = 1, | |
| 1097 | tree = 2, | |
| 1098 | blob = 3, | |
| 1099 | tag = 4, | |
| 1100 | ofs_delta = 6, | |
| 1101 | ref_delta = 7, | |
| 1102 | }; | |
| 1103 | ||
| 1104 | const Undeltified = struct { | |
| 1105 | uncompressed_length: u64, | |
| 1106 | }; | |
| 1107 | ||
| 1108 | const OfsDelta = struct { | |
| 1109 | offset: u64, | |
| 1110 | uncompressed_length: u64, | |
| 1111 | }; | |
| 1112 | ||
| 1113 | const RefDelta = struct { | |
| 1114 | base_object: Oid, | |
| 1115 | uncompressed_length: u64, | |
| 1116 | }; | |
| 1117 | ||
| 1118 | fn objectType(header: EntryHeader) Object.Type { | |
| 1119 | return switch (header) { | |
| 1120 | inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), | |
| 1121 | else => unreachable, | |
| 1122 | }; | |
| 1123 | } | |
| 1124 | ||
| 1125 | fn uncompressedLength(header: EntryHeader) u64 { | |
| 1126 | return switch (header) { | |
| 1127 | inline else => |entry| entry.uncompressed_length, | |
| 1128 | }; | |
| 1129 | } | |
| 1130 | ||
| 1131 | fn read(format: Oid.Format, reader: anytype) !EntryHeader { | |
| 1132 | const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; | |
| 1133 | const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { | |
| 1134 | error.EndOfStream => return error.InvalidFormat, | |
| 1135 | else => |other| return other, | |
| 1136 | }); | |
| 1137 | const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; | |
| 1138 | var uncompressed_length: u64 = initial.len; | |
| 1139 | uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; | |
| 1140 | const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; | |
| 1141 | return switch (@"type") { | |
| 1142 | inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ | |
| 1143 | .uncompressed_length = uncompressed_length, | |
| 1144 | }), | |
| 1145 | .ofs_delta => .{ .ofs_delta = .{ | |
| 1146 | .offset = try readOffsetVarInt(reader), | |
| 1147 | .uncompressed_length = uncompressed_length, | |
| 1148 | } }, | |
| 1149 | .ref_delta => .{ .ref_delta = .{ | |
| 1150 | .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { | |
| 1151 | error.EndOfStream => return error.InvalidFormat, | |
| 1152 | else => |other| return other, | |
| 1153 | }, | |
| 1154 | .uncompressed_length = uncompressed_length, | |
| 1155 | } }, | |
| 1156 | }; | |
| 1157 | } | |
| 1158 | }; | |
| 1159 | ||
| 1160 | fn readSizeVarInt(r: anytype) !u64 { | |
| 1161 | const Byte = packed struct { value: u7, has_next: bool }; | |
| 1162 | var b: Byte = @bitCast(try r.readByte()); | |
| 1163 | var value: u64 = b.value; | |
| 1164 | var shift: u6 = 0; | |
| 1165 | while (b.has_next) { | |
| 1166 | b = @bitCast(try r.readByte()); | |
| 1167 | shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; | |
| 1168 | value |= @as(u64, b.value) << shift; | |
| 1169 | } | |
| 1170 | return value; | |
| 1171 | } | |
| 1172 | ||
| 1173 | fn readOffsetVarInt(r: anytype) !u64 { | |
| 1174 | const Byte = packed struct { value: u7, has_next: bool }; | |
| 1175 | var b: Byte = @bitCast(try r.readByte()); | |
| 1176 | var value: u64 = b.value; | |
| 1177 | while (b.has_next) { | |
| 1178 | b = @bitCast(try r.readByte()); | |
| 1179 | value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; | |
| 1180 | value |= b.value; | |
| 1181 | } | |
| 1182 | return value; | |
| 1183 | } | |
| 1184 | ||
| 1185 | const IndexHeader = struct { | |
| 1186 | fan_out_table: [256]u32, | |
| 1187 | ||
| 1188 | const signature = "\xFFtOc"; | |
| 1189 | const supported_version = 2; | |
| 1190 | const size = 4 + 4 + @sizeOf([256]u32); | |
| 1191 | ||
| 1192 | fn read(reader: anytype) !IndexHeader { | |
| 1193 | var header_bytes = try reader.readBytesNoEof(size); | |
| 1194 | if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; | |
| 1195 | const version = mem.readInt(u32, header_bytes[4..8], .big); | |
| 1196 | if (version != supported_version) return error.UnsupportedVersion; | |
| 1197 | ||
| 1198 | var fan_out_table: [256]u32 = undefined; | |
| 1199 | var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); | |
| 1200 | const fan_out_table_reader = fan_out_table_stream.reader(); | |
| 1201 | for (&fan_out_table) |*entry| { | |
| 1202 | entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; | |
| 1203 | } | |
| 1204 | return .{ .fan_out_table = fan_out_table }; | |
| 1205 | } | |
| 1206 | }; | |
| 1207 | ||
| 1208 | const IndexEntry = struct { | |
| 1209 | offset: u64, | |
| 1210 | crc32: u32, | |
| 1211 | }; | |
| 1212 | ||
| 1213 | /// Writes out a version 2 index for the given packfile, as documented in | |
| 1214 | /// [pack-format](https://git-scm.com/docs/pack-format). | |
| 1215 | pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { | |
| 1216 | try pack.seekTo(0); | |
| 1217 | ||
| 1218 | var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; | |
| 1219 | defer index_entries.deinit(allocator); | |
| 1220 | var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty; | |
| 1221 | defer pending_deltas.deinit(allocator); | |
| 1222 | ||
| 1223 | const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); | |
| 1224 | ||
| 1225 | var cache: ObjectCache = .{}; | |
| 1226 | defer cache.deinit(allocator); | |
| 1227 | var remaining_deltas = pending_deltas.items.len; | |
| 1228 | while (remaining_deltas > 0) { | |
| 1229 | var i: usize = remaining_deltas; | |
| 1230 | while (i > 0) { | |
| 1231 | i -= 1; | |
| 1232 | const delta = pending_deltas.items[i]; | |
| 1233 | if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { | |
| 1234 | try index_entries.put(allocator, oid, delta); | |
| 1235 | _ = pending_deltas.swapRemove(i); | |
| 1236 | } | |
| 1237 | } | |
| 1238 | if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; | |
| 1239 | remaining_deltas = pending_deltas.items.len; | |
| 1240 | } | |
| 1241 | ||
| 1242 | var oids: std.ArrayListUnmanaged(Oid) = .empty; | |
| 1243 | defer oids.deinit(allocator); | |
| 1244 | try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); | |
| 1245 | var index_entries_iter = index_entries.iterator(); | |
| 1246 | while (index_entries_iter.next()) |entry| { | |
| 1247 | oids.appendAssumeCapacity(entry.key_ptr.*); | |
| 1248 | } | |
| 1249 | mem.sortUnstable(Oid, oids.items, {}, struct { | |
| 1250 | fn lessThan(_: void, o1: Oid, o2: Oid) bool { | |
| 1251 | return mem.lessThan(u8, o1.slice(), o2.slice()); | |
| 1252 | } | |
| 1253 | }.lessThan); | |
| 1254 | ||
| 1255 | var fan_out_table: [256]u32 = undefined; | |
| 1256 | var count: u32 = 0; | |
| 1257 | var fan_out_index: u8 = 0; | |
| 1258 | for (oids.items) |oid| { | |
| 1259 | const key = oid.slice()[0]; | |
| 1260 | if (key > fan_out_index) { | |
| 1261 | @memset(fan_out_table[fan_out_index..key], count); | |
| 1262 | fan_out_index = key; | |
| 1263 | } | |
| 1264 | count += 1; | |
| 1265 | } | |
| 1266 | @memset(fan_out_table[fan_out_index..], count); | |
| 1267 | ||
| 1268 | var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); | |
| 1269 | const writer = index_hashed_writer.writer(); | |
| 1270 | try writer.writeAll(IndexHeader.signature); | |
| 1271 | try writer.writeInt(u32, IndexHeader.supported_version, .big); | |
| 1272 | for (fan_out_table) |fan_out_entry| { | |
| 1273 | try writer.writeInt(u32, fan_out_entry, .big); | |
| 1274 | } | |
| 1275 | ||
| 1276 | for (oids.items) |oid| { | |
| 1277 | try writer.writeAll(oid.slice()); | |
| 1278 | } | |
| 1279 | ||
| 1280 | for (oids.items) |oid| { | |
| 1281 | try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); | |
| 1282 | } | |
| 1283 | ||
| 1284 | var big_offsets: std.ArrayListUnmanaged(u64) = .empty; | |
| 1285 | defer big_offsets.deinit(allocator); | |
| 1286 | for (oids.items) |oid| { | |
| 1287 | const offset = index_entries.get(oid).?.offset; | |
| 1288 | if (offset <= std.math.maxInt(u31)) { | |
| 1289 | try writer.writeInt(u32, @intCast(offset), .big); | |
| 1290 | } else { | |
| 1291 | const index = big_offsets.items.len; | |
| 1292 | try big_offsets.append(allocator, offset); | |
| 1293 | try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); | |
| 1294 | } | |
| 1295 | } | |
| 1296 | for (big_offsets.items) |offset| { | |
| 1297 | try writer.writeInt(u64, offset, .big); | |
| 1298 | } | |
| 1299 | ||
| 1300 | try writer.writeAll(pack_checksum.slice()); | |
| 1301 | const index_checksum = index_hashed_writer.hasher.finalResult(); | |
| 1302 | try index_writer.writeAll(index_checksum.slice()); | |
| 1303 | } | |
| 1304 | ||
| 1305 | /// Performs the first pass over the packfile data for index construction. | |
| 1306 | /// This will index all non-delta objects, queue delta objects for further | |
| 1307 | /// processing, and return the pack checksum (which is part of the index | |
| 1308 | /// format). | |
| 1309 | fn indexPackFirstPass( | |
| 1310 | allocator: Allocator, | |
| 1311 | format: Oid.Format, | |
| 1312 | pack: std.fs.File, | |
| 1313 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), | |
| 1314 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), | |
| 1315 | ) !Oid { | |
| 1316 | var pack_buffered_reader = std.io.bufferedReader(pack.reader()); | |
| 1317 | var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); | |
| 1318 | var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); | |
| 1319 | const pack_reader = pack_hashed_reader.reader(); | |
| 1320 | ||
| 1321 | const pack_header = try PackHeader.read(pack_reader); | |
| 1322 | ||
| 1323 | var current_entry: u32 = 0; | |
| 1324 | while (current_entry < pack_header.total_objects) : (current_entry += 1) { | |
| 1325 | const entry_offset = pack_counting_reader.bytes_read; | |
| 1326 | var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); | |
| 1327 | const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); | |
| 1328 | switch (entry_header) { | |
| 1329 | .commit, .tree, .blob, .tag => |object| { | |
| 1330 | var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); | |
| 1331 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); | |
| 1332 | var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); | |
| 1333 | const entry_writer = entry_hashed_writer.writer(); | |
| 1334 | // The object header is not included in the pack data but is | |
| 1335 | // part of the object's ID | |
| 1336 | try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); | |
| 1337 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1338 | try fifo.pump(entry_counting_reader.reader(), entry_writer); | |
| 1339 | if (entry_counting_reader.bytes_read != object.uncompressed_length) { | |
| 1340 | return error.InvalidObject; | |
| 1341 | } | |
| 1342 | const oid = entry_hashed_writer.hasher.finalResult(); | |
| 1343 | try index_entries.put(allocator, oid, .{ | |
| 1344 | .offset = entry_offset, | |
| 1345 | .crc32 = entry_crc32_reader.hasher.final(), | |
| 1346 | }); | |
| 1347 | }, | |
| 1348 | inline .ofs_delta, .ref_delta => |delta| { | |
| 1349 | var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); | |
| 1350 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); | |
| 1351 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1352 | try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); | |
| 1353 | if (entry_counting_reader.bytes_read != delta.uncompressed_length) { | |
| 1354 | return error.InvalidObject; | |
| 1355 | } | |
| 1356 | try pending_deltas.append(allocator, .{ | |
| 1357 | .offset = entry_offset, | |
| 1358 | .crc32 = entry_crc32_reader.hasher.final(), | |
| 1359 | }); | |
| 1360 | }, | |
| 1361 | } | |
| 1362 | } | |
| 1363 | ||
| 1364 | const pack_checksum = pack_hashed_reader.hasher.finalResult(); | |
| 1365 | const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); | |
| 1366 | if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { | |
| 1367 | return error.CorruptedPack; | |
| 1368 | } | |
| 1369 | _ = pack_reader.readByte() catch |e| switch (e) { | |
| 1370 | error.EndOfStream => return pack_checksum, | |
| 1371 | else => |other| return other, | |
| 1372 | }; | |
| 1373 | return error.InvalidFormat; | |
| 1374 | } | |
| 1375 | ||
| 1376 | /// Attempts to determine the final object ID of the given deltified object. | |
| 1377 | /// May return null if this is not yet possible (if the delta is a ref-based | |
| 1378 | /// delta and we do not yet know the offset of the base object). | |
| 1379 | fn indexPackHashDelta( | |
| 1380 | allocator: Allocator, | |
| 1381 | format: Oid.Format, | |
| 1382 | pack: std.fs.File, | |
| 1383 | delta: IndexEntry, | |
| 1384 | index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), | |
| 1385 | cache: *ObjectCache, | |
| 1386 | ) !?Oid { | |
| 1387 | // Figure out the chain of deltas to resolve | |
| 1388 | var base_offset = delta.offset; | |
| 1389 | var base_header: EntryHeader = undefined; | |
| 1390 | var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; | |
| 1391 | defer delta_offsets.deinit(allocator); | |
| 1392 | const base_object = while (true) { | |
| 1393 | if (cache.get(base_offset)) |base_object| break base_object; | |
| 1394 | ||
| 1395 | try pack.seekTo(base_offset); | |
| 1396 | base_header = try EntryHeader.read(format, pack.reader()); | |
| 1397 | switch (base_header) { | |
| 1398 | .ofs_delta => |ofs_delta| { | |
| 1399 | try delta_offsets.append(allocator, base_offset); | |
| 1400 | base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; | |
| 1401 | }, | |
| 1402 | .ref_delta => |ref_delta| { | |
| 1403 | try delta_offsets.append(allocator, base_offset); | |
| 1404 | base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; | |
| 1405 | }, | |
| 1406 | else => { | |
| 1407 | const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); | |
| 1408 | errdefer allocator.free(base_data); | |
| 1409 | const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; | |
| 1410 | try cache.put(allocator, base_offset, base_object); | |
| 1411 | break base_object; | |
| 1412 | }, | |
| 1413 | } | |
| 1414 | }; | |
| 1415 | ||
| 1416 | const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); | |
| 1417 | ||
| 1418 | var entry_hasher: Oid.Hasher = .init(format); | |
| 1419 | var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); | |
| 1420 | try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); | |
| 1421 | entry_hasher.update(base_data); | |
| 1422 | return entry_hasher.finalResult(); | |
| 1423 | } | |
| 1424 | ||
| 1425 | /// Resolves a chain of deltas, returning the final base object data. `pack` is | |
| 1426 | /// assumed to be looking at the start of the object data for the base object of | |
| 1427 | /// the chain, and will then apply the deltas in `delta_offsets` in reverse order | |
| 1428 | /// to obtain the final object. | |
| 1429 | fn resolveDeltaChain( | |
| 1430 | allocator: Allocator, | |
| 1431 | format: Oid.Format, | |
| 1432 | pack: std.fs.File, | |
| 1433 | base_object: Object, | |
| 1434 | delta_offsets: []const u64, | |
| 1435 | cache: *ObjectCache, | |
| 1436 | ) ![]const u8 { | |
| 1437 | var base_data = base_object.data; | |
| 1438 | var i: usize = delta_offsets.len; | |
| 1439 | while (i > 0) { | |
| 1440 | i -= 1; | |
| 1441 | ||
| 1442 | const delta_offset = delta_offsets[i]; | |
| 1443 | try pack.seekTo(delta_offset); | |
| 1444 | const delta_header = try EntryHeader.read(format, pack.reader()); | |
| 1445 | const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); | |
| 1446 | defer allocator.free(delta_data); | |
| 1447 | var delta_stream = std.io.fixedBufferStream(delta_data); | |
| 1448 | const delta_reader = delta_stream.reader(); | |
| 1449 | _ = try readSizeVarInt(delta_reader); // base object size | |
| 1450 | const expanded_size = try readSizeVarInt(delta_reader); | |
| 1451 | ||
| 1452 | const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; | |
| 1453 | const expanded_data = try allocator.alloc(u8, expanded_alloc_size); | |
| 1454 | errdefer allocator.free(expanded_data); | |
| 1455 | var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); | |
| 1456 | var base_stream = std.io.fixedBufferStream(base_data); | |
| 1457 | try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); | |
| 1458 | if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; | |
| 1459 | ||
| 1460 | try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); | |
| 1461 | base_data = expanded_data; | |
| 1462 | } | |
| 1463 | return base_data; | |
| 1464 | } | |
| 1465 | ||
| 1466 | /// Reads the complete contents of an object from `reader`. This function may | |
| 1467 | /// read more bytes than required from `reader`, so the reader position after | |
| 1468 | /// returning is not reliable. | |
| 1469 | fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { | |
| 1470 | const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; | |
| 1471 | var buffered_reader = std.io.bufferedReader(reader); | |
| 1472 | var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); | |
| 1473 | const data = try allocator.alloc(u8, alloc_size); | |
| 1474 | errdefer allocator.free(data); | |
| 1475 | try decompress_stream.reader().readNoEof(data); | |
| 1476 | _ = decompress_stream.reader().readByte() catch |e| switch (e) { | |
| 1477 | error.EndOfStream => return data, | |
| 1478 | else => |other| return other, | |
| 1479 | }; | |
| 1480 | return error.InvalidFormat; | |
| 1481 | } | |
| 1482 | ||
| 1483 | /// Expands delta data from `delta_reader` to `writer`. `base_object` must | |
| 1484 | /// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). | |
| 1485 | /// | |
| 1486 | /// The format of the delta data is documented in | |
| 1487 | /// [pack-format](https://git-scm.com/docs/pack-format). | |
| 1488 | fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { | |
| 1489 | while (true) { | |
| 1490 | const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { | |
| 1491 | error.EndOfStream => return, | |
| 1492 | else => |other| return other, | |
| 1493 | }); | |
| 1494 | if (inst.copy) { | |
| 1495 | const available: packed struct { | |
| 1496 | offset1: bool, | |
| 1497 | offset2: bool, | |
| 1498 | offset3: bool, | |
| 1499 | offset4: bool, | |
| 1500 | size1: bool, | |
| 1501 | size2: bool, | |
| 1502 | size3: bool, | |
| 1503 | } = @bitCast(inst.value); | |
| 1504 | const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ | |
| 1505 | .offset1 = if (available.offset1) try delta_reader.readByte() else 0, | |
| 1506 | .offset2 = if (available.offset2) try delta_reader.readByte() else 0, | |
| 1507 | .offset3 = if (available.offset3) try delta_reader.readByte() else 0, | |
| 1508 | .offset4 = if (available.offset4) try delta_reader.readByte() else 0, | |
| 1509 | }; | |
| 1510 | const offset: u32 = @bitCast(offset_parts); | |
| 1511 | const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ | |
| 1512 | .size1 = if (available.size1) try delta_reader.readByte() else 0, | |
| 1513 | .size2 = if (available.size2) try delta_reader.readByte() else 0, | |
| 1514 | .size3 = if (available.size3) try delta_reader.readByte() else 0, | |
| 1515 | }; | |
| 1516 | var size: u24 = @bitCast(size_parts); | |
| 1517 | if (size == 0) size = 0x10000; | |
| 1518 | try base_object.seekTo(offset); | |
| 1519 | var copy_reader = std.io.limitedReader(base_object.reader(), size); | |
| 1520 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1521 | try fifo.pump(copy_reader.reader(), writer); | |
| 1522 | } else if (inst.value != 0) { | |
| 1523 | var data_reader = std.io.limitedReader(delta_reader, inst.value); | |
| 1524 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | |
| 1525 | try fifo.pump(data_reader.reader(), writer); | |
| 1526 | } else { | |
| 1527 | return error.InvalidDeltaInstruction; | |
| 1528 | } | |
| 1529 | } | |
| 1530 | } | |
| 1531 | ||
| 1532 | /// Runs the packfile indexing and checkout test. | |
| 1533 | /// | |
| 1534 | /// The two testrepo repositories under testdata contain identical commit | |
| 1535 | /// histories and contents. | |
| 1536 | /// | |
| 1537 | /// To verify the contents of the packfiles using Git alone, run the | |
| 1538 | /// following commands in an empty directory: | |
| 1539 | /// | |
| 1540 | /// 1. `git init --object-format=(sha1|sha256)` | |
| 1541 | /// 2. `git unpack-objects <path/to/testrepo.pack` | |
| 1542 | /// 3. `git fsck` - will print one "dangling commit": | |
| 1543 | /// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb` | |
| 1544 | /// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a` | |
| 1545 | /// 4. `git checkout $commit` | |
| 1546 | fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void { | |
| 1547 | const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack"); | |
| 1548 | ||
| 1549 | var git_dir = testing.tmpDir(.{}); | |
| 1550 | defer git_dir.cleanup(); | |
| 1551 | var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true }); | |
| 1552 | defer pack_file.close(); | |
| 1553 | try pack_file.writeAll(testrepo_pack); | |
| 1554 | ||
| 1555 | var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); | |
| 1556 | defer index_file.close(); | |
| 1557 | try indexPack(testing.allocator, format, pack_file, index_file.writer()); | |
| 1558 | ||
| 1559 | // Arbitrary size limit on files read while checking the repository contents | |
| 1560 | // (all files in the test repo are known to be smaller than this) | |
| 1561 | const max_file_size = 8192; | |
| 1562 | ||
| 1563 | const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); | |
| 1564 | defer testing.allocator.free(index_file_data); | |
| 1565 | // testrepo.idx is generated by Git. The index created by this file should | |
| 1566 | // match it exactly. Running `git verify-pack -v testrepo.pack` can verify | |
| 1567 | // this. | |
| 1568 | const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); | |
| 1569 | try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); | |
| 1570 | ||
| 1571 | var repository = try Repository.init(testing.allocator, format, pack_file, index_file); | |
| 1572 | defer repository.deinit(); | |
| 1573 | ||
| 1574 | var worktree = testing.tmpDir(.{ .iterate = true }); | |
| 1575 | defer worktree.cleanup(); | |
| 1576 | ||
| 1577 | const commit_id = try Oid.parse(format, head_commit); | |
| 1578 | ||
| 1579 | var diagnostics: Diagnostics = .{ .allocator = testing.allocator }; | |
| 1580 | defer diagnostics.deinit(); | |
| 1581 | try repository.checkout(worktree.dir, commit_id, &diagnostics); | |
| 1582 | try testing.expect(diagnostics.errors.items.len == 0); | |
| 1583 | ||
| 1584 | const expected_files: []const []const u8 = &.{ | |
| 1585 | "dir/file", | |
| 1586 | "dir/subdir/file", | |
| 1587 | "dir/subdir/file2", | |
| 1588 | "dir2/file", | |
| 1589 | "dir3/file", | |
| 1590 | "dir3/file2", | |
| 1591 | "file", | |
| 1592 | "file2", | |
| 1593 | "file3", | |
| 1594 | "file4", | |
| 1595 | "file5", | |
| 1596 | "file6", | |
| 1597 | "file7", | |
| 1598 | "file8", | |
| 1599 | "file9", | |
| 1600 | }; | |
| 1601 | var actual_files: std.ArrayListUnmanaged([]u8) = .empty; | |
| 1602 | defer actual_files.deinit(testing.allocator); | |
| 1603 | defer for (actual_files.items) |file| testing.allocator.free(file); | |
| 1604 | var walker = try worktree.dir.walk(testing.allocator); | |
| 1605 | defer walker.deinit(); | |
| 1606 | while (try walker.next()) |entry| { | |
| 1607 | if (entry.kind != .file) continue; | |
| 1608 | const path = try testing.allocator.dupe(u8, entry.path); | |
| 1609 | errdefer testing.allocator.free(path); | |
| 1610 | mem.replaceScalar(u8, path, std.fs.path.sep, '/'); | |
| 1611 | try actual_files.append(testing.allocator, path); | |
| 1612 | } | |
| 1613 | mem.sortUnstable([]u8, actual_files.items, {}, struct { | |
| 1614 | fn lessThan(_: void, a: []u8, b: []u8) bool { | |
| 1615 | return mem.lessThan(u8, a, b); | |
| 1616 | } | |
| 1617 | }.lessThan); | |
| 1618 | try testing.expectEqualDeep(expected_files, actual_files.items); | |
| 1619 | ||
| 1620 | const expected_file_contents = | |
| 1621 | \\revision 1 | |
| 1622 | \\revision 2 | |
| 1623 | \\revision 4 | |
| 1624 | \\revision 5 | |
| 1625 | \\revision 7 | |
| 1626 | \\revision 8 | |
| 1627 | \\revision 9 | |
| 1628 | \\revision 10 | |
| 1629 | \\revision 12 | |
| 1630 | \\revision 13 | |
| 1631 | \\revision 14 | |
| 1632 | \\revision 18 | |
| 1633 | \\revision 19 | |
| 1634 | \\ | |
| 1635 | ; | |
| 1636 | const actual_file_contents = try worktree.dir.readFileAlloc(testing.allocator, "file", max_file_size); | |
| 1637 | defer testing.allocator.free(actual_file_contents); | |
| 1638 | try testing.expectEqualStrings(expected_file_contents, actual_file_contents); | |
| 1639 | } | |
| 1640 | ||
| 1641 | test "SHA-1 packfile indexing and checkout" { | |
| 1642 | try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb"); | |
| 1643 | } | |
| 1644 | ||
| 1645 | test "SHA-256 packfile indexing and checkout" { | |
| 1646 | try runRepositoryTest(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"); | |
| 1647 | } | |
| 1648 | ||
| 1649 | /// Checks out a commit of a packfile. Intended for experimenting with and | |
| 1650 | /// benchmarking possible optimizations to the indexing and checkout behavior. | |
| 1651 | pub fn main() !void { | |
| 1652 | const allocator = std.heap.c_allocator; | |
| 1653 | ||
| 1654 | const args = try std.process.argsAlloc(allocator); | |
| 1655 | defer std.process.argsFree(allocator, args); | |
| 1656 | if (args.len != 5) { | |
| 1657 | return error.InvalidArguments; // Arguments: format packfile commit worktree | |
| 1658 | } | |
| 1659 | ||
| 1660 | const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat; | |
| 1661 | ||
| 1662 | var pack_file = try std.fs.cwd().openFile(args[2], .{}); | |
| 1663 | defer pack_file.close(); | |
| 1664 | const commit = try Oid.parse(format, args[3]); | |
| 1665 | var worktree = try std.fs.cwd().makeOpenPath(args[4], .{}); | |
| 1666 | defer worktree.close(); | |
| 1667 | ||
| 1668 | var git_dir = try worktree.makeOpenPath(".git", .{}); | |
| 1669 | defer git_dir.close(); | |
| 1670 | ||
| 1671 | std.debug.print("Starting index...\n", .{}); | |
| 1672 | var index_file = try git_dir.createFile("idx", .{ .read = true }); | |
| 1673 | defer index_file.close(); | |
| 1674 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); | |
| 1675 | try indexPack(allocator, format, pack_file, index_buffered_writer.writer()); | |
| 1676 | try index_buffered_writer.flush(); | |
| 1677 | try index_file.sync(); | |
| 1678 | ||
| 1679 | std.debug.print("Starting checkout...\n", .{}); | |
| 1680 | var repository = try Repository.init(allocator, format, pack_file, index_file); | |
| 1681 | defer repository.deinit(); | |
| 1682 | var diagnostics: Diagnostics = .{ .allocator = allocator }; | |
| 1683 | defer diagnostics.deinit(); | |
| 1684 | try repository.checkout(worktree, commit, &diagnostics); | |
| 1685 | ||
| 1686 | for (diagnostics.errors.items) |err| { | |
| 1687 | std.debug.print("Diagnostic: {}\n", .{err}); | |
| 1688 | } | |
| 1689 | } |
src/Package/Fetch/git/testdata/testrepo-sha1.idx deleted
| Binary files a/src/Package/Fetch/git/testdata/testrepo-sha1.idx and /dev/null differ |
src/Package/Fetch/git/testdata/testrepo-sha1.pack deleted
| Binary files a/src/Package/Fetch/git/testdata/testrepo-sha1.pack and /dev/null differ |
src/Package/Fetch/git/testdata/testrepo-sha256.idx deleted
| Binary files a/src/Package/Fetch/git/testdata/testrepo-sha256.idx and /dev/null differ |
src/Package/Fetch/git/testdata/testrepo-sha256.pack deleted
| Binary files a/src/Package/Fetch/git/testdata/testrepo-sha256.pack and /dev/null differ |
src/Package/Fetch/testdata/duplicate_paths.tar.gz deleted
| Binary files a/src/Package/Fetch/testdata/duplicate_paths.tar.gz and /dev/null differ |
src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz deleted
| Binary files a/src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz and /dev/null differ |
src/Package/Fetch/testdata/executables.tar.gz deleted
| Binary files a/src/Package/Fetch/testdata/executables.tar.gz and /dev/null differ |
src/Package/Fetch/testdata/no_root.tar.gz deleted
| Binary files a/src/Package/Fetch/testdata/no_root.tar.gz and /dev/null differ |
src/Package/Manifest.zig deleted-704
| ... | ... | @@ -1,704 +0,0 @@ |
| 1 | const Manifest = @This(); | |
| 2 | const std = @import("std"); | |
| 3 | const mem = std.mem; | |
| 4 | const Allocator = std.mem.Allocator; | |
| 5 | const assert = std.debug.assert; | |
| 6 | const Ast = std.zig.Ast; | |
| 7 | const testing = std.testing; | |
| 8 | const Package = @import("../Package.zig"); | |
| 9 | ||
| 10 | pub const max_bytes = 10 * 1024 * 1024; | |
| 11 | pub const basename = "build.zig.zon"; | |
| 12 | pub const max_name_len = 32; | |
| 13 | pub const max_version_len = 32; | |
| 14 | ||
| 15 | pub const Dependency = struct { | |
| 16 | location: Location, | |
| 17 | location_tok: Ast.TokenIndex, | |
| 18 | location_node: Ast.Node.Index, | |
| 19 | hash: ?[]const u8, | |
| 20 | hash_tok: Ast.OptionalTokenIndex, | |
| 21 | hash_node: Ast.Node.OptionalIndex, | |
| 22 | node: Ast.Node.Index, | |
| 23 | name_tok: Ast.TokenIndex, | |
| 24 | lazy: bool, | |
| 25 | ||
| 26 | pub const Location = union(enum) { | |
| 27 | url: []const u8, | |
| 28 | path: []const u8, | |
| 29 | }; | |
| 30 | }; | |
| 31 | ||
| 32 | pub const ErrorMessage = struct { | |
| 33 | msg: []const u8, | |
| 34 | tok: Ast.TokenIndex, | |
| 35 | off: u32, | |
| 36 | }; | |
| 37 | ||
| 38 | name: []const u8, | |
| 39 | id: u32, | |
| 40 | version: std.SemanticVersion, | |
| 41 | version_node: Ast.Node.Index, | |
| 42 | dependencies: std.StringArrayHashMapUnmanaged(Dependency), | |
| 43 | dependencies_node: Ast.Node.OptionalIndex, | |
| 44 | paths: std.StringArrayHashMapUnmanaged(void), | |
| 45 | minimum_zig_version: ?std.SemanticVersion, | |
| 46 | ||
| 47 | errors: []ErrorMessage, | |
| 48 | arena_state: std.heap.ArenaAllocator.State, | |
| 49 | ||
| 50 | pub const ParseOptions = struct { | |
| 51 | allow_missing_paths_field: bool = false, | |
| 52 | /// Deprecated, to be removed after 0.14.0 is tagged. | |
| 53 | allow_name_string: bool = true, | |
| 54 | /// Deprecated, to be removed after 0.14.0 is tagged. | |
| 55 | allow_missing_fingerprint: bool = true, | |
| 56 | }; | |
| 57 | ||
| 58 | pub const Error = Allocator.Error; | |
| 59 | ||
| 60 | pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest { | |
| 61 | const main_node_index = ast.nodeData(.root).node; | |
| 62 | ||
| 63 | var arena_instance = std.heap.ArenaAllocator.init(gpa); | |
| 64 | errdefer arena_instance.deinit(); | |
| 65 | ||
| 66 | var p: Parse = .{ | |
| 67 | .gpa = gpa, | |
| 68 | .ast = ast, | |
| 69 | .arena = arena_instance.allocator(), | |
| 70 | .errors = .{}, | |
| 71 | ||
| 72 | .name = undefined, | |
| 73 | .id = 0, | |
| 74 | .version = undefined, | |
| 75 | .version_node = undefined, | |
| 76 | .dependencies = .{}, | |
| 77 | .dependencies_node = .none, | |
| 78 | .paths = .{}, | |
| 79 | .allow_missing_paths_field = options.allow_missing_paths_field, | |
| 80 | .allow_name_string = options.allow_name_string, | |
| 81 | .allow_missing_fingerprint = options.allow_missing_fingerprint, | |
| 82 | .minimum_zig_version = null, | |
| 83 | .buf = .{}, | |
| 84 | }; | |
| 85 | defer p.buf.deinit(gpa); | |
| 86 | defer p.errors.deinit(gpa); | |
| 87 | defer p.dependencies.deinit(gpa); | |
| 88 | defer p.paths.deinit(gpa); | |
| 89 | ||
| 90 | p.parseRoot(main_node_index) catch |err| switch (err) { | |
| 91 | error.ParseFailure => assert(p.errors.items.len > 0), | |
| 92 | else => |e| return e, | |
| 93 | }; | |
| 94 | ||
| 95 | return .{ | |
| 96 | .name = p.name, | |
| 97 | .id = p.id, | |
| 98 | .version = p.version, | |
| 99 | .version_node = p.version_node, | |
| 100 | .dependencies = try p.dependencies.clone(p.arena), | |
| 101 | .dependencies_node = p.dependencies_node, | |
| 102 | .paths = try p.paths.clone(p.arena), | |
| 103 | .minimum_zig_version = p.minimum_zig_version, | |
| 104 | .errors = try p.arena.dupe(ErrorMessage, p.errors.items), | |
| 105 | .arena_state = arena_instance.state, | |
| 106 | }; | |
| 107 | } | |
| 108 | ||
| 109 | pub fn deinit(man: *Manifest, gpa: Allocator) void { | |
| 110 | man.arena_state.promote(gpa).deinit(); | |
| 111 | man.* = undefined; | |
| 112 | } | |
| 113 | ||
| 114 | pub fn copyErrorsIntoBundle( | |
| 115 | man: Manifest, | |
| 116 | ast: Ast, | |
| 117 | /// ErrorBundle null-terminated string index | |
| 118 | src_path: u32, | |
| 119 | eb: *std.zig.ErrorBundle.Wip, | |
| 120 | ) Allocator.Error!void { | |
| 121 | for (man.errors) |msg| { | |
| 122 | const start_loc = ast.tokenLocation(0, msg.tok); | |
| 123 | ||
| 124 | try eb.addRootErrorMessage(.{ | |
| 125 | .msg = try eb.addString(msg.msg), | |
| 126 | .src_loc = try eb.addSourceLocation(.{ | |
| 127 | .src_path = src_path, | |
| 128 | .span_start = ast.tokenStart(msg.tok), | |
| 129 | .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), | |
| 130 | .span_main = ast.tokenStart(msg.tok) + msg.off, | |
| 131 | .line = @intCast(start_loc.line), | |
| 132 | .column = @intCast(start_loc.column), | |
| 133 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | |
| 134 | }), | |
| 135 | }); | |
| 136 | } | |
| 137 | } | |
| 138 | ||
| 139 | const Parse = struct { | |
| 140 | gpa: Allocator, | |
| 141 | ast: Ast, | |
| 142 | arena: Allocator, | |
| 143 | buf: std.ArrayListUnmanaged(u8), | |
| 144 | errors: std.ArrayListUnmanaged(ErrorMessage), | |
| 145 | ||
| 146 | name: []const u8, | |
| 147 | id: u32, | |
| 148 | version: std.SemanticVersion, | |
| 149 | version_node: Ast.Node.Index, | |
| 150 | dependencies: std.StringArrayHashMapUnmanaged(Dependency), | |
| 151 | dependencies_node: Ast.Node.OptionalIndex, | |
| 152 | paths: std.StringArrayHashMapUnmanaged(void), | |
| 153 | allow_missing_paths_field: bool, | |
| 154 | allow_name_string: bool, | |
| 155 | allow_missing_fingerprint: bool, | |
| 156 | minimum_zig_version: ?std.SemanticVersion, | |
| 157 | ||
| 158 | const InnerError = error{ ParseFailure, OutOfMemory }; | |
| 159 | ||
| 160 | fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { | |
| 161 | const ast = p.ast; | |
| 162 | const main_token = ast.nodeMainToken(node); | |
| 163 | ||
| 164 | var buf: [2]Ast.Node.Index = undefined; | |
| 165 | const struct_init = ast.fullStructInit(&buf, node) orelse { | |
| 166 | return fail(p, main_token, "expected top level expression to be a struct", .{}); | |
| 167 | }; | |
| 168 | ||
| 169 | var have_name = false; | |
| 170 | var have_version = false; | |
| 171 | var have_included_paths = false; | |
| 172 | var fingerprint: ?Package.Fingerprint = null; | |
| 173 | ||
| 174 | for (struct_init.ast.fields) |field_init| { | |
| 175 | const name_token = ast.firstToken(field_init) - 2; | |
| 176 | const field_name = try identifierTokenString(p, name_token); | |
| 177 | // We could get fancy with reflection and comptime logic here but doing | |
| 178 | // things manually provides an opportunity to do any additional verification | |
| 179 | // that is desirable on a per-field basis. | |
| 180 | if (mem.eql(u8, field_name, "dependencies")) { | |
| 181 | p.dependencies_node = field_init.toOptional(); | |
| 182 | try parseDependencies(p, field_init); | |
| 183 | } else if (mem.eql(u8, field_name, "paths")) { | |
| 184 | have_included_paths = true; | |
| 185 | try parseIncludedPaths(p, field_init); | |
| 186 | } else if (mem.eql(u8, field_name, "name")) { | |
| 187 | p.name = try parseName(p, field_init); | |
| 188 | have_name = true; | |
| 189 | } else if (mem.eql(u8, field_name, "fingerprint")) { | |
| 190 | fingerprint = try parseFingerprint(p, field_init); | |
| 191 | } else if (mem.eql(u8, field_name, "version")) { | |
| 192 | p.version_node = field_init; | |
| 193 | const version_text = try parseString(p, field_init); | |
| 194 | if (version_text.len > max_version_len) { | |
| 195 | try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); | |
| 196 | } | |
| 197 | p.version = std.SemanticVersion.parse(version_text) catch |err| v: { | |
| 198 | try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); | |
| 199 | break :v undefined; | |
| 200 | }; | |
| 201 | have_version = true; | |
| 202 | } else if (mem.eql(u8, field_name, "minimum_zig_version")) { | |
| 203 | const version_text = try parseString(p, field_init); | |
| 204 | p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { | |
| 205 | try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); | |
| 206 | break :v null; | |
| 207 | }; | |
| 208 | } else { | |
| 209 | // Ignore unknown fields so that we can add fields in future zig | |
| 210 | // versions without breaking older zig versions. | |
| 211 | } | |
| 212 | } | |
| 213 | ||
| 214 | if (!have_name) { | |
| 215 | try appendError(p, main_token, "missing top-level 'name' field", .{}); | |
| 216 | } else { | |
| 217 | if (fingerprint) |n| { | |
| 218 | if (!n.validate(p.name)) { | |
| 219 | return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ | |
| 220 | n.int(), Package.Fingerprint.generate(p.name).int(), | |
| 221 | }); | |
| 222 | } | |
| 223 | p.id = n.id; | |
| 224 | } else if (!p.allow_missing_fingerprint) { | |
| 225 | try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ | |
| 226 | Package.Fingerprint.generate(p.name).int(), | |
| 227 | }); | |
| 228 | } else { | |
| 229 | p.id = 0; | |
| 230 | } | |
| 231 | } | |
| 232 | ||
| 233 | if (!have_version) { | |
| 234 | try appendError(p, main_token, "missing top-level 'version' field", .{}); | |
| 235 | } | |
| 236 | ||
| 237 | if (!have_included_paths) { | |
| 238 | if (p.allow_missing_paths_field) { | |
| 239 | try p.paths.put(p.gpa, "", {}); | |
| 240 | } else { | |
| 241 | try appendError(p, main_token, "missing top-level 'paths' field", .{}); | |
| 242 | } | |
| 243 | } | |
| 244 | } | |
| 245 | ||
| 246 | fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { | |
| 247 | const ast = p.ast; | |
| 248 | ||
| 249 | var buf: [2]Ast.Node.Index = undefined; | |
| 250 | const struct_init = ast.fullStructInit(&buf, node) orelse { | |
| 251 | const tok = ast.nodeMainToken(node); | |
| 252 | return fail(p, tok, "expected dependencies expression to be a struct", .{}); | |
| 253 | }; | |
| 254 | ||
| 255 | for (struct_init.ast.fields) |field_init| { | |
| 256 | const name_token = ast.firstToken(field_init) - 2; | |
| 257 | const dep_name = try identifierTokenString(p, name_token); | |
| 258 | const dep = try parseDependency(p, field_init); | |
| 259 | try p.dependencies.put(p.gpa, dep_name, dep); | |
| 260 | } | |
| 261 | } | |
| 262 | ||
| 263 | fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { | |
| 264 | const ast = p.ast; | |
| 265 | ||
| 266 | var buf: [2]Ast.Node.Index = undefined; | |
| 267 | const struct_init = ast.fullStructInit(&buf, node) orelse { | |
| 268 | const tok = ast.nodeMainToken(node); | |
| 269 | return fail(p, tok, "expected dependency expression to be a struct", .{}); | |
| 270 | }; | |
| 271 | ||
| 272 | var dep: Dependency = .{ | |
| 273 | .location = undefined, | |
| 274 | .location_tok = undefined, | |
| 275 | .location_node = undefined, | |
| 276 | .hash = null, | |
| 277 | .hash_tok = .none, | |
| 278 | .hash_node = .none, | |
| 279 | .node = node, | |
| 280 | .name_tok = undefined, | |
| 281 | .lazy = false, | |
| 282 | }; | |
| 283 | var has_location = false; | |
| 284 | ||
| 285 | for (struct_init.ast.fields) |field_init| { | |
| 286 | const name_token = ast.firstToken(field_init) - 2; | |
| 287 | dep.name_tok = name_token; | |
| 288 | const field_name = try identifierTokenString(p, name_token); | |
| 289 | // We could get fancy with reflection and comptime logic here but doing | |
| 290 | // things manually provides an opportunity to do any additional verification | |
| 291 | // that is desirable on a per-field basis. | |
| 292 | if (mem.eql(u8, field_name, "url")) { | |
| 293 | if (has_location) { | |
| 294 | return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); | |
| 295 | } | |
| 296 | dep.location = .{ | |
| 297 | .url = parseString(p, field_init) catch |err| switch (err) { | |
| 298 | error.ParseFailure => continue, | |
| 299 | else => |e| return e, | |
| 300 | }, | |
| 301 | }; | |
| 302 | has_location = true; | |
| 303 | dep.location_tok = ast.nodeMainToken(field_init); | |
| 304 | dep.location_node = field_init; | |
| 305 | } else if (mem.eql(u8, field_name, "path")) { | |
| 306 | if (has_location) { | |
| 307 | return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); | |
| 308 | } | |
| 309 | dep.location = .{ | |
| 310 | .path = parseString(p, field_init) catch |err| switch (err) { | |
| 311 | error.ParseFailure => continue, | |
| 312 | else => |e| return e, | |
| 313 | }, | |
| 314 | }; | |
| 315 | has_location = true; | |
| 316 | dep.location_tok = ast.nodeMainToken(field_init); | |
| 317 | dep.location_node = field_init; | |
| 318 | } else if (mem.eql(u8, field_name, "hash")) { | |
| 319 | dep.hash = parseHash(p, field_init) catch |err| switch (err) { | |
| 320 | error.ParseFailure => continue, | |
| 321 | else => |e| return e, | |
| 322 | }; | |
| 323 | dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); | |
| 324 | dep.hash_node = field_init.toOptional(); | |
| 325 | } else if (mem.eql(u8, field_name, "lazy")) { | |
| 326 | dep.lazy = parseBool(p, field_init) catch |err| switch (err) { | |
| 327 | error.ParseFailure => continue, | |
| 328 | else => |e| return e, | |
| 329 | }; | |
| 330 | } else { | |
| 331 | // Ignore unknown fields so that we can add fields in future zig | |
| 332 | // versions without breaking older zig versions. | |
| 333 | } | |
| 334 | } | |
| 335 | ||
| 336 | if (!has_location) { | |
| 337 | try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); | |
| 338 | } | |
| 339 | ||
| 340 | return dep; | |
| 341 | } | |
| 342 | ||
| 343 | fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { | |
| 344 | const ast = p.ast; | |
| 345 | ||
| 346 | var buf: [2]Ast.Node.Index = undefined; | |
| 347 | const array_init = ast.fullArrayInit(&buf, node) orelse { | |
| 348 | const tok = ast.nodeMainToken(node); | |
| 349 | return fail(p, tok, "expected paths expression to be a list of strings", .{}); | |
| 350 | }; | |
| 351 | ||
| 352 | for (array_init.ast.elements) |elem_node| { | |
| 353 | const path_string = try parseString(p, elem_node); | |
| 354 | // This is normalized so that it can be used in string comparisons | |
| 355 | // against file system paths. | |
| 356 | const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); | |
| 357 | try p.paths.put(p.gpa, normalized, {}); | |
| 358 | } | |
| 359 | } | |
| 360 | ||
| 361 | fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { | |
| 362 | const ast = p.ast; | |
| 363 | if (ast.nodeTag(node) != .identifier) { | |
| 364 | return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); | |
| 365 | } | |
| 366 | const ident_token = ast.nodeMainToken(node); | |
| 367 | const token_bytes = ast.tokenSlice(ident_token); | |
| 368 | if (mem.eql(u8, token_bytes, "true")) { | |
| 369 | return true; | |
| 370 | } else if (mem.eql(u8, token_bytes, "false")) { | |
| 371 | return false; | |
| 372 | } else { | |
| 373 | return fail(p, ident_token, "expected boolean", .{}); | |
| 374 | } | |
| 375 | } | |
| 376 | ||
| 377 | fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { | |
| 378 | const ast = p.ast; | |
| 379 | const main_token = ast.nodeMainToken(node); | |
| 380 | if (ast.nodeTag(node) != .number_literal) { | |
| 381 | return fail(p, main_token, "expected integer literal", .{}); | |
| 382 | } | |
| 383 | const token_bytes = ast.tokenSlice(main_token); | |
| 384 | const parsed = std.zig.parseNumberLiteral(token_bytes); | |
| 385 | switch (parsed) { | |
| 386 | .int => |n| return @bitCast(n), | |
| 387 | .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ | |
| 388 | @tagName(parsed), | |
| 389 | }), | |
| 390 | .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), | |
| 391 | } | |
| 392 | } | |
| 393 | ||
| 394 | fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { | |
| 395 | const ast = p.ast; | |
| 396 | const main_token = ast.nodeMainToken(node); | |
| 397 | ||
| 398 | if (p.allow_name_string and ast.nodeTag(node) == .string_literal) { | |
| 399 | const name = try parseString(p, node); | |
| 400 | if (!std.zig.isValidId(name)) | |
| 401 | return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{}); | |
| 402 | ||
| 403 | if (name.len > max_name_len) | |
| 404 | return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ | |
| 405 | std.zig.fmtId(name), max_name_len, | |
| 406 | }); | |
| 407 | ||
| 408 | return name; | |
| 409 | } | |
| 410 | ||
| 411 | if (ast.nodeTag(node) != .enum_literal) | |
| 412 | return fail(p, main_token, "expected enum literal", .{}); | |
| 413 | ||
| 414 | const ident_name = ast.tokenSlice(main_token); | |
| 415 | if (mem.startsWith(u8, ident_name, "@")) | |
| 416 | return fail(p, main_token, "name must be a valid bare zig identifier", .{}); | |
| 417 | ||
| 418 | if (ident_name.len > max_name_len) | |
| 419 | return fail(p, main_token, "name '{}' exceeds max length of {d}", .{ | |
| 420 | std.zig.fmtId(ident_name), max_name_len, | |
| 421 | }); | |
| 422 | ||
| 423 | return ident_name; | |
| 424 | } | |
| 425 | ||
| 426 | fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { | |
| 427 | const ast = p.ast; | |
| 428 | if (ast.nodeTag(node) != .string_literal) { | |
| 429 | return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); | |
| 430 | } | |
| 431 | const str_lit_token = ast.nodeMainToken(node); | |
| 432 | const token_bytes = ast.tokenSlice(str_lit_token); | |
| 433 | p.buf.clearRetainingCapacity(); | |
| 434 | try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); | |
| 435 | const duped = try p.arena.dupe(u8, p.buf.items); | |
| 436 | return duped; | |
| 437 | } | |
| 438 | ||
| 439 | fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { | |
| 440 | const ast = p.ast; | |
| 441 | const tok = ast.nodeMainToken(node); | |
| 442 | const h = try parseString(p, node); | |
| 443 | ||
| 444 | if (h.len > Package.Hash.max_len) { | |
| 445 | return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len}); | |
| 446 | } | |
| 447 | ||
| 448 | return h; | |
| 449 | } | |
| 450 | ||
| 451 | /// TODO: try to DRY this with AstGen.identifierTokenString | |
| 452 | fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { | |
| 453 | const ast = p.ast; | |
| 454 | assert(ast.tokenTag(token) == .identifier); | |
| 455 | const ident_name = ast.tokenSlice(token); | |
| 456 | if (!mem.startsWith(u8, ident_name, "@")) { | |
| 457 | return ident_name; | |
| 458 | } | |
| 459 | p.buf.clearRetainingCapacity(); | |
| 460 | try parseStrLit(p, token, &p.buf, ident_name, 1); | |
| 461 | const duped = try p.arena.dupe(u8, p.buf.items); | |
| 462 | return duped; | |
| 463 | } | |
| 464 | ||
| 465 | /// TODO: try to DRY this with AstGen.parseStrLit | |
| 466 | fn parseStrLit( | |
| 467 | p: *Parse, | |
| 468 | token: Ast.TokenIndex, | |
| 469 | buf: *std.ArrayListUnmanaged(u8), | |
| 470 | bytes: []const u8, | |
| 471 | offset: u32, | |
| 472 | ) InnerError!void { | |
| 473 | const raw_string = bytes[offset..]; | |
| 474 | var buf_managed = buf.toManaged(p.gpa); | |
| 475 | const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); | |
| 476 | buf.* = buf_managed.moveToUnmanaged(); | |
| 477 | switch (try result) { | |
| 478 | .success => {}, | |
| 479 | .failure => |err| try p.appendStrLitError(err, token, bytes, offset), | |
| 480 | } | |
| 481 | } | |
| 482 | ||
| 483 | /// TODO: try to DRY this with AstGen.failWithStrLitError | |
| 484 | fn appendStrLitError( | |
| 485 | p: *Parse, | |
| 486 | err: std.zig.string_literal.Error, | |
| 487 | token: Ast.TokenIndex, | |
| 488 | bytes: []const u8, | |
| 489 | offset: u32, | |
| 490 | ) Allocator.Error!void { | |
| 491 | const raw_string = bytes[offset..]; | |
| 492 | switch (err) { | |
| 493 | .invalid_escape_character => |bad_index| { | |
| 494 | try p.appendErrorOff( | |
| 495 | token, | |
| 496 | offset + @as(u32, @intCast(bad_index)), | |
| 497 | "invalid escape character: '{c}'", | |
| 498 | .{raw_string[bad_index]}, | |
| 499 | ); | |
| 500 | }, | |
| 501 | .expected_hex_digit => |bad_index| { | |
| 502 | try p.appendErrorOff( | |
| 503 | token, | |
| 504 | offset + @as(u32, @intCast(bad_index)), | |
| 505 | "expected hex digit, found '{c}'", | |
| 506 | .{raw_string[bad_index]}, | |
| 507 | ); | |
| 508 | }, | |
| 509 | .empty_unicode_escape_sequence => |bad_index| { | |
| 510 | try p.appendErrorOff( | |
| 511 | token, | |
| 512 | offset + @as(u32, @intCast(bad_index)), | |
| 513 | "empty unicode escape sequence", | |
| 514 | .{}, | |
| 515 | ); | |
| 516 | }, | |
| 517 | .expected_hex_digit_or_rbrace => |bad_index| { | |
| 518 | try p.appendErrorOff( | |
| 519 | token, | |
| 520 | offset + @as(u32, @intCast(bad_index)), | |
| 521 | "expected hex digit or '}}', found '{c}'", | |
| 522 | .{raw_string[bad_index]}, | |
| 523 | ); | |
| 524 | }, | |
| 525 | .invalid_unicode_codepoint => |bad_index| { | |
| 526 | try p.appendErrorOff( | |
| 527 | token, | |
| 528 | offset + @as(u32, @intCast(bad_index)), | |
| 529 | "unicode escape does not correspond to a valid unicode scalar value", | |
| 530 | .{}, | |
| 531 | ); | |
| 532 | }, | |
| 533 | .expected_lbrace => |bad_index| { | |
| 534 | try p.appendErrorOff( | |
| 535 | token, | |
| 536 | offset + @as(u32, @intCast(bad_index)), | |
| 537 | "expected '{{', found '{c}", | |
| 538 | .{raw_string[bad_index]}, | |
| 539 | ); | |
| 540 | }, | |
| 541 | .expected_rbrace => |bad_index| { | |
| 542 | try p.appendErrorOff( | |
| 543 | token, | |
| 544 | offset + @as(u32, @intCast(bad_index)), | |
| 545 | "expected '}}', found '{c}", | |
| 546 | .{raw_string[bad_index]}, | |
| 547 | ); | |
| 548 | }, | |
| 549 | .expected_single_quote => |bad_index| { | |
| 550 | try p.appendErrorOff( | |
| 551 | token, | |
| 552 | offset + @as(u32, @intCast(bad_index)), | |
| 553 | "expected single quote ('), found '{c}", | |
| 554 | .{raw_string[bad_index]}, | |
| 555 | ); | |
| 556 | }, | |
| 557 | .invalid_character => |bad_index| { | |
| 558 | try p.appendErrorOff( | |
| 559 | token, | |
| 560 | offset + @as(u32, @intCast(bad_index)), | |
| 561 | "invalid byte in string or character literal: '{c}'", | |
| 562 | .{raw_string[bad_index]}, | |
| 563 | ); | |
| 564 | }, | |
| 565 | .empty_char_literal => { | |
| 566 | try p.appendErrorOff(token, offset, "empty character literal", .{}); | |
| 567 | }, | |
| 568 | } | |
| 569 | } | |
| 570 | ||
| 571 | fn fail( | |
| 572 | p: *Parse, | |
| 573 | tok: Ast.TokenIndex, | |
| 574 | comptime fmt: []const u8, | |
| 575 | args: anytype, | |
| 576 | ) InnerError { | |
| 577 | try appendError(p, tok, fmt, args); | |
| 578 | return error.ParseFailure; | |
| 579 | } | |
| 580 | ||
| 581 | fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { | |
| 582 | return appendErrorOff(p, tok, 0, fmt, args); | |
| 583 | } | |
| 584 | ||
| 585 | fn appendErrorOff( | |
| 586 | p: *Parse, | |
| 587 | tok: Ast.TokenIndex, | |
| 588 | byte_offset: u32, | |
| 589 | comptime fmt: []const u8, | |
| 590 | args: anytype, | |
| 591 | ) Allocator.Error!void { | |
| 592 | try p.errors.append(p.gpa, .{ | |
| 593 | .msg = try std.fmt.allocPrint(p.arena, fmt, args), | |
| 594 | .tok = tok, | |
| 595 | .off = byte_offset, | |
| 596 | }); | |
| 597 | } | |
| 598 | }; | |
| 599 | ||
| 600 | test "basic" { | |
| 601 | const gpa = testing.allocator; | |
| 602 | ||
| 603 | const example = | |
| 604 | \\.{ | |
| 605 | \\ .name = "foo", | |
| 606 | \\ .version = "3.2.1", | |
| 607 | \\ .paths = .{""}, | |
| 608 | \\ .dependencies = .{ | |
| 609 | \\ .bar = .{ | |
| 610 | \\ .url = "https://example.com/baz.tar.gz", | |
| 611 | \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", | |
| 612 | \\ }, | |
| 613 | \\ }, | |
| 614 | \\} | |
| 615 | ; | |
| 616 | ||
| 617 | var ast = try Ast.parse(gpa, example, .zon); | |
| 618 | defer ast.deinit(gpa); | |
| 619 | ||
| 620 | try testing.expect(ast.errors.len == 0); | |
| 621 | ||
| 622 | var manifest = try Manifest.parse(gpa, ast, .{}); | |
| 623 | defer manifest.deinit(gpa); | |
| 624 | ||
| 625 | try testing.expect(manifest.errors.len == 0); | |
| 626 | try testing.expectEqualStrings("foo", manifest.name); | |
| 627 | ||
| 628 | try testing.expectEqual(@as(std.SemanticVersion, .{ | |
| 629 | .major = 3, | |
| 630 | .minor = 2, | |
| 631 | .patch = 1, | |
| 632 | }), manifest.version); | |
| 633 | ||
| 634 | try testing.expect(manifest.dependencies.count() == 1); | |
| 635 | try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); | |
| 636 | try testing.expectEqualStrings( | |
| 637 | "https://example.com/baz.tar.gz", | |
| 638 | manifest.dependencies.values()[0].location.url, | |
| 639 | ); | |
| 640 | try testing.expectEqualStrings( | |
| 641 | "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", | |
| 642 | manifest.dependencies.values()[0].hash orelse return error.TestFailed, | |
| 643 | ); | |
| 644 | ||
| 645 | try testing.expect(manifest.minimum_zig_version == null); | |
| 646 | } | |
| 647 | ||
| 648 | test "minimum_zig_version" { | |
| 649 | const gpa = testing.allocator; | |
| 650 | ||
| 651 | const example = | |
| 652 | \\.{ | |
| 653 | \\ .name = "foo", | |
| 654 | \\ .version = "3.2.1", | |
| 655 | \\ .paths = .{""}, | |
| 656 | \\ .minimum_zig_version = "0.11.1", | |
| 657 | \\} | |
| 658 | ; | |
| 659 | ||
| 660 | var ast = try Ast.parse(gpa, example, .zon); | |
| 661 | defer ast.deinit(gpa); | |
| 662 | ||
| 663 | try testing.expect(ast.errors.len == 0); | |
| 664 | ||
| 665 | var manifest = try Manifest.parse(gpa, ast, .{}); | |
| 666 | defer manifest.deinit(gpa); | |
| 667 | ||
| 668 | try testing.expect(manifest.errors.len == 0); | |
| 669 | try testing.expect(manifest.dependencies.count() == 0); | |
| 670 | ||
| 671 | try testing.expect(manifest.minimum_zig_version != null); | |
| 672 | ||
| 673 | try testing.expectEqual(@as(std.SemanticVersion, .{ | |
| 674 | .major = 0, | |
| 675 | .minor = 11, | |
| 676 | .patch = 1, | |
| 677 | }), manifest.minimum_zig_version.?); | |
| 678 | } | |
| 679 | ||
| 680 | test "minimum_zig_version - invalid version" { | |
| 681 | const gpa = testing.allocator; | |
| 682 | ||
| 683 | const example = | |
| 684 | \\.{ | |
| 685 | \\ .name = "foo", | |
| 686 | \\ .version = "3.2.1", | |
| 687 | \\ .minimum_zig_version = "X.11.1", | |
| 688 | \\ .paths = .{""}, | |
| 689 | \\} | |
| 690 | ; | |
| 691 | ||
| 692 | var ast = try Ast.parse(gpa, example, .zon); | |
| 693 | defer ast.deinit(gpa); | |
| 694 | ||
| 695 | try testing.expect(ast.errors.len == 0); | |
| 696 | ||
| 697 | var manifest = try Manifest.parse(gpa, ast, .{}); | |
| 698 | defer manifest.deinit(gpa); | |
| 699 | ||
| 700 | try testing.expect(manifest.errors.len == 1); | |
| 701 | try testing.expect(manifest.dependencies.count() == 0); | |
| 702 | ||
| 703 | try testing.expect(manifest.minimum_zig_version == null); | |
| 704 | } |
src/Package/Module.zig deleted-563
| ... | ... | @@ -1,563 +0,0 @@ |
| 1 | //! Corresponds to something that Zig source code can `@import`. | |
| 2 | ||
| 3 | /// Only files inside this directory can be imported. | |
| 4 | root: Cache.Path, | |
| 5 | /// Relative to `root`. May contain path separators. | |
| 6 | root_src_path: []const u8, | |
| 7 | /// Name used in compile errors. Looks like "root.foo.bar". | |
| 8 | fully_qualified_name: []const u8, | |
| 9 | /// The dependency table of this module. Shared dependencies such as 'std', | |
| 10 | /// 'builtin', and 'root' are not specified in every dependency table, but | |
| 11 | /// instead only in the table of `main_mod`. `Module.importFile` is | |
| 12 | /// responsible for detecting these names and using the correct package. | |
| 13 | deps: Deps = .{}, | |
| 14 | ||
| 15 | resolved_target: ResolvedTarget, | |
| 16 | optimize_mode: std.builtin.OptimizeMode, | |
| 17 | code_model: std.builtin.CodeModel, | |
| 18 | single_threaded: bool, | |
| 19 | error_tracing: bool, | |
| 20 | valgrind: bool, | |
| 21 | pic: bool, | |
| 22 | strip: bool, | |
| 23 | omit_frame_pointer: bool, | |
| 24 | stack_check: bool, | |
| 25 | stack_protector: u32, | |
| 26 | red_zone: bool, | |
| 27 | sanitize_c: bool, | |
| 28 | sanitize_thread: bool, | |
| 29 | fuzz: bool, | |
| 30 | unwind_tables: std.builtin.UnwindTables, | |
| 31 | cc_argv: []const []const u8, | |
| 32 | /// (SPIR-V) whether to generate a structured control flow graph or not | |
| 33 | structured_cfg: bool, | |
| 34 | no_builtin: bool, | |
| 35 | ||
| 36 | /// If the module is an `@import("builtin")` module, this is the `File` that | |
| 37 | /// is preallocated for it. Otherwise this field is null. | |
| 38 | builtin_file: ?*File, | |
| 39 | ||
| 40 | pub const Deps = std.StringArrayHashMapUnmanaged(*Module); | |
| 41 | ||
| 42 | pub fn isBuiltin(m: Module) bool { | |
| 43 | return m.builtin_file != null; | |
| 44 | } | |
| 45 | ||
| 46 | pub const Tree = struct { | |
| 47 | /// Each `Package` exposes a `Module` with build.zig as its root source file. | |
| 48 | build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), | |
| 49 | }; | |
| 50 | ||
| 51 | pub const CreateOptions = struct { | |
| 52 | /// Where to store builtin.zig. The global cache directory is used because | |
| 53 | /// it is a pure function based on CLI flags. | |
| 54 | global_cache_directory: Cache.Directory, | |
| 55 | paths: Paths, | |
| 56 | fully_qualified_name: []const u8, | |
| 57 | ||
| 58 | cc_argv: []const []const u8, | |
| 59 | inherited: Inherited, | |
| 60 | global: Compilation.Config, | |
| 61 | /// If this is null then `resolved_target` must be non-null. | |
| 62 | parent: ?*Package.Module, | |
| 63 | ||
| 64 | builtin_mod: ?*Package.Module, | |
| 65 | ||
| 66 | /// Allocated into the given `arena`. Should be shared across all module creations in a Compilation. | |
| 67 | /// Ignored if `builtin_mod` is passed or if `!have_zcu`. | |
| 68 | /// Otherwise, may be `null` only if this Compilation consists of a single module. | |
| 69 | builtin_modules: ?*std.StringHashMapUnmanaged(*Module), | |
| 70 | ||
| 71 | pub const Paths = struct { | |
| 72 | root: Cache.Path, | |
| 73 | /// Relative to `root`. May contain path separators. | |
| 74 | root_src_path: []const u8, | |
| 75 | }; | |
| 76 | ||
| 77 | pub const Inherited = struct { | |
| 78 | /// If this is null then `parent` must be non-null. | |
| 79 | resolved_target: ?ResolvedTarget = null, | |
| 80 | optimize_mode: ?std.builtin.OptimizeMode = null, | |
| 81 | code_model: ?std.builtin.CodeModel = null, | |
| 82 | single_threaded: ?bool = null, | |
| 83 | error_tracing: ?bool = null, | |
| 84 | valgrind: ?bool = null, | |
| 85 | pic: ?bool = null, | |
| 86 | strip: ?bool = null, | |
| 87 | omit_frame_pointer: ?bool = null, | |
| 88 | stack_check: ?bool = null, | |
| 89 | /// null means default. | |
| 90 | /// 0 means no stack protector. | |
| 91 | /// other number means stack protection with that buffer size. | |
| 92 | stack_protector: ?u32 = null, | |
| 93 | red_zone: ?bool = null, | |
| 94 | unwind_tables: ?std.builtin.UnwindTables = null, | |
| 95 | sanitize_c: ?bool = null, | |
| 96 | sanitize_thread: ?bool = null, | |
| 97 | fuzz: ?bool = null, | |
| 98 | structured_cfg: ?bool = null, | |
| 99 | no_builtin: ?bool = null, | |
| 100 | }; | |
| 101 | }; | |
| 102 | ||
| 103 | pub const ResolvedTarget = struct { | |
| 104 | result: std.Target, | |
| 105 | is_native_os: bool, | |
| 106 | is_native_abi: bool, | |
| 107 | llvm_cpu_features: ?[*:0]const u8 = null, | |
| 108 | }; | |
| 109 | ||
| 110 | /// At least one of `parent` and `resolved_target` must be non-null. | |
| 111 | pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { | |
| 112 | if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); | |
| 113 | if (options.inherited.fuzz == true) assert(options.global.any_fuzz); | |
| 114 | if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); | |
| 115 | if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); | |
| 116 | if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); | |
| 117 | ||
| 118 | const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; | |
| 119 | const target = resolved_target.result; | |
| 120 | ||
| 121 | const optimize_mode = options.inherited.optimize_mode orelse | |
| 122 | if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; | |
| 123 | ||
| 124 | const strip = b: { | |
| 125 | if (options.inherited.strip) |x| break :b x; | |
| 126 | if (options.parent) |p| break :b p.strip; | |
| 127 | break :b options.global.root_strip; | |
| 128 | }; | |
| 129 | ||
| 130 | const valgrind = b: { | |
| 131 | if (!target_util.hasValgrindSupport(target)) { | |
| 132 | if (options.inherited.valgrind == true) | |
| 133 | return error.ValgrindUnsupportedOnTarget; | |
| 134 | break :b false; | |
| 135 | } | |
| 136 | if (options.inherited.valgrind) |x| break :b x; | |
| 137 | if (options.parent) |p| break :b p.valgrind; | |
| 138 | if (strip) break :b false; | |
| 139 | break :b optimize_mode == .Debug; | |
| 140 | }; | |
| 141 | ||
| 142 | const zig_backend = target_util.zigBackend(target, options.global.use_llvm); | |
| 143 | ||
| 144 | const single_threaded = b: { | |
| 145 | if (target_util.alwaysSingleThreaded(target)) { | |
| 146 | if (options.inherited.single_threaded == false) | |
| 147 | return error.TargetRequiresSingleThreaded; | |
| 148 | break :b true; | |
| 149 | } | |
| 150 | ||
| 151 | if (options.global.have_zcu) { | |
| 152 | if (!target_util.supportsThreads(target, zig_backend)) { | |
| 153 | if (options.inherited.single_threaded == false) | |
| 154 | return error.BackendRequiresSingleThreaded; | |
| 155 | break :b true; | |
| 156 | } | |
| 157 | } | |
| 158 | ||
| 159 | if (options.inherited.single_threaded) |x| break :b x; | |
| 160 | if (options.parent) |p| break :b p.single_threaded; | |
| 161 | break :b target_util.defaultSingleThreaded(target); | |
| 162 | }; | |
| 163 | ||
| 164 | const error_tracing = b: { | |
| 165 | if (options.inherited.error_tracing) |x| break :b x; | |
| 166 | if (options.parent) |p| break :b p.error_tracing; | |
| 167 | break :b options.global.root_error_tracing; | |
| 168 | }; | |
| 169 | ||
| 170 | const pic = b: { | |
| 171 | if (target_util.requiresPIC(target, options.global.link_libc)) { | |
| 172 | if (options.inherited.pic == false) | |
| 173 | return error.TargetRequiresPic; | |
| 174 | break :b true; | |
| 175 | } | |
| 176 | if (options.global.pie) { | |
| 177 | if (options.inherited.pic == false) | |
| 178 | return error.PieRequiresPic; | |
| 179 | break :b true; | |
| 180 | } | |
| 181 | if (options.global.link_mode == .dynamic) { | |
| 182 | if (options.inherited.pic == false) | |
| 183 | return error.DynamicLinkingRequiresPic; | |
| 184 | break :b true; | |
| 185 | } | |
| 186 | if (options.inherited.pic) |x| break :b x; | |
| 187 | if (options.parent) |p| break :b p.pic; | |
| 188 | break :b false; | |
| 189 | }; | |
| 190 | ||
| 191 | const red_zone = b: { | |
| 192 | if (!target_util.hasRedZone(target)) { | |
| 193 | if (options.inherited.red_zone == true) | |
| 194 | return error.TargetHasNoRedZone; | |
| 195 | break :b false; | |
| 196 | } | |
| 197 | if (options.inherited.red_zone) |x| break :b x; | |
| 198 | if (options.parent) |p| break :b p.red_zone; | |
| 199 | break :b true; | |
| 200 | }; | |
| 201 | ||
| 202 | const omit_frame_pointer = b: { | |
| 203 | if (options.inherited.omit_frame_pointer) |x| break :b x; | |
| 204 | if (options.parent) |p| break :b p.omit_frame_pointer; | |
| 205 | if (optimize_mode == .ReleaseSmall) { | |
| 206 | // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. | |
| 207 | // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) | |
| 208 | // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). | |
| 209 | break :b !target.cpu.arch.isX86(); | |
| 210 | } | |
| 211 | break :b false; | |
| 212 | }; | |
| 213 | ||
| 214 | const sanitize_thread = b: { | |
| 215 | if (options.inherited.sanitize_thread) |x| break :b x; | |
| 216 | if (options.parent) |p| break :b p.sanitize_thread; | |
| 217 | break :b false; | |
| 218 | }; | |
| 219 | ||
| 220 | const unwind_tables = b: { | |
| 221 | if (options.inherited.unwind_tables) |x| break :b x; | |
| 222 | if (options.parent) |p| break :b p.unwind_tables; | |
| 223 | ||
| 224 | break :b target_util.defaultUnwindTables( | |
| 225 | target, | |
| 226 | options.global.link_libunwind, | |
| 227 | sanitize_thread or options.global.any_sanitize_thread, | |
| 228 | ); | |
| 229 | }; | |
| 230 | ||
| 231 | const fuzz = b: { | |
| 232 | if (options.inherited.fuzz) |x| break :b x; | |
| 233 | if (options.parent) |p| break :b p.fuzz; | |
| 234 | break :b false; | |
| 235 | }; | |
| 236 | ||
| 237 | const code_model = b: { | |
| 238 | if (options.inherited.code_model) |x| break :b x; | |
| 239 | if (options.parent) |p| break :b p.code_model; | |
| 240 | break :b .default; | |
| 241 | }; | |
| 242 | ||
| 243 | const is_safe_mode = switch (optimize_mode) { | |
| 244 | .Debug, .ReleaseSafe => true, | |
| 245 | .ReleaseFast, .ReleaseSmall => false, | |
| 246 | }; | |
| 247 | ||
| 248 | const sanitize_c = b: { | |
| 249 | if (options.inherited.sanitize_c) |x| break :b x; | |
| 250 | if (options.parent) |p| break :b p.sanitize_c; | |
| 251 | break :b is_safe_mode; | |
| 252 | }; | |
| 253 | ||
| 254 | const stack_check = b: { | |
| 255 | if (!target_util.supportsStackProbing(target)) { | |
| 256 | if (options.inherited.stack_check == true) | |
| 257 | return error.StackCheckUnsupportedByTarget; | |
| 258 | break :b false; | |
| 259 | } | |
| 260 | if (options.inherited.stack_check) |x| break :b x; | |
| 261 | if (options.parent) |p| break :b p.stack_check; | |
| 262 | break :b is_safe_mode; | |
| 263 | }; | |
| 264 | ||
| 265 | const stack_protector: u32 = sp: { | |
| 266 | const use_zig_backend = options.global.have_zcu or | |
| 267 | (options.global.any_c_source_files and options.global.c_frontend == .aro); | |
| 268 | if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { | |
| 269 | if (options.inherited.stack_protector) |x| { | |
| 270 | if (x > 0) return error.StackProtectorUnsupportedByTarget; | |
| 271 | } | |
| 272 | break :sp 0; | |
| 273 | } | |
| 274 | ||
| 275 | if (options.global.any_c_source_files and options.global.c_frontend == .clang and | |
| 276 | !target_util.clangSupportsStackProtector(target)) | |
| 277 | { | |
| 278 | if (options.inherited.stack_protector) |x| { | |
| 279 | if (x > 0) return error.StackProtectorUnsupportedByTarget; | |
| 280 | } | |
| 281 | break :sp 0; | |
| 282 | } | |
| 283 | ||
| 284 | // This logic is checking for linking libc because otherwise our start code | |
| 285 | // which is trying to set up TLS (i.e. the fs/gs registers) but the stack | |
| 286 | // protection code depends on fs/gs registers being already set up. | |
| 287 | // If we were able to annotate start code, or perhaps the entire std lib, | |
| 288 | // as being exempt from stack protection checks, we could change this logic | |
| 289 | // to supporting stack protection even when not linking libc. | |
| 290 | // TODO file issue about this | |
| 291 | if (!options.global.link_libc) { | |
| 292 | if (options.inherited.stack_protector) |x| { | |
| 293 | if (x > 0) return error.StackProtectorUnavailableWithoutLibC; | |
| 294 | } | |
| 295 | break :sp 0; | |
| 296 | } | |
| 297 | ||
| 298 | if (options.inherited.stack_protector) |x| break :sp x; | |
| 299 | if (options.parent) |p| break :sp p.stack_protector; | |
| 300 | if (!is_safe_mode) break :sp 0; | |
| 301 | ||
| 302 | break :sp target_util.default_stack_protector_buffer_size; | |
| 303 | }; | |
| 304 | ||
| 305 | const structured_cfg = b: { | |
| 306 | if (options.inherited.structured_cfg) |x| break :b x; | |
| 307 | if (options.parent) |p| break :b p.structured_cfg; | |
| 308 | // We always want a structured control flow in shaders. This option is | |
| 309 | // only relevant for OpenCL kernels. | |
| 310 | break :b switch (target.os.tag) { | |
| 311 | .opencl => false, | |
| 312 | else => true, | |
| 313 | }; | |
| 314 | }; | |
| 315 | ||
| 316 | const no_builtin = b: { | |
| 317 | if (options.inherited.no_builtin) |x| break :b x; | |
| 318 | if (options.parent) |p| break :b p.no_builtin; | |
| 319 | ||
| 320 | break :b target.cpu.arch.isBpf(); | |
| 321 | }; | |
| 322 | ||
| 323 | const llvm_cpu_features: ?[*:0]const u8 = b: { | |
| 324 | if (resolved_target.llvm_cpu_features) |x| break :b x; | |
| 325 | if (!options.global.use_llvm) break :b null; | |
| 326 | ||
| 327 | var buf = std.ArrayList(u8).init(arena); | |
| 328 | var disabled_features = std.ArrayList(u8).init(arena); | |
| 329 | defer disabled_features.deinit(); | |
| 330 | ||
| 331 | // Append disabled features after enabled ones, so that their effects aren't overwritten. | |
| 332 | for (target.cpu.arch.allFeaturesList()) |feature| { | |
| 333 | if (feature.llvm_name) |llvm_name| { | |
| 334 | const is_enabled = target.cpu.features.isEnabled(feature.index); | |
| 335 | ||
| 336 | if (is_enabled) { | |
| 337 | try buf.ensureUnusedCapacity(2 + llvm_name.len); | |
| 338 | buf.appendAssumeCapacity('+'); | |
| 339 | buf.appendSliceAssumeCapacity(llvm_name); | |
| 340 | buf.appendAssumeCapacity(','); | |
| 341 | } else { | |
| 342 | try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); | |
| 343 | disabled_features.appendAssumeCapacity('-'); | |
| 344 | disabled_features.appendSliceAssumeCapacity(llvm_name); | |
| 345 | disabled_features.appendAssumeCapacity(','); | |
| 346 | } | |
| 347 | } | |
| 348 | } | |
| 349 | ||
| 350 | try buf.appendSlice(disabled_features.items); | |
| 351 | if (buf.items.len == 0) break :b ""; | |
| 352 | assert(std.mem.endsWith(u8, buf.items, ",")); | |
| 353 | buf.items[buf.items.len - 1] = 0; | |
| 354 | buf.shrinkAndFree(buf.items.len); | |
| 355 | break :b buf.items[0 .. buf.items.len - 1 :0].ptr; | |
| 356 | }; | |
| 357 | ||
| 358 | const mod = try arena.create(Module); | |
| 359 | mod.* = .{ | |
| 360 | .root = options.paths.root, | |
| 361 | .root_src_path = options.paths.root_src_path, | |
| 362 | .fully_qualified_name = options.fully_qualified_name, | |
| 363 | .resolved_target = .{ | |
| 364 | .result = target, | |
| 365 | .is_native_os = resolved_target.is_native_os, | |
| 366 | .is_native_abi = resolved_target.is_native_abi, | |
| 367 | .llvm_cpu_features = llvm_cpu_features, | |
| 368 | }, | |
| 369 | .optimize_mode = optimize_mode, | |
| 370 | .single_threaded = single_threaded, | |
| 371 | .error_tracing = error_tracing, | |
| 372 | .valgrind = valgrind, | |
| 373 | .pic = pic, | |
| 374 | .strip = strip, | |
| 375 | .omit_frame_pointer = omit_frame_pointer, | |
| 376 | .stack_check = stack_check, | |
| 377 | .stack_protector = stack_protector, | |
| 378 | .code_model = code_model, | |
| 379 | .red_zone = red_zone, | |
| 380 | .sanitize_c = sanitize_c, | |
| 381 | .sanitize_thread = sanitize_thread, | |
| 382 | .fuzz = fuzz, | |
| 383 | .unwind_tables = unwind_tables, | |
| 384 | .cc_argv = options.cc_argv, | |
| 385 | .structured_cfg = structured_cfg, | |
| 386 | .no_builtin = no_builtin, | |
| 387 | .builtin_file = null, | |
| 388 | }; | |
| 389 | ||
| 390 | const opt_builtin_mod = options.builtin_mod orelse b: { | |
| 391 | if (!options.global.have_zcu) break :b null; | |
| 392 | ||
| 393 | const generated_builtin_source = try Builtin.generate(.{ | |
| 394 | .target = target, | |
| 395 | .zig_backend = zig_backend, | |
| 396 | .output_mode = options.global.output_mode, | |
| 397 | .link_mode = options.global.link_mode, | |
| 398 | .unwind_tables = unwind_tables, | |
| 399 | .is_test = options.global.is_test, | |
| 400 | .single_threaded = single_threaded, | |
| 401 | .link_libc = options.global.link_libc, | |
| 402 | .link_libcpp = options.global.link_libcpp, | |
| 403 | .optimize_mode = optimize_mode, | |
| 404 | .error_tracing = error_tracing, | |
| 405 | .valgrind = valgrind, | |
| 406 | .sanitize_thread = sanitize_thread, | |
| 407 | .fuzz = fuzz, | |
| 408 | .pic = pic, | |
| 409 | .pie = options.global.pie, | |
| 410 | .strip = strip, | |
| 411 | .code_model = code_model, | |
| 412 | .omit_frame_pointer = omit_frame_pointer, | |
| 413 | .wasi_exec_model = options.global.wasi_exec_model, | |
| 414 | }, arena); | |
| 415 | ||
| 416 | const new = if (options.builtin_modules) |builtins| new: { | |
| 417 | const gop = try builtins.getOrPut(arena, generated_builtin_source); | |
| 418 | if (gop.found_existing) break :b gop.value_ptr.*; | |
| 419 | errdefer builtins.removeByPtr(gop.key_ptr); | |
| 420 | const new = try arena.create(Module); | |
| 421 | gop.value_ptr.* = new; | |
| 422 | break :new new; | |
| 423 | } else try arena.create(Module); | |
| 424 | errdefer if (options.builtin_modules) |builtins| assert(builtins.remove(generated_builtin_source)); | |
| 425 | ||
| 426 | const new_file = try arena.create(File); | |
| 427 | ||
| 428 | const hex_digest = digest: { | |
| 429 | var hasher: Cache.Hasher = Cache.hasher_init; | |
| 430 | hasher.update(generated_builtin_source); | |
| 431 | ||
| 432 | var bin_digest: Cache.BinDigest = undefined; | |
| 433 | hasher.final(&bin_digest); | |
| 434 | ||
| 435 | var hex_digest: Cache.HexDigest = undefined; | |
| 436 | _ = std.fmt.bufPrint( | |
| 437 | &hex_digest, | |
| 438 | "{s}", | |
| 439 | .{std.fmt.fmtSliceHexLower(&bin_digest)}, | |
| 440 | ) catch unreachable; | |
| 441 | ||
| 442 | break :digest hex_digest; | |
| 443 | }; | |
| 444 | ||
| 445 | const builtin_sub_path = try arena.dupe(u8, "b" ++ std.fs.path.sep_str ++ hex_digest); | |
| 446 | ||
| 447 | new.* = .{ | |
| 448 | .root = .{ | |
| 449 | .root_dir = options.global_cache_directory, | |
| 450 | .sub_path = builtin_sub_path, | |
| 451 | }, | |
| 452 | .root_src_path = "builtin.zig", | |
| 453 | .fully_qualified_name = if (options.parent == null) | |
| 454 | "builtin" | |
| 455 | else | |
| 456 | try std.fmt.allocPrint(arena, "{s}.builtin", .{options.fully_qualified_name}), | |
| 457 | .resolved_target = .{ | |
| 458 | .result = target, | |
| 459 | .is_native_os = resolved_target.is_native_os, | |
| 460 | .is_native_abi = resolved_target.is_native_abi, | |
| 461 | .llvm_cpu_features = llvm_cpu_features, | |
| 462 | }, | |
| 463 | .optimize_mode = optimize_mode, | |
| 464 | .single_threaded = single_threaded, | |
| 465 | .error_tracing = error_tracing, | |
| 466 | .valgrind = valgrind, | |
| 467 | .pic = pic, | |
| 468 | .strip = strip, | |
| 469 | .omit_frame_pointer = omit_frame_pointer, | |
| 470 | .stack_check = stack_check, | |
| 471 | .stack_protector = stack_protector, | |
| 472 | .code_model = code_model, | |
| 473 | .red_zone = red_zone, | |
| 474 | .sanitize_c = sanitize_c, | |
| 475 | .sanitize_thread = sanitize_thread, | |
| 476 | .fuzz = fuzz, | |
| 477 | .unwind_tables = unwind_tables, | |
| 478 | .cc_argv = &.{}, | |
| 479 | .structured_cfg = structured_cfg, | |
| 480 | .no_builtin = no_builtin, | |
| 481 | .builtin_file = new_file, | |
| 482 | }; | |
| 483 | new_file.* = .{ | |
| 484 | .sub_file_path = "builtin.zig", | |
| 485 | .stat = undefined, | |
| 486 | .source = generated_builtin_source, | |
| 487 | .tree = null, | |
| 488 | .zir = null, | |
| 489 | .zoir = null, | |
| 490 | .status = .never_loaded, | |
| 491 | .mod = new, | |
| 492 | }; | |
| 493 | break :b new; | |
| 494 | }; | |
| 495 | ||
| 496 | if (opt_builtin_mod) |builtin_mod| { | |
| 497 | try mod.deps.ensureUnusedCapacity(arena, 1); | |
| 498 | mod.deps.putAssumeCapacityNoClobber("builtin", builtin_mod); | |
| 499 | } | |
| 500 | ||
| 501 | return mod; | |
| 502 | } | |
| 503 | ||
| 504 | /// All fields correspond to `CreateOptions`. | |
| 505 | pub const LimitedOptions = struct { | |
| 506 | root: Cache.Path, | |
| 507 | root_src_path: []const u8, | |
| 508 | fully_qualified_name: []const u8, | |
| 509 | }; | |
| 510 | ||
| 511 | /// This one can only be used if the Module will only be used for AstGen and earlier in | |
| 512 | /// the pipeline. Illegal behavior occurs if a limited module touches Sema. | |
| 513 | pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { | |
| 514 | const mod = try gpa.create(Module); | |
| 515 | mod.* = .{ | |
| 516 | .root = options.root, | |
| 517 | .root_src_path = options.root_src_path, | |
| 518 | .fully_qualified_name = options.fully_qualified_name, | |
| 519 | ||
| 520 | .resolved_target = undefined, | |
| 521 | .optimize_mode = undefined, | |
| 522 | .code_model = undefined, | |
| 523 | .single_threaded = undefined, | |
| 524 | .error_tracing = undefined, | |
| 525 | .valgrind = undefined, | |
| 526 | .pic = undefined, | |
| 527 | .strip = undefined, | |
| 528 | .omit_frame_pointer = undefined, | |
| 529 | .stack_check = undefined, | |
| 530 | .stack_protector = undefined, | |
| 531 | .red_zone = undefined, | |
| 532 | .sanitize_c = undefined, | |
| 533 | .sanitize_thread = undefined, | |
| 534 | .fuzz = undefined, | |
| 535 | .unwind_tables = undefined, | |
| 536 | .cc_argv = undefined, | |
| 537 | .structured_cfg = undefined, | |
| 538 | .no_builtin = undefined, | |
| 539 | .builtin_file = null, | |
| 540 | }; | |
| 541 | return mod; | |
| 542 | } | |
| 543 | ||
| 544 | /// Asserts that the module has a builtin module, which is not true for non-zig | |
| 545 | /// modules such as ones only used for `@embedFile`, or the root module when | |
| 546 | /// there is no Zig Compilation Unit. | |
| 547 | pub fn getBuiltinDependency(m: Module) *Module { | |
| 548 | const result = m.deps.values()[0]; | |
| 549 | assert(result.isBuiltin()); | |
| 550 | return result; | |
| 551 | } | |
| 552 | ||
| 553 | const Module = @This(); | |
| 554 | const Package = @import("../Package.zig"); | |
| 555 | const std = @import("std"); | |
| 556 | const Allocator = std.mem.Allocator; | |
| 557 | const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; | |
| 558 | const target_util = @import("../target.zig"); | |
| 559 | const Cache = std.Build.Cache; | |
| 560 | const Builtin = @import("../Builtin.zig"); | |
| 561 | const assert = std.debug.assert; | |
| 562 | const Compilation = @import("../Compilation.zig"); | |
| 563 | const File = @import("../Zcu.zig").File; |
src/dev.zig-5
| ... | ... | @@ -54,7 +54,6 @@ pub const Env = enum { |
| 54 | 54 | .test_command, |
| 55 | 55 | .run_command, |
| 56 | 56 | .ar_command, |
| 57 | .build_command, | |
| 58 | 57 | .clang_command, |
| 59 | 58 | .stdio_listen, |
| 60 | 59 | .build_import_lib, |
| ... | ... | @@ -87,7 +86,6 @@ pub const Env = enum { |
| 87 | 86 | .translate_c_command, |
| 88 | 87 | .fmt_command, |
| 89 | 88 | .jit_command, |
| 90 | .fetch_command, | |
| 91 | 89 | .init_command, |
| 92 | 90 | .targets_command, |
| 93 | 91 | .version_command, |
| ... | ... | @@ -135,7 +133,6 @@ pub const Env = enum { |
| 135 | 133 | else => Env.ast_gen.supports(feature), |
| 136 | 134 | }, |
| 137 | 135 | .@"x86_64-linux" => switch (feature) { |
| 138 | .build_command, | |
| 139 | 136 | .stdio_listen, |
| 140 | 137 | .incremental, |
| 141 | 138 | .x86_64_backend, |
| ... | ... | @@ -178,13 +175,11 @@ pub const Feature = enum { |
| 178 | 175 | test_command, |
| 179 | 176 | run_command, |
| 180 | 177 | ar_command, |
| 181 | build_command, | |
| 182 | 178 | clang_command, |
| 183 | 179 | cc_command, |
| 184 | 180 | translate_c_command, |
| 185 | 181 | fmt_command, |
| 186 | 182 | jit_command, |
| 187 | fetch_command, | |
| 188 | 183 | init_command, |
| 189 | 184 | targets_command, |
| 190 | 185 | version_command, |
src/main.zig+70-1331
| ... | ... | @@ -1,5 +1,7 @@ |
| 1 | const std = @import("std"); | |
| 2 | 1 | const builtin = @import("builtin"); |
| 2 | const native_os = builtin.os.tag; | |
| 3 | ||
| 4 | const std = @import("std"); | |
| 3 | 5 | const assert = std.debug.assert; |
| 4 | 6 | const io = std.io; |
| 5 | 7 | const fs = std.fs; |
| ... | ... | @@ -12,7 +14,6 @@ const Color = std.zig.Color; |
| 12 | 14 | const warn = std.log.warn; |
| 13 | 15 | const ThreadPool = std.Thread.Pool; |
| 14 | 16 | const cleanExit = std.process.cleanExit; |
| 15 | const native_os = builtin.os.tag; | |
| 16 | 17 | const Cache = std.Build.Cache; |
| 17 | 18 | const Path = std.Build.Cache.Path; |
| 18 | 19 | const Directory = std.Build.Cache.Directory; |
| ... | ... | @@ -34,6 +35,7 @@ const crash_report = @import("crash_report.zig"); |
| 34 | 35 | const Zcu = @import("Zcu.zig"); |
| 35 | 36 | const mingw = @import("mingw.zig"); |
| 36 | 37 | const dev = @import("dev.zig"); |
| 38 | const Module = @import("Module.zig"); | |
| 37 | 39 | |
| 38 | 40 | test { |
| 39 | 41 | _ = Package; |
| ... | ... | @@ -289,8 +291,14 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 289 | 291 | dev.check(.ar_command); |
| 290 | 292 | return process.exit(try llvmArMain(arena, args)); |
| 291 | 293 | } else if (mem.eql(u8, cmd, "build")) { |
| 292 | dev.check(.build_command); | |
| 293 | return cmdBuild(gpa, arena, cmd_args); | |
| 294 | return jitCmd(gpa, arena, cmd_args, .{ | |
| 295 | .cmd_name = "build", | |
| 296 | .root_src_path = "build.zig", | |
| 297 | .prepend_zig_lib_dir_path = true, | |
| 298 | .prepend_global_cache_path = true, | |
| 299 | .prepend_zig_exe_path = true, | |
| 300 | .optimize_mode = .ReleaseSafe, // Sprinkle some safety on the networking code. | |
| 301 | }); | |
| 294 | 302 | } else if (mem.eql(u8, cmd, "clang") or |
| 295 | 303 | mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) |
| 296 | 304 | { |
| ... | ... | @@ -329,7 +337,13 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 329 | 337 | .root_src_path = "objcopy.zig", |
| 330 | 338 | }); |
| 331 | 339 | } else if (mem.eql(u8, cmd, "fetch")) { |
| 332 | return cmdFetch(gpa, arena, cmd_args); | |
| 340 | return jitCmd(gpa, arena, cmd_args, .{ | |
| 341 | .cmd_name = "fetch", | |
| 342 | .root_src_path = "fetch.zig", | |
| 343 | .prepend_global_cache_path = true, | |
| 344 | .prepend_zig_lib_dir_path = true, | |
| 345 | .optimize_mode = .ReleaseSafe, // Sprinkle some safety on the networking code. | |
| 346 | }); | |
| 333 | 347 | } else if (mem.eql(u8, cmd, "libc")) { |
| 334 | 348 | return jitCmd(gpa, arena, cmd_args, .{ |
| 335 | 349 | .cmd_name = "libc", |
| ... | ... | @@ -790,14 +804,14 @@ const Framework = struct { |
| 790 | 804 | }; |
| 791 | 805 | |
| 792 | 806 | const CliModule = struct { |
| 793 | paths: Package.Module.CreateOptions.Paths, | |
| 807 | paths: Module.CreateOptions.Paths, | |
| 794 | 808 | cc_argv: []const []const u8, |
| 795 | inherited: Package.Module.CreateOptions.Inherited, | |
| 809 | inherited: Module.CreateOptions.Inherited, | |
| 796 | 810 | target_arch_os_abi: ?[]const u8, |
| 797 | 811 | target_mcpu: ?[]const u8, |
| 798 | 812 | |
| 799 | 813 | deps: []const Dep, |
| 800 | resolved: ?*Package.Module, | |
| 814 | resolved: ?*Module, | |
| 801 | 815 | |
| 802 | 816 | c_source_files_start: usize, |
| 803 | 817 | c_source_files_end: usize, |
| ... | ... | @@ -944,7 +958,7 @@ fn buildOutputType( |
| 944 | 958 | |
| 945 | 959 | // These get set by CLI flags and then snapshotted when a `-M` flag is |
| 946 | 960 | // encountered. |
| 947 | var mod_opts: Package.Module.CreateOptions.Inherited = .{}; | |
| 961 | var mod_opts: Module.CreateOptions.Inherited = .{}; | |
| 948 | 962 | |
| 949 | 963 | // These get appended to by CLI flags and then slurped when a `-M` flag |
| 950 | 964 | // is encountered. |
| ... | ... | @@ -2991,7 +3005,7 @@ fn buildOutputType( |
| 2991 | 3005 | create_module.opts.emit_bin = emit_bin != .no; |
| 2992 | 3006 | create_module.opts.any_c_source_files = create_module.c_source_files.items.len != 0; |
| 2993 | 3007 | |
| 2994 | var builtin_modules: std.StringHashMapUnmanaged(*Package.Module) = .empty; | |
| 3008 | var builtin_modules: std.StringHashMapUnmanaged(*Module) = .empty; | |
| 2995 | 3009 | // `builtin_modules` allocated into `arena`, so no deinit |
| 2996 | 3010 | const main_mod = try createModule(gpa, arena, &create_module, 0, null, zig_lib_directory, &builtin_modules, color); |
| 2997 | 3011 | for (create_module.modules.keys(), create_module.modules.values()) |key, cli_mod| { |
| ... | ... | @@ -3023,7 +3037,7 @@ fn buildOutputType( |
| 3023 | 3037 | |
| 3024 | 3038 | const root_mod = if (arg_mode == .zig_test) root_mod: { |
| 3025 | 3039 | const test_mod = if (test_runner_path) |test_runner| test_mod: { |
| 3026 | const test_mod = try Package.Module.create(arena, .{ | |
| 3040 | const test_mod = try Module.create(arena, .{ | |
| 3027 | 3041 | .global_cache_directory = global_cache_directory, |
| 3028 | 3042 | .paths = .{ |
| 3029 | 3043 | .root = .{ |
| ... | ... | @@ -3042,7 +3056,7 @@ fn buildOutputType( |
| 3042 | 3056 | }); |
| 3043 | 3057 | test_mod.deps = try main_mod.deps.clone(arena); |
| 3044 | 3058 | break :test_mod test_mod; |
| 3045 | } else try Package.Module.create(arena, .{ | |
| 3059 | } else try Module.create(arena, .{ | |
| 3046 | 3060 | .global_cache_directory = global_cache_directory, |
| 3047 | 3061 | .paths = .{ |
| 3048 | 3062 | .root = .{ |
| ... | ... | @@ -3824,11 +3838,11 @@ fn createModule( |
| 3824 | 3838 | arena: Allocator, |
| 3825 | 3839 | create_module: *CreateModule, |
| 3826 | 3840 | index: usize, |
| 3827 | parent: ?*Package.Module, | |
| 3841 | parent: ?*Module, | |
| 3828 | 3842 | zig_lib_directory: Cache.Directory, |
| 3829 | builtin_modules: *std.StringHashMapUnmanaged(*Package.Module), | |
| 3843 | builtin_modules: *std.StringHashMapUnmanaged(*Module), | |
| 3830 | 3844 | color: std.zig.Color, |
| 3831 | ) Allocator.Error!*Package.Module { | |
| 3845 | ) Allocator.Error!*Module { | |
| 3832 | 3846 | const cli_mod = &create_module.modules.values()[index]; |
| 3833 | 3847 | if (cli_mod.resolved) |m| return m; |
| 3834 | 3848 | |
| ... | ... | @@ -4114,7 +4128,7 @@ fn createModule( |
| 4114 | 4128 | }; |
| 4115 | 4129 | } |
| 4116 | 4130 | |
| 4117 | const mod = Package.Module.create(arena, .{ | |
| 4131 | const mod = Module.create(arena, .{ | |
| 4118 | 4132 | .global_cache_directory = create_module.global_cache_directory, |
| 4119 | 4133 | .paths = cli_mod.paths, |
| 4120 | 4134 | .fully_qualified_name = name, |
| ... | ... | @@ -4740,12 +4754,19 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 4740 | 4754 | } |
| 4741 | 4755 | } |
| 4742 | 4756 | |
| 4743 | var templates = findTemplates(gpa, arena); | |
| 4744 | defer templates.deinit(); | |
| 4757 | const self_exe_path = introspect.findZigExePath(arena) catch |err| { | |
| 4758 | fatal("unable to find self exe path: {s}", .{@errorName(err)}); | |
| 4759 | }; | |
| 4760 | const zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { | |
| 4761 | fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) }); | |
| 4762 | }; | |
| 4763 | ||
| 4764 | var templates = std.zig.Package.Templates.find(gpa, zig_lib_directory); | |
| 4765 | defer templates.deinit(gpa); | |
| 4745 | 4766 | |
| 4746 | 4767 | const cwd_path = try process.getCwdAlloc(arena); |
| 4747 | 4768 | const cwd_basename = fs.path.basename(cwd_path); |
| 4748 | const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename); | |
| 4769 | const sanitized_root_name = try Package.sanitizeExampleName(arena, cwd_basename); | |
| 4749 | 4770 | |
| 4750 | 4771 | const s = fs.path.sep_str; |
| 4751 | 4772 | const template_paths = [_][]const u8{ |
| ... | ... | @@ -4757,9 +4778,10 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 4757 | 4778 | var ok_count: usize = 0; |
| 4758 | 4779 | |
| 4759 | 4780 | const fingerprint: Package.Fingerprint = .generate(sanitized_root_name); |
| 4781 | const zig_ver = build_options.version; | |
| 4760 | 4782 | |
| 4761 | 4783 | for (template_paths) |template_path| { |
| 4762 | if (templates.write(arena, fs.cwd(), sanitized_root_name, template_path, fingerprint)) |_| { | |
| 4784 | if (templates.write(arena, fs.cwd(), sanitized_root_name, template_path, fingerprint, zig_ver)) |_| { | |
| 4763 | 4785 | std.log.info("created {s}", .{template_path}); |
| 4764 | 4786 | ok_count += 1; |
| 4765 | 4787 | } else |err| switch (err) { |
| ... | ... | @@ -4776,700 +4798,6 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { |
| 4776 | 4798 | return cleanExit(); |
| 4777 | 4799 | } |
| 4778 | 4800 | |
| 4779 | fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 { | |
| 4780 | var result: std.ArrayListUnmanaged(u8) = .empty; | |
| 4781 | for (bytes, 0..) |byte, i| switch (byte) { | |
| 4782 | '0'...'9' => { | |
| 4783 | if (i == 0) try result.append(arena, '_'); | |
| 4784 | try result.append(arena, byte); | |
| 4785 | }, | |
| 4786 | '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte), | |
| 4787 | '-', '.', ' ' => try result.append(arena, '_'), | |
| 4788 | else => continue, | |
| 4789 | }; | |
| 4790 | if (!std.zig.isValidId(result.items)) return "foo"; | |
| 4791 | if (result.items.len > Package.Manifest.max_name_len) | |
| 4792 | result.shrinkRetainingCapacity(Package.Manifest.max_name_len); | |
| 4793 | ||
| 4794 | return result.toOwnedSlice(arena); | |
| 4795 | } | |
| 4796 | ||
| 4797 | test sanitizeExampleName { | |
| 4798 | var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); | |
| 4799 | defer arena_instance.deinit(); | |
| 4800 | const arena = arena_instance.allocator(); | |
| 4801 | ||
| 4802 | try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+")); | |
| 4803 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "")); | |
| 4804 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!")); | |
| 4805 | try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a")); | |
| 4806 | try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!")); | |
| 4807 | try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234")); | |
| 4808 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error")); | |
| 4809 | try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test")); | |
| 4810 | try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests")); | |
| 4811 | try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); | |
| 4812 | } | |
| 4813 | ||
| 4814 | fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { | |
| 4815 | dev.check(.build_command); | |
| 4816 | ||
| 4817 | var build_file: ?[]const u8 = null; | |
| 4818 | var override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena); | |
| 4819 | var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); | |
| 4820 | var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena); | |
| 4821 | var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena); | |
| 4822 | var child_argv = std.ArrayList([]const u8).init(arena); | |
| 4823 | var reference_trace: ?u32 = null; | |
| 4824 | var debug_compile_errors = false; | |
| 4825 | var verbose_link = (native_os != .wasi or builtin.link_libc) and | |
| 4826 | EnvVar.ZIG_VERBOSE_LINK.isSet(); | |
| 4827 | var verbose_cc = (native_os != .wasi or builtin.link_libc) and | |
| 4828 | EnvVar.ZIG_VERBOSE_CC.isSet(); | |
| 4829 | var verbose_air = false; | |
| 4830 | var verbose_intern_pool = false; | |
| 4831 | var verbose_generic_instances = false; | |
| 4832 | var verbose_llvm_ir: ?[]const u8 = null; | |
| 4833 | var verbose_llvm_bc: ?[]const u8 = null; | |
| 4834 | var verbose_cimport = false; | |
| 4835 | var verbose_llvm_cpu_features = false; | |
| 4836 | var fetch_only = false; | |
| 4837 | var system_pkg_dir_path: ?[]const u8 = null; | |
| 4838 | var debug_target: ?[]const u8 = null; | |
| 4839 | ||
| 4840 | const argv_index_exe = child_argv.items.len; | |
| 4841 | _ = try child_argv.addOne(); | |
| 4842 | ||
| 4843 | const self_exe_path = try introspect.findZigExePath(arena); | |
| 4844 | try child_argv.append(self_exe_path); | |
| 4845 | ||
| 4846 | const argv_index_zig_lib_dir = child_argv.items.len; | |
| 4847 | _ = try child_argv.addOne(); | |
| 4848 | ||
| 4849 | const argv_index_build_file = child_argv.items.len; | |
| 4850 | _ = try child_argv.addOne(); | |
| 4851 | ||
| 4852 | const argv_index_cache_dir = child_argv.items.len; | |
| 4853 | _ = try child_argv.addOne(); | |
| 4854 | ||
| 4855 | const argv_index_global_cache_dir = child_argv.items.len; | |
| 4856 | _ = try child_argv.addOne(); | |
| 4857 | ||
| 4858 | try child_argv.appendSlice(&.{ | |
| 4859 | "--seed", | |
| 4860 | try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}), | |
| 4861 | }); | |
| 4862 | const argv_index_seed = child_argv.items.len - 1; | |
| 4863 | ||
| 4864 | // This parent process needs a way to obtain results from the configuration | |
| 4865 | // phase of the child process. In the future, the make phase will be | |
| 4866 | // executed in a separate process than the configure phase, and we can then | |
| 4867 | // use stdout from the configuration phase for this purpose. | |
| 4868 | // | |
| 4869 | // However, currently, both phases are in the same process, and Run Step | |
| 4870 | // provides API for making the runned subprocesses inherit stdout and stderr | |
| 4871 | // which means these streams are not available for passing metadata back | |
| 4872 | // to the parent. | |
| 4873 | // | |
| 4874 | // Until make and configure phases are separated into different processes, | |
| 4875 | // the strategy is to choose a temporary file name ahead of time, and then | |
| 4876 | // read this file in the parent to obtain the results, in the case the child | |
| 4877 | // exits with code 3. | |
| 4878 | const results_tmp_file_nonce = std.fmt.hex(std.crypto.random.int(u64)); | |
| 4879 | try child_argv.append("-Z" ++ results_tmp_file_nonce); | |
| 4880 | ||
| 4881 | var color: Color = .auto; | |
| 4882 | var n_jobs: ?u32 = null; | |
| 4883 | ||
| 4884 | { | |
| 4885 | var i: usize = 0; | |
| 4886 | while (i < args.len) : (i += 1) { | |
| 4887 | const arg = args[i]; | |
| 4888 | if (mem.startsWith(u8, arg, "-")) { | |
| 4889 | if (mem.eql(u8, arg, "--build-file")) { | |
| 4890 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4891 | i += 1; | |
| 4892 | build_file = args[i]; | |
| 4893 | continue; | |
| 4894 | } else if (mem.eql(u8, arg, "--zig-lib-dir")) { | |
| 4895 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4896 | i += 1; | |
| 4897 | override_lib_dir = args[i]; | |
| 4898 | continue; | |
| 4899 | } else if (mem.eql(u8, arg, "--build-runner")) { | |
| 4900 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4901 | i += 1; | |
| 4902 | override_build_runner = args[i]; | |
| 4903 | continue; | |
| 4904 | } else if (mem.eql(u8, arg, "--cache-dir")) { | |
| 4905 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4906 | i += 1; | |
| 4907 | override_local_cache_dir = args[i]; | |
| 4908 | continue; | |
| 4909 | } else if (mem.eql(u8, arg, "--global-cache-dir")) { | |
| 4910 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4911 | i += 1; | |
| 4912 | override_global_cache_dir = args[i]; | |
| 4913 | continue; | |
| 4914 | } else if (mem.eql(u8, arg, "-freference-trace")) { | |
| 4915 | reference_trace = 256; | |
| 4916 | } else if (mem.eql(u8, arg, "--fetch")) { | |
| 4917 | fetch_only = true; | |
| 4918 | } else if (mem.eql(u8, arg, "--system")) { | |
| 4919 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4920 | i += 1; | |
| 4921 | system_pkg_dir_path = args[i]; | |
| 4922 | try child_argv.append("--system"); | |
| 4923 | continue; | |
| 4924 | } else if (mem.startsWith(u8, arg, "-freference-trace=")) { | |
| 4925 | const num = arg["-freference-trace=".len..]; | |
| 4926 | reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { | |
| 4927 | fatal("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); | |
| 4928 | }; | |
| 4929 | } else if (mem.eql(u8, arg, "-fno-reference-trace")) { | |
| 4930 | reference_trace = null; | |
| 4931 | } else if (mem.eql(u8, arg, "--debug-log")) { | |
| 4932 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4933 | try child_argv.appendSlice(args[i .. i + 2]); | |
| 4934 | i += 1; | |
| 4935 | if (!build_options.enable_logging) { | |
| 4936 | warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{}); | |
| 4937 | } else { | |
| 4938 | try log_scopes.append(arena, args[i]); | |
| 4939 | } | |
| 4940 | continue; | |
| 4941 | } else if (mem.eql(u8, arg, "--debug-compile-errors")) { | |
| 4942 | if (build_options.enable_debug_extensions) { | |
| 4943 | debug_compile_errors = true; | |
| 4944 | } else { | |
| 4945 | warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{}); | |
| 4946 | } | |
| 4947 | } else if (mem.eql(u8, arg, "--debug-target")) { | |
| 4948 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4949 | i += 1; | |
| 4950 | if (build_options.enable_debug_extensions) { | |
| 4951 | debug_target = args[i]; | |
| 4952 | } else { | |
| 4953 | warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{}); | |
| 4954 | } | |
| 4955 | } else if (mem.eql(u8, arg, "--verbose-link")) { | |
| 4956 | verbose_link = true; | |
| 4957 | } else if (mem.eql(u8, arg, "--verbose-cc")) { | |
| 4958 | verbose_cc = true; | |
| 4959 | } else if (mem.eql(u8, arg, "--verbose-air")) { | |
| 4960 | verbose_air = true; | |
| 4961 | } else if (mem.eql(u8, arg, "--verbose-intern-pool")) { | |
| 4962 | verbose_intern_pool = true; | |
| 4963 | } else if (mem.eql(u8, arg, "--verbose-generic-instances")) { | |
| 4964 | verbose_generic_instances = true; | |
| 4965 | } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { | |
| 4966 | verbose_llvm_ir = "-"; | |
| 4967 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { | |
| 4968 | verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; | |
| 4969 | } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { | |
| 4970 | verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; | |
| 4971 | } else if (mem.eql(u8, arg, "--verbose-cimport")) { | |
| 4972 | verbose_cimport = true; | |
| 4973 | } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { | |
| 4974 | verbose_llvm_cpu_features = true; | |
| 4975 | } else if (mem.eql(u8, arg, "--color")) { | |
| 4976 | if (i + 1 >= args.len) fatal("expected [auto|on|off] after {s}", .{arg}); | |
| 4977 | i += 1; | |
| 4978 | color = std.meta.stringToEnum(Color, args[i]) orelse { | |
| 4979 | fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] }); | |
| 4980 | }; | |
| 4981 | try child_argv.appendSlice(&.{ arg, args[i] }); | |
| 4982 | continue; | |
| 4983 | } else if (mem.startsWith(u8, arg, "-j")) { | |
| 4984 | const str = arg["-j".len..]; | |
| 4985 | const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| { | |
| 4986 | fatal("unable to parse jobs count '{s}': {s}", .{ | |
| 4987 | str, @errorName(err), | |
| 4988 | }); | |
| 4989 | }; | |
| 4990 | if (num < 1) { | |
| 4991 | fatal("number of jobs must be at least 1\n", .{}); | |
| 4992 | } | |
| 4993 | n_jobs = num; | |
| 4994 | } else if (mem.eql(u8, arg, "--seed")) { | |
| 4995 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 4996 | i += 1; | |
| 4997 | child_argv.items[argv_index_seed] = args[i]; | |
| 4998 | continue; | |
| 4999 | } else if (mem.eql(u8, arg, "--")) { | |
| 5000 | // The rest of the args are supposed to get passed onto | |
| 5001 | // build runner's `build.args` | |
| 5002 | try child_argv.appendSlice(args[i..]); | |
| 5003 | break; | |
| 5004 | } | |
| 5005 | } | |
| 5006 | try child_argv.append(arg); | |
| 5007 | } | |
| 5008 | } | |
| 5009 | ||
| 5010 | const work_around_btrfs_bug = native_os == .linux and | |
| 5011 | EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); | |
| 5012 | const root_prog_node = std.Progress.start(.{ | |
| 5013 | .disable_printing = (color == .off), | |
| 5014 | .root_name = "Compile Build Script", | |
| 5015 | }); | |
| 5016 | defer root_prog_node.end(); | |
| 5017 | ||
| 5018 | // Normally the build runner is compiled for the host target but here is | |
| 5019 | // some code to help when debugging edits to the build runner so that you | |
| 5020 | // can make sure it compiles successfully on other targets. | |
| 5021 | const resolved_target: Package.Module.ResolvedTarget = t: { | |
| 5022 | if (build_options.enable_debug_extensions) { | |
| 5023 | if (debug_target) |triple| { | |
| 5024 | const target_query = try std.Target.Query.parse(.{ | |
| 5025 | .arch_os_abi = triple, | |
| 5026 | }); | |
| 5027 | break :t .{ | |
| 5028 | .result = std.zig.resolveTargetQueryOrFatal(target_query), | |
| 5029 | .is_native_os = false, | |
| 5030 | .is_native_abi = false, | |
| 5031 | }; | |
| 5032 | } | |
| 5033 | } | |
| 5034 | break :t .{ | |
| 5035 | .result = std.zig.resolveTargetQueryOrFatal(.{}), | |
| 5036 | .is_native_os = true, | |
| 5037 | .is_native_abi = true, | |
| 5038 | }; | |
| 5039 | }; | |
| 5040 | ||
| 5041 | const exe_basename = try std.zig.binNameAlloc(arena, .{ | |
| 5042 | .root_name = "build", | |
| 5043 | .target = resolved_target.result, | |
| 5044 | .output_mode = .Exe, | |
| 5045 | }); | |
| 5046 | const emit_bin: Compilation.EmitLoc = .{ | |
| 5047 | .directory = null, // Use the local zig-cache. | |
| 5048 | .basename = exe_basename, | |
| 5049 | }; | |
| 5050 | ||
| 5051 | process.raiseFileDescriptorLimit(); | |
| 5052 | ||
| 5053 | var zig_lib_directory: Directory = if (override_lib_dir) |lib_dir| .{ | |
| 5054 | .path = lib_dir, | |
| 5055 | .handle = fs.cwd().openDir(lib_dir, .{}) catch |err| { | |
| 5056 | fatal("unable to open zig lib directory from 'zig-lib-dir' argument: '{s}': {s}", .{ lib_dir, @errorName(err) }); | |
| 5057 | }, | |
| 5058 | } else introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { | |
| 5059 | fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) }); | |
| 5060 | }; | |
| 5061 | defer zig_lib_directory.handle.close(); | |
| 5062 | ||
| 5063 | const cwd_path = try process.getCwdAlloc(arena); | |
| 5064 | child_argv.items[argv_index_zig_lib_dir] = zig_lib_directory.path orelse cwd_path; | |
| 5065 | ||
| 5066 | const build_root = try findBuildRoot(arena, .{ | |
| 5067 | .cwd_path = cwd_path, | |
| 5068 | .build_file = build_file, | |
| 5069 | }); | |
| 5070 | child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; | |
| 5071 | ||
| 5072 | var global_cache_directory: Directory = l: { | |
| 5073 | const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); | |
| 5074 | const dir = fs.cwd().makeOpenPath(p, .{}) catch |err| { | |
| 5075 | const base_msg = "unable to open or create the global Zig cache at '{s}': {s}.{s}"; | |
| 5076 | const extra = "\nIf this location is not writable then consider specifying an " ++ | |
| 5077 | "alternative with the ZIG_GLOBAL_CACHE_DIR environment variable or the " ++ | |
| 5078 | "--global-cache-dir option."; | |
| 5079 | const show_extra = err == error.AccessDenied or err == error.ReadOnlyFileSystem; | |
| 5080 | fatal(base_msg, .{ p, @errorName(err), if (show_extra) extra else "" }); | |
| 5081 | }; | |
| 5082 | break :l .{ | |
| 5083 | .handle = dir, | |
| 5084 | .path = p, | |
| 5085 | }; | |
| 5086 | }; | |
| 5087 | defer global_cache_directory.handle.close(); | |
| 5088 | ||
| 5089 | child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path; | |
| 5090 | ||
| 5091 | var local_cache_directory: Directory = l: { | |
| 5092 | if (override_local_cache_dir) |local_cache_dir_path| { | |
| 5093 | break :l .{ | |
| 5094 | .handle = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}), | |
| 5095 | .path = local_cache_dir_path, | |
| 5096 | }; | |
| 5097 | } | |
| 5098 | const cache_dir_path = try build_root.directory.join(arena, &.{default_local_zig_cache_basename}); | |
| 5099 | break :l .{ | |
| 5100 | .handle = try build_root.directory.handle.makeOpenPath(default_local_zig_cache_basename, .{}), | |
| 5101 | .path = cache_dir_path, | |
| 5102 | }; | |
| 5103 | }; | |
| 5104 | defer local_cache_directory.handle.close(); | |
| 5105 | ||
| 5106 | child_argv.items[argv_index_cache_dir] = local_cache_directory.path orelse cwd_path; | |
| 5107 | ||
| 5108 | var thread_pool: ThreadPool = undefined; | |
| 5109 | try thread_pool.init(.{ | |
| 5110 | .allocator = gpa, | |
| 5111 | .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)), | |
| 5112 | .track_ids = true, | |
| 5113 | .stack_size = thread_stack_size, | |
| 5114 | }); | |
| 5115 | defer thread_pool.deinit(); | |
| 5116 | ||
| 5117 | // Dummy http client that is not actually used when fetch_command is unsupported. | |
| 5118 | // Prevents bootstrap from depending on a bunch of unnecessary stuff. | |
| 5119 | var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { | |
| 5120 | allocator: Allocator, | |
| 5121 | fn deinit(_: @This()) void {} | |
| 5122 | } = .{ .allocator = gpa }; | |
| 5123 | defer http_client.deinit(); | |
| 5124 | ||
| 5125 | var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; | |
| 5126 | ||
| 5127 | // This loop is re-evaluated when the build script exits with an indication that it | |
| 5128 | // could not continue due to missing lazy dependencies. | |
| 5129 | while (true) { | |
| 5130 | // We want to release all the locks before executing the child process, so we make a nice | |
| 5131 | // big block here to ensure the cleanup gets run when we extract out our argv. | |
| 5132 | { | |
| 5133 | const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{ | |
| 5134 | .root = .{ | |
| 5135 | .root_dir = Cache.Directory.cwd(), | |
| 5136 | .sub_path = fs.path.dirname(runner) orelse "", | |
| 5137 | }, | |
| 5138 | .root_src_path = fs.path.basename(runner), | |
| 5139 | } else .{ | |
| 5140 | .root = .{ | |
| 5141 | .root_dir = zig_lib_directory, | |
| 5142 | .sub_path = "compiler", | |
| 5143 | }, | |
| 5144 | .root_src_path = "build_runner.zig", | |
| 5145 | }; | |
| 5146 | ||
| 5147 | const config = try Compilation.Config.resolve(.{ | |
| 5148 | .output_mode = .Exe, | |
| 5149 | .resolved_target = resolved_target, | |
| 5150 | .have_zcu = true, | |
| 5151 | .emit_bin = true, | |
| 5152 | .is_test = false, | |
| 5153 | }); | |
| 5154 | ||
| 5155 | const root_mod = try Package.Module.create(arena, .{ | |
| 5156 | .global_cache_directory = global_cache_directory, | |
| 5157 | .paths = main_mod_paths, | |
| 5158 | .fully_qualified_name = "root", | |
| 5159 | .cc_argv = &.{}, | |
| 5160 | .inherited = .{ | |
| 5161 | .resolved_target = resolved_target, | |
| 5162 | }, | |
| 5163 | .global = config, | |
| 5164 | .parent = null, | |
| 5165 | .builtin_mod = null, | |
| 5166 | .builtin_modules = null, // all modules will inherit this one's builtin | |
| 5167 | }); | |
| 5168 | ||
| 5169 | const builtin_mod = root_mod.getBuiltinDependency(); | |
| 5170 | ||
| 5171 | const build_mod = try Package.Module.create(arena, .{ | |
| 5172 | .global_cache_directory = global_cache_directory, | |
| 5173 | .paths = .{ | |
| 5174 | .root = .{ .root_dir = build_root.directory }, | |
| 5175 | .root_src_path = build_root.build_zig_basename, | |
| 5176 | }, | |
| 5177 | .fully_qualified_name = "root.@build", | |
| 5178 | .cc_argv = &.{}, | |
| 5179 | .inherited = .{}, | |
| 5180 | .global = config, | |
| 5181 | .parent = root_mod, | |
| 5182 | .builtin_mod = builtin_mod, | |
| 5183 | .builtin_modules = null, // `builtin_mod` is specified | |
| 5184 | }); | |
| 5185 | ||
| 5186 | var cleanup_build_dir: ?fs.Dir = null; | |
| 5187 | defer if (cleanup_build_dir) |*dir| dir.close(); | |
| 5188 | ||
| 5189 | if (dev.env.supports(.fetch_command)) { | |
| 5190 | const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); | |
| 5191 | defer fetch_prog_node.end(); | |
| 5192 | ||
| 5193 | var job_queue: Package.Fetch.JobQueue = .{ | |
| 5194 | .http_client = &http_client, | |
| 5195 | .thread_pool = &thread_pool, | |
| 5196 | .global_cache = global_cache_directory, | |
| 5197 | .read_only = false, | |
| 5198 | .recursive = true, | |
| 5199 | .debug_hash = false, | |
| 5200 | .work_around_btrfs_bug = work_around_btrfs_bug, | |
| 5201 | .unlazy_set = unlazy_set, | |
| 5202 | }; | |
| 5203 | defer job_queue.deinit(); | |
| 5204 | ||
| 5205 | if (system_pkg_dir_path) |p| { | |
| 5206 | job_queue.global_cache = .{ | |
| 5207 | .path = p, | |
| 5208 | .handle = fs.cwd().openDir(p, .{}) catch |err| { | |
| 5209 | fatal("unable to open system package directory '{s}': {s}", .{ | |
| 5210 | p, @errorName(err), | |
| 5211 | }); | |
| 5212 | }, | |
| 5213 | }; | |
| 5214 | job_queue.read_only = true; | |
| 5215 | cleanup_build_dir = job_queue.global_cache.handle; | |
| 5216 | } else { | |
| 5217 | try http_client.initDefaultProxies(arena); | |
| 5218 | } | |
| 5219 | ||
| 5220 | try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); | |
| 5221 | try job_queue.table.ensureUnusedCapacity(gpa, 1); | |
| 5222 | ||
| 5223 | var fetch: Package.Fetch = .{ | |
| 5224 | .arena = std.heap.ArenaAllocator.init(gpa), | |
| 5225 | .location = .{ .relative_path = build_mod.root }, | |
| 5226 | .location_tok = 0, | |
| 5227 | .hash_tok = .none, | |
| 5228 | .name_tok = 0, | |
| 5229 | .lazy_status = .eager, | |
| 5230 | .parent_package_root = build_mod.root, | |
| 5231 | .parent_manifest_ast = null, | |
| 5232 | .prog_node = fetch_prog_node, | |
| 5233 | .job_queue = &job_queue, | |
| 5234 | .omit_missing_hash_error = true, | |
| 5235 | .allow_missing_paths_field = false, | |
| 5236 | .allow_missing_fingerprint = false, | |
| 5237 | .allow_name_string = false, | |
| 5238 | .use_latest_commit = false, | |
| 5239 | ||
| 5240 | .package_root = undefined, | |
| 5241 | .error_bundle = undefined, | |
| 5242 | .manifest = null, | |
| 5243 | .manifest_ast = undefined, | |
| 5244 | .computed_hash = undefined, | |
| 5245 | .has_build_zig = true, | |
| 5246 | .oom_flag = false, | |
| 5247 | .latest_commit = null, | |
| 5248 | ||
| 5249 | .module = build_mod, | |
| 5250 | }; | |
| 5251 | job_queue.all_fetches.appendAssumeCapacity(&fetch); | |
| 5252 | ||
| 5253 | job_queue.table.putAssumeCapacityNoClobber( | |
| 5254 | Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory), | |
| 5255 | &fetch, | |
| 5256 | ); | |
| 5257 | ||
| 5258 | job_queue.thread_pool.spawnWg(&job_queue.wait_group, Package.Fetch.workerRun, .{ | |
| 5259 | &fetch, "root", | |
| 5260 | }); | |
| 5261 | job_queue.wait_group.wait(); | |
| 5262 | ||
| 5263 | try job_queue.consolidateErrors(); | |
| 5264 | ||
| 5265 | if (fetch.error_bundle.root_list.items.len > 0) { | |
| 5266 | var errors = try fetch.error_bundle.toOwnedBundle(""); | |
| 5267 | errors.renderToStdErr(color.renderOptions()); | |
| 5268 | process.exit(1); | |
| 5269 | } | |
| 5270 | ||
| 5271 | if (fetch_only) return cleanExit(); | |
| 5272 | ||
| 5273 | var source_buf = std.ArrayList(u8).init(gpa); | |
| 5274 | defer source_buf.deinit(); | |
| 5275 | try job_queue.createDependenciesSource(&source_buf); | |
| 5276 | const deps_mod = try createDependenciesModule( | |
| 5277 | arena, | |
| 5278 | source_buf.items, | |
| 5279 | root_mod, | |
| 5280 | global_cache_directory, | |
| 5281 | local_cache_directory, | |
| 5282 | builtin_mod, | |
| 5283 | config, | |
| 5284 | ); | |
| 5285 | ||
| 5286 | { | |
| 5287 | // We need a Module for each package's build.zig. | |
| 5288 | const hashes = job_queue.table.keys(); | |
| 5289 | const fetches = job_queue.table.values(); | |
| 5290 | try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); | |
| 5291 | for (hashes, fetches) |*hash, f| { | |
| 5292 | if (f == &fetch) { | |
| 5293 | // The first one is a dummy package for the current project. | |
| 5294 | continue; | |
| 5295 | } | |
| 5296 | if (!f.has_build_zig) | |
| 5297 | continue; | |
| 5298 | const hash_slice = hash.toSlice(); | |
| 5299 | const m = try Package.Module.create(arena, .{ | |
| 5300 | .global_cache_directory = global_cache_directory, | |
| 5301 | .paths = .{ | |
| 5302 | .root = try f.package_root.clone(arena), | |
| 5303 | .root_src_path = Package.build_zig_basename, | |
| 5304 | }, | |
| 5305 | .fully_qualified_name = try std.fmt.allocPrint( | |
| 5306 | arena, | |
| 5307 | "root.@dependencies.{s}", | |
| 5308 | .{hash_slice}, | |
| 5309 | ), | |
| 5310 | .cc_argv = &.{}, | |
| 5311 | .inherited = .{}, | |
| 5312 | .global = config, | |
| 5313 | .parent = root_mod, | |
| 5314 | .builtin_mod = builtin_mod, | |
| 5315 | .builtin_modules = null, // `builtin_mod` is specified | |
| 5316 | }); | |
| 5317 | const hash_cloned = try arena.dupe(u8, hash_slice); | |
| 5318 | deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); | |
| 5319 | f.module = m; | |
| 5320 | } | |
| 5321 | ||
| 5322 | // Each build.zig module needs access to each of its | |
| 5323 | // dependencies' build.zig modules by name. | |
| 5324 | for (fetches) |f| { | |
| 5325 | const mod = f.module orelse continue; | |
| 5326 | const man = f.manifest orelse continue; | |
| 5327 | const dep_names = man.dependencies.keys(); | |
| 5328 | try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); | |
| 5329 | for (dep_names, man.dependencies.values()) |name, dep| { | |
| 5330 | const dep_digest = Package.Fetch.depDigest( | |
| 5331 | f.package_root, | |
| 5332 | global_cache_directory, | |
| 5333 | dep, | |
| 5334 | ) orelse continue; | |
| 5335 | const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; | |
| 5336 | const name_cloned = try arena.dupe(u8, name); | |
| 5337 | mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); | |
| 5338 | } | |
| 5339 | } | |
| 5340 | } | |
| 5341 | } else try createEmptyDependenciesModule( | |
| 5342 | arena, | |
| 5343 | root_mod, | |
| 5344 | global_cache_directory, | |
| 5345 | local_cache_directory, | |
| 5346 | builtin_mod, | |
| 5347 | config, | |
| 5348 | ); | |
| 5349 | ||
| 5350 | try root_mod.deps.put(arena, "@build", build_mod); | |
| 5351 | ||
| 5352 | const comp = Compilation.create(gpa, arena, .{ | |
| 5353 | .zig_lib_directory = zig_lib_directory, | |
| 5354 | .local_cache_directory = local_cache_directory, | |
| 5355 | .global_cache_directory = global_cache_directory, | |
| 5356 | .root_name = "build", | |
| 5357 | .config = config, | |
| 5358 | .root_mod = root_mod, | |
| 5359 | .main_mod = build_mod, | |
| 5360 | .emit_bin = emit_bin, | |
| 5361 | .emit_h = null, | |
| 5362 | .self_exe_path = self_exe_path, | |
| 5363 | .thread_pool = &thread_pool, | |
| 5364 | .verbose_cc = verbose_cc, | |
| 5365 | .verbose_link = verbose_link, | |
| 5366 | .verbose_air = verbose_air, | |
| 5367 | .verbose_intern_pool = verbose_intern_pool, | |
| 5368 | .verbose_generic_instances = verbose_generic_instances, | |
| 5369 | .verbose_llvm_ir = verbose_llvm_ir, | |
| 5370 | .verbose_llvm_bc = verbose_llvm_bc, | |
| 5371 | .verbose_cimport = verbose_cimport, | |
| 5372 | .verbose_llvm_cpu_features = verbose_llvm_cpu_features, | |
| 5373 | .cache_mode = .whole, | |
| 5374 | .reference_trace = reference_trace, | |
| 5375 | .debug_compile_errors = debug_compile_errors, | |
| 5376 | }) catch |err| { | |
| 5377 | fatal("unable to create compilation: {s}", .{@errorName(err)}); | |
| 5378 | }; | |
| 5379 | defer comp.destroy(); | |
| 5380 | ||
| 5381 | updateModule(comp, color, root_prog_node) catch |err| switch (err) { | |
| 5382 | error.SemanticAnalyzeFail => process.exit(2), | |
| 5383 | else => |e| return e, | |
| 5384 | }; | |
| 5385 | ||
| 5386 | // Since incremental compilation isn't done yet, we use cache_mode = whole | |
| 5387 | // above, and thus the output file is already closed. | |
| 5388 | //try comp.makeBinFileExecutable(); | |
| 5389 | child_argv.items[argv_index_exe] = | |
| 5390 | try local_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?}); | |
| 5391 | } | |
| 5392 | ||
| 5393 | if (process.can_spawn) { | |
| 5394 | var child = std.process.Child.init(child_argv.items, gpa); | |
| 5395 | child.stdin_behavior = .Inherit; | |
| 5396 | child.stdout_behavior = .Inherit; | |
| 5397 | child.stderr_behavior = .Inherit; | |
| 5398 | ||
| 5399 | const term = t: { | |
| 5400 | std.debug.lockStdErr(); | |
| 5401 | defer std.debug.unlockStdErr(); | |
| 5402 | break :t child.spawnAndWait() catch |err| { | |
| 5403 | fatal("failed to spawn build runner {s}: {s}", .{ child_argv.items[0], @errorName(err) }); | |
| 5404 | }; | |
| 5405 | }; | |
| 5406 | ||
| 5407 | switch (term) { | |
| 5408 | .Exited => |code| { | |
| 5409 | if (code == 0) return cleanExit(); | |
| 5410 | // Indicates that the build runner has reported compile errors | |
| 5411 | // and this parent process does not need to report any further | |
| 5412 | // diagnostics. | |
| 5413 | if (code == 2) process.exit(2); | |
| 5414 | ||
| 5415 | if (code == 3) { | |
| 5416 | if (!dev.env.supports(.fetch_command)) process.exit(3); | |
| 5417 | // Indicates the configure phase failed due to missing lazy | |
| 5418 | // dependencies and stdout contains the hashes of the ones | |
| 5419 | // that are missing. | |
| 5420 | const s = fs.path.sep_str; | |
| 5421 | const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce; | |
| 5422 | const stdout = local_cache_directory.handle.readFileAlloc(arena, tmp_sub_path, 50 * 1024 * 1024) catch |err| { | |
| 5423 | fatal("unable to read results of configure phase from '{}{s}': {s}", .{ | |
| 5424 | local_cache_directory, tmp_sub_path, @errorName(err), | |
| 5425 | }); | |
| 5426 | }; | |
| 5427 | local_cache_directory.handle.deleteFile(tmp_sub_path) catch {}; | |
| 5428 | ||
| 5429 | var it = mem.splitScalar(u8, stdout, '\n'); | |
| 5430 | var any_errors = false; | |
| 5431 | while (it.next()) |hash| { | |
| 5432 | if (hash.len == 0) continue; | |
| 5433 | if (hash.len > Package.Hash.max_len) { | |
| 5434 | std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ | |
| 5435 | hash.len, hash, | |
| 5436 | }); | |
| 5437 | any_errors = true; | |
| 5438 | continue; | |
| 5439 | } | |
| 5440 | try unlazy_set.put(arena, .fromSlice(hash), {}); | |
| 5441 | } | |
| 5442 | if (any_errors) process.exit(3); | |
| 5443 | if (system_pkg_dir_path) |p| { | |
| 5444 | // In this mode, the system needs to provide these packages; they | |
| 5445 | // cannot be fetched by Zig. | |
| 5446 | for (unlazy_set.keys()) |*hash| { | |
| 5447 | std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ | |
| 5448 | p, hash.toSlice(), | |
| 5449 | }); | |
| 5450 | } | |
| 5451 | std.log.info("remote package fetching disabled due to --system mode", .{}); | |
| 5452 | std.log.info("dependencies might be avoidable depending on build configuration", .{}); | |
| 5453 | process.exit(3); | |
| 5454 | } | |
| 5455 | continue; | |
| 5456 | } | |
| 5457 | ||
| 5458 | const cmd = try std.mem.join(arena, " ", child_argv.items); | |
| 5459 | fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); | |
| 5460 | }, | |
| 5461 | else => { | |
| 5462 | const cmd = try std.mem.join(arena, " ", child_argv.items); | |
| 5463 | fatal("the following build command crashed:\n{s}", .{cmd}); | |
| 5464 | }, | |
| 5465 | } | |
| 5466 | } else { | |
| 5467 | const cmd = try std.mem.join(arena, " ", child_argv.items); | |
| 5468 | fatal("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(native_os), cmd }); | |
| 5469 | } | |
| 5470 | } | |
| 5471 | } | |
| 5472 | ||
| 5473 | 4801 | const JitCmdOptions = struct { |
| 5474 | 4802 | cmd_name: []const u8, |
| 5475 | 4803 | root_src_path: []const u8, |
| ... | ... | @@ -5481,6 +4809,7 @@ const JitCmdOptions = struct { |
| 5481 | 4809 | /// Send error bundles via std.zig.Server over stdout |
| 5482 | 4810 | server: bool = false, |
| 5483 | 4811 | progress_node: ?std.Progress.Node = null, |
| 4812 | optimize_mode: std.builtin.OptimizeMode = .ReleaseFast, | |
| 5484 | 4813 | }; |
| 5485 | 4814 | |
| 5486 | 4815 | fn jitCmd( |
| ... | ... | @@ -5497,7 +4826,7 @@ fn jitCmd( |
| 5497 | 4826 | }); |
| 5498 | 4827 | |
| 5499 | 4828 | const target_query: std.Target.Query = .{}; |
| 5500 | const resolved_target: Package.Module.ResolvedTarget = .{ | |
| 4829 | const resolved_target: Module.ResolvedTarget = .{ | |
| 5501 | 4830 | .result = std.zig.resolveTargetQueryOrFatal(target_query), |
| 5502 | 4831 | .is_native_os = true, |
| 5503 | 4832 | .is_native_abi = true, |
| ... | ... | @@ -5520,7 +4849,7 @@ fn jitCmd( |
| 5520 | 4849 | const optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet()) |
| 5521 | 4850 | .Debug |
| 5522 | 4851 | else |
| 5523 | .ReleaseFast; | |
| 4852 | options.optimize_mode; | |
| 5524 | 4853 | const strip = optimize_mode != .Debug; |
| 5525 | 4854 | const override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena); |
| 5526 | 4855 | const override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); |
| ... | ... | @@ -5554,12 +4883,12 @@ fn jitCmd( |
| 5554 | 4883 | defer thread_pool.deinit(); |
| 5555 | 4884 | |
| 5556 | 4885 | var child_argv: std.ArrayListUnmanaged([]const u8) = .empty; |
| 5557 | try child_argv.ensureUnusedCapacity(arena, args.len + 4); | |
| 4886 | try child_argv.ensureUnusedCapacity(arena, args.len + 6); | |
| 5558 | 4887 | |
| 5559 | 4888 | // We want to release all the locks before executing the child process, so we make a nice |
| 5560 | 4889 | // big block here to ensure the cleanup gets run when we extract out our argv. |
| 5561 | 4890 | { |
| 5562 | const main_mod_paths: Package.Module.CreateOptions.Paths = .{ | |
| 4891 | const main_mod_paths: Module.CreateOptions.Paths = .{ | |
| 5563 | 4892 | .root = .{ |
| 5564 | 4893 | .root_dir = zig_lib_directory, |
| 5565 | 4894 | .sub_path = "compiler", |
| ... | ... | @@ -5577,7 +4906,7 @@ fn jitCmd( |
| 5577 | 4906 | .is_test = false, |
| 5578 | 4907 | }); |
| 5579 | 4908 | |
| 5580 | const root_mod = try Package.Module.create(arena, .{ | |
| 4909 | const root_mod = try Module.create(arena, .{ | |
| 5581 | 4910 | .global_cache_directory = global_cache_directory, |
| 5582 | 4911 | .paths = main_mod_paths, |
| 5583 | 4912 | .fully_qualified_name = "root", |
| ... | ... | @@ -5594,7 +4923,7 @@ fn jitCmd( |
| 5594 | 4923 | }); |
| 5595 | 4924 | |
| 5596 | 4925 | if (options.depend_on_aro) { |
| 5597 | const aro_mod = try Package.Module.create(arena, .{ | |
| 4926 | const aro_mod = try Module.create(arena, .{ | |
| 5598 | 4927 | .global_cache_directory = global_cache_directory, |
| 5599 | 4928 | .paths = .{ |
| 5600 | 4929 | .root = .{ |
| ... | ... | @@ -5669,7 +4998,24 @@ fn jitCmd( |
| 5669 | 4998 | if (options.prepend_global_cache_path) |
| 5670 | 4999 | child_argv.appendAssumeCapacity(global_cache_directory.path.?); |
| 5671 | 5000 | |
| 5672 | child_argv.appendSliceAssumeCapacity(args); | |
| 5001 | if (options.add_seed_argument) { | |
| 5002 | child_argv.appendSliceAssumeCapacity(&.{ | |
| 5003 | "--seed", try std.fmt.allocPrint(arena, "0x{x}", .{std.crypto.random.int(u32)}), | |
| 5004 | }); | |
| 5005 | const seed_arg_index = child_argv.items.len - 1; | |
| 5006 | var i: usize = 0; | |
| 5007 | while (i < args.len) { | |
| 5008 | if (mem.eql(u8, args[i], "--seed") and i + 1 <= args.len) { | |
| 5009 | child_argv.items[seed_arg_index] = args[i + 1]; | |
| 5010 | i += 2; | |
| 5011 | } else { | |
| 5012 | child_argv.appendAssumeCapacity(args[i]); | |
| 5013 | i += 1; | |
| 5014 | } | |
| 5015 | } | |
| 5016 | } else { | |
| 5017 | child_argv.appendSliceAssumeCapacity(args); | |
| 5018 | } | |
| 5673 | 5019 | |
| 5674 | 5020 | if (process.can_execv and options.capture == null) { |
| 5675 | 5021 | const err = process.execv(gpa, child_argv.items); |
| ... | ... | @@ -6241,7 +5587,7 @@ fn cmdAstCheck( |
| 6241 | 5587 | break :mode .zig; |
| 6242 | 5588 | }; |
| 6243 | 5589 | |
| 6244 | file.mod = try Package.Module.createLimited(arena, .{ | |
| 5590 | file.mod = try Module.createLimited(arena, .{ | |
| 6245 | 5591 | .root = Path.cwd(), |
| 6246 | 5592 | .root_src_path = file.sub_file_path, |
| 6247 | 5593 | .fully_qualified_name = "root", |
| ... | ... | @@ -6647,7 +5993,7 @@ fn cmdChangelist( |
| 6647 | 5993 | .mod = undefined, |
| 6648 | 5994 | }; |
| 6649 | 5995 | |
| 6650 | file.mod = try Package.Module.createLimited(arena, .{ | |
| 5996 | file.mod = try Module.createLimited(arena, .{ | |
| 6651 | 5997 | .root = Path.cwd(), |
| 6652 | 5998 | .root_src_path = file.sub_file_path, |
| 6653 | 5999 | .fully_qualified_name = "root", |
| ... | ... | @@ -7008,613 +6354,6 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes { |
| 7008 | 6354 | fatal("unsupported rc includes type: '{s}'", .{arg}); |
| 7009 | 6355 | } |
| 7010 | 6356 | |
| 7011 | const usage_fetch = | |
| 7012 | \\Usage: zig fetch [options] <url> | |
| 7013 | \\Usage: zig fetch [options] <path> | |
| 7014 | \\ | |
| 7015 | \\ Copy a package into the global cache and print its hash. | |
| 7016 | \\ <url> must point to one of the following: | |
| 7017 | \\ - A git+http / git+https server for the package | |
| 7018 | \\ - A tarball file (with or without compression) containing | |
| 7019 | \\ package source | |
| 7020 | \\ - A git bundle file containing package source | |
| 7021 | \\ | |
| 7022 | \\Examples: | |
| 7023 | \\ | |
| 7024 | \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git | |
| 7025 | \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz | |
| 7026 | \\ | |
| 7027 | \\Options: | |
| 7028 | \\ -h, --help Print this help and exit | |
| 7029 | \\ --global-cache-dir [path] Override path to global Zig cache directory | |
| 7030 | \\ --debug-hash Print verbose hash information to stdout | |
| 7031 | \\ --save Add the fetched package to build.zig.zon | |
| 7032 | \\ --save=[name] Add the fetched package to build.zig.zon as name | |
| 7033 | \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim | |
| 7034 | \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim | |
| 7035 | \\ | |
| 7036 | ; | |
| 7037 | ||
| 7038 | fn cmdFetch( | |
| 7039 | gpa: Allocator, | |
| 7040 | arena: Allocator, | |
| 7041 | args: []const []const u8, | |
| 7042 | ) !void { | |
| 7043 | dev.check(.fetch_command); | |
| 7044 | ||
| 7045 | const color: Color = .auto; | |
| 7046 | const work_around_btrfs_bug = native_os == .linux and | |
| 7047 | EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); | |
| 7048 | var opt_path_or_url: ?[]const u8 = null; | |
| 7049 | var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); | |
| 7050 | var debug_hash: bool = false; | |
| 7051 | var save: union(enum) { | |
| 7052 | no, | |
| 7053 | yes: ?[]const u8, | |
| 7054 | exact: ?[]const u8, | |
| 7055 | } = .no; | |
| 7056 | ||
| 7057 | { | |
| 7058 | var i: usize = 0; | |
| 7059 | while (i < args.len) : (i += 1) { | |
| 7060 | const arg = args[i]; | |
| 7061 | if (mem.startsWith(u8, arg, "-")) { | |
| 7062 | if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { | |
| 7063 | const stdout = io.getStdOut().writer(); | |
| 7064 | try stdout.writeAll(usage_fetch); | |
| 7065 | return cleanExit(); | |
| 7066 | } else if (mem.eql(u8, arg, "--global-cache-dir")) { | |
| 7067 | if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); | |
| 7068 | i += 1; | |
| 7069 | override_global_cache_dir = args[i]; | |
| 7070 | } else if (mem.eql(u8, arg, "--debug-hash")) { | |
| 7071 | debug_hash = true; | |
| 7072 | } else if (mem.eql(u8, arg, "--save")) { | |
| 7073 | save = .{ .yes = null }; | |
| 7074 | } else if (mem.startsWith(u8, arg, "--save=")) { | |
| 7075 | save = .{ .yes = arg["--save=".len..] }; | |
| 7076 | } else if (mem.eql(u8, arg, "--save-exact")) { | |
| 7077 | save = .{ .exact = null }; | |
| 7078 | } else if (mem.startsWith(u8, arg, "--save-exact=")) { | |
| 7079 | save = .{ .exact = arg["--save-exact=".len..] }; | |
| 7080 | } else { | |
| 7081 | fatal("unrecognized parameter: '{s}'", .{arg}); | |
| 7082 | } | |
| 7083 | } else if (opt_path_or_url != null) { | |
| 7084 | fatal("unexpected extra parameter: '{s}'", .{arg}); | |
| 7085 | } else { | |
| 7086 | opt_path_or_url = arg; | |
| 7087 | } | |
| 7088 | } | |
| 7089 | } | |
| 7090 | ||
| 7091 | const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); | |
| 7092 | ||
| 7093 | var thread_pool: ThreadPool = undefined; | |
| 7094 | try thread_pool.init(.{ .allocator = gpa }); | |
| 7095 | defer thread_pool.deinit(); | |
| 7096 | ||
| 7097 | var http_client: std.http.Client = .{ .allocator = gpa }; | |
| 7098 | defer http_client.deinit(); | |
| 7099 | ||
| 7100 | try http_client.initDefaultProxies(arena); | |
| 7101 | ||
| 7102 | var root_prog_node = std.Progress.start(.{ | |
| 7103 | .root_name = "Fetch", | |
| 7104 | }); | |
| 7105 | defer root_prog_node.end(); | |
| 7106 | ||
| 7107 | var global_cache_directory: Directory = l: { | |
| 7108 | const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); | |
| 7109 | break :l .{ | |
| 7110 | .handle = try fs.cwd().makeOpenPath(p, .{}), | |
| 7111 | .path = p, | |
| 7112 | }; | |
| 7113 | }; | |
| 7114 | defer global_cache_directory.handle.close(); | |
| 7115 | ||
| 7116 | var job_queue: Package.Fetch.JobQueue = .{ | |
| 7117 | .http_client = &http_client, | |
| 7118 | .thread_pool = &thread_pool, | |
| 7119 | .global_cache = global_cache_directory, | |
| 7120 | .recursive = false, | |
| 7121 | .read_only = false, | |
| 7122 | .debug_hash = debug_hash, | |
| 7123 | .work_around_btrfs_bug = work_around_btrfs_bug, | |
| 7124 | }; | |
| 7125 | defer job_queue.deinit(); | |
| 7126 | ||
| 7127 | var fetch: Package.Fetch = .{ | |
| 7128 | .arena = std.heap.ArenaAllocator.init(gpa), | |
| 7129 | .location = .{ .path_or_url = path_or_url }, | |
| 7130 | .location_tok = 0, | |
| 7131 | .hash_tok = .none, | |
| 7132 | .name_tok = 0, | |
| 7133 | .lazy_status = .eager, | |
| 7134 | .parent_package_root = undefined, | |
| 7135 | .parent_manifest_ast = null, | |
| 7136 | .prog_node = root_prog_node, | |
| 7137 | .job_queue = &job_queue, | |
| 7138 | .omit_missing_hash_error = true, | |
| 7139 | .allow_missing_paths_field = false, | |
| 7140 | .allow_missing_fingerprint = true, | |
| 7141 | .allow_name_string = true, | |
| 7142 | .use_latest_commit = true, | |
| 7143 | ||
| 7144 | .package_root = undefined, | |
| 7145 | .error_bundle = undefined, | |
| 7146 | .manifest = null, | |
| 7147 | .manifest_ast = undefined, | |
| 7148 | .computed_hash = undefined, | |
| 7149 | .has_build_zig = false, | |
| 7150 | .oom_flag = false, | |
| 7151 | .latest_commit = null, | |
| 7152 | ||
| 7153 | .module = null, | |
| 7154 | }; | |
| 7155 | defer fetch.deinit(); | |
| 7156 | ||
| 7157 | fetch.run() catch |err| switch (err) { | |
| 7158 | error.OutOfMemory => fatal("out of memory", .{}), | |
| 7159 | error.FetchFailed => {}, // error bundle checked below | |
| 7160 | }; | |
| 7161 | ||
| 7162 | if (fetch.error_bundle.root_list.items.len > 0) { | |
| 7163 | var errors = try fetch.error_bundle.toOwnedBundle(""); | |
| 7164 | errors.renderToStdErr(color.renderOptions()); | |
| 7165 | process.exit(1); | |
| 7166 | } | |
| 7167 | ||
| 7168 | const package_hash = fetch.computedPackageHash(); | |
| 7169 | const package_hash_slice = package_hash.toSlice(); | |
| 7170 | ||
| 7171 | root_prog_node.end(); | |
| 7172 | root_prog_node = .{ .index = .none }; | |
| 7173 | ||
| 7174 | const name = switch (save) { | |
| 7175 | .no => { | |
| 7176 | try io.getStdOut().writer().print("{s}\n", .{package_hash_slice}); | |
| 7177 | return cleanExit(); | |
| 7178 | }, | |
| 7179 | .yes, .exact => |name| name: { | |
| 7180 | if (name) |n| break :name n; | |
| 7181 | const fetched_manifest = fetch.manifest orelse | |
| 7182 | fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); | |
| 7183 | break :name fetched_manifest.name; | |
| 7184 | }, | |
| 7185 | }; | |
| 7186 | ||
| 7187 | const cwd_path = try process.getCwdAlloc(arena); | |
| 7188 | ||
| 7189 | var build_root = try findBuildRoot(arena, .{ | |
| 7190 | .cwd_path = cwd_path, | |
| 7191 | }); | |
| 7192 | defer build_root.deinit(); | |
| 7193 | ||
| 7194 | // The name to use in case the manifest file needs to be created now. | |
| 7195 | const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path); | |
| 7196 | var manifest, var ast = try loadManifest(gpa, arena, .{ | |
| 7197 | .root_name = try sanitizeExampleName(arena, init_root_name), | |
| 7198 | .dir = build_root.directory.handle, | |
| 7199 | .color = color, | |
| 7200 | }); | |
| 7201 | defer { | |
| 7202 | manifest.deinit(gpa); | |
| 7203 | ast.deinit(gpa); | |
| 7204 | } | |
| 7205 | ||
| 7206 | var fixups: Ast.Fixups = .{}; | |
| 7207 | defer fixups.deinit(gpa); | |
| 7208 | ||
| 7209 | var saved_path_or_url = path_or_url; | |
| 7210 | ||
| 7211 | if (fetch.latest_commit) |latest_commit| resolved: { | |
| 7212 | const latest_commit_hex = try std.fmt.allocPrint(arena, "{}", .{latest_commit}); | |
| 7213 | ||
| 7214 | var uri = try std.Uri.parse(path_or_url); | |
| 7215 | ||
| 7216 | if (uri.fragment) |fragment| { | |
| 7217 | const target_ref = try fragment.toRawMaybeAlloc(arena); | |
| 7218 | ||
| 7219 | // the refspec may already be fully resolved | |
| 7220 | if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; | |
| 7221 | ||
| 7222 | std.log.info("resolved ref '{s}' to commit {s}", .{ target_ref, latest_commit_hex }); | |
| 7223 | ||
| 7224 | // include the original refspec in a query parameter, could be used to check for updates | |
| 7225 | uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={%}", .{fragment}) }; | |
| 7226 | } else { | |
| 7227 | std.log.info("resolved to commit {s}", .{latest_commit_hex}); | |
| 7228 | } | |
| 7229 | ||
| 7230 | // replace the refspec with the resolved commit SHA | |
| 7231 | uri.fragment = .{ .raw = latest_commit_hex }; | |
| 7232 | ||
| 7233 | switch (save) { | |
| 7234 | .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{}", .{uri}), | |
| 7235 | .no, .exact => {}, // keep the original URL | |
| 7236 | } | |
| 7237 | } | |
| 7238 | ||
| 7239 | const new_node_init = try std.fmt.allocPrint(arena, | |
| 7240 | \\.{{ | |
| 7241 | \\ .url = "{}", | |
| 7242 | \\ .hash = "{}", | |
| 7243 | \\ }} | |
| 7244 | , .{ | |
| 7245 | std.zig.fmtEscapes(saved_path_or_url), | |
| 7246 | std.zig.fmtEscapes(package_hash_slice), | |
| 7247 | }); | |
| 7248 | ||
| 7249 | const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{ | |
| 7250 | std.zig.fmtId(name), new_node_init, | |
| 7251 | }); | |
| 7252 | ||
| 7253 | const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ | |
| 7254 | new_node_text, | |
| 7255 | }); | |
| 7256 | ||
| 7257 | const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ | |
| 7258 | dependencies_init, | |
| 7259 | }); | |
| 7260 | ||
| 7261 | if (manifest.dependencies.get(name)) |dep| { | |
| 7262 | if (dep.hash) |h| { | |
| 7263 | switch (dep.location) { | |
| 7264 | .url => |u| { | |
| 7265 | if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { | |
| 7266 | std.log.info("existing dependency named '{s}' is up-to-date", .{name}); | |
| 7267 | process.exit(0); | |
| 7268 | } | |
| 7269 | }, | |
| 7270 | .path => {}, | |
| 7271 | } | |
| 7272 | } | |
| 7273 | ||
| 7274 | const location_replace = try std.fmt.allocPrint( | |
| 7275 | arena, | |
| 7276 | "\"{}\"", | |
| 7277 | .{std.zig.fmtEscapes(saved_path_or_url)}, | |
| 7278 | ); | |
| 7279 | const hash_replace = try std.fmt.allocPrint( | |
| 7280 | arena, | |
| 7281 | "\"{}\"", | |
| 7282 | .{std.zig.fmtEscapes(package_hash_slice)}, | |
| 7283 | ); | |
| 7284 | ||
| 7285 | warn("overwriting existing dependency named '{s}'", .{name}); | |
| 7286 | try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); | |
| 7287 | if (dep.hash_node.unwrap()) |hash_node| { | |
| 7288 | try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); | |
| 7289 | } else { | |
| 7290 | // https://github.com/ziglang/zig/issues/21690 | |
| 7291 | } | |
| 7292 | } else if (manifest.dependencies.count() > 0) { | |
| 7293 | // Add fixup for adding another dependency. | |
| 7294 | const deps = manifest.dependencies.values(); | |
| 7295 | const last_dep_node = deps[deps.len - 1].node; | |
| 7296 | try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); | |
| 7297 | } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { | |
| 7298 | // Add fixup for replacing the entire dependencies struct. | |
| 7299 | try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); | |
| 7300 | } else { | |
| 7301 | // Add fixup for adding dependencies struct. | |
| 7302 | try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); | |
| 7303 | } | |
| 7304 | ||
| 7305 | var rendered = std.ArrayList(u8).init(gpa); | |
| 7306 | defer rendered.deinit(); | |
| 7307 | try ast.renderToArrayList(&rendered, fixups); | |
| 7308 | ||
| 7309 | build_root.directory.handle.writeFile(.{ .sub_path = Package.Manifest.basename, .data = rendered.items }) catch |err| { | |
| 7310 | fatal("unable to write {s} file: {s}", .{ Package.Manifest.basename, @errorName(err) }); | |
| 7311 | }; | |
| 7312 | ||
| 7313 | return cleanExit(); | |
| 7314 | } | |
| 7315 | ||
| 7316 | fn createEmptyDependenciesModule( | |
| 7317 | arena: Allocator, | |
| 7318 | main_mod: *Package.Module, | |
| 7319 | global_cache_directory: Cache.Directory, | |
| 7320 | local_cache_directory: Cache.Directory, | |
| 7321 | builtin_mod: *Package.Module, | |
| 7322 | global_options: Compilation.Config, | |
| 7323 | ) !void { | |
| 7324 | var source = std.ArrayList(u8).init(arena); | |
| 7325 | try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); | |
| 7326 | _ = try createDependenciesModule( | |
| 7327 | arena, | |
| 7328 | source.items, | |
| 7329 | main_mod, | |
| 7330 | global_cache_directory, | |
| 7331 | local_cache_directory, | |
| 7332 | builtin_mod, | |
| 7333 | global_options, | |
| 7334 | ); | |
| 7335 | } | |
| 7336 | ||
| 7337 | /// Creates the dependencies.zig file and corresponding `Package.Module` for the | |
| 7338 | /// build runner to obtain via `@import("@dependencies")`. | |
| 7339 | fn createDependenciesModule( | |
| 7340 | arena: Allocator, | |
| 7341 | source: []const u8, | |
| 7342 | main_mod: *Package.Module, | |
| 7343 | global_cache_directory: Cache.Directory, | |
| 7344 | local_cache_directory: Cache.Directory, | |
| 7345 | builtin_mod: *Package.Module, | |
| 7346 | global_options: Compilation.Config, | |
| 7347 | ) !*Package.Module { | |
| 7348 | // Atomically create the file in a directory named after the hash of its contents. | |
| 7349 | const basename = "dependencies.zig"; | |
| 7350 | const rand_int = std.crypto.random.int(u64); | |
| 7351 | const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); | |
| 7352 | { | |
| 7353 | var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); | |
| 7354 | defer tmp_dir.close(); | |
| 7355 | try tmp_dir.writeFile(.{ .sub_path = basename, .data = source }); | |
| 7356 | } | |
| 7357 | ||
| 7358 | var hh: Cache.HashHelper = .{}; | |
| 7359 | hh.addBytes(build_options.version); | |
| 7360 | hh.addBytes(source); | |
| 7361 | const hex_digest = hh.final(); | |
| 7362 | ||
| 7363 | const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); | |
| 7364 | try Package.Fetch.renameTmpIntoCache( | |
| 7365 | local_cache_directory.handle, | |
| 7366 | tmp_dir_sub_path, | |
| 7367 | o_dir_sub_path, | |
| 7368 | ); | |
| 7369 | ||
| 7370 | const deps_mod = try Package.Module.create(arena, .{ | |
| 7371 | .global_cache_directory = global_cache_directory, | |
| 7372 | .paths = .{ | |
| 7373 | .root = .{ | |
| 7374 | .root_dir = local_cache_directory, | |
| 7375 | .sub_path = o_dir_sub_path, | |
| 7376 | }, | |
| 7377 | .root_src_path = basename, | |
| 7378 | }, | |
| 7379 | .fully_qualified_name = "root.@dependencies", | |
| 7380 | .parent = main_mod, | |
| 7381 | .cc_argv = &.{}, | |
| 7382 | .inherited = .{}, | |
| 7383 | .global = global_options, | |
| 7384 | .builtin_mod = builtin_mod, | |
| 7385 | .builtin_modules = null, // `builtin_mod` is specified | |
| 7386 | }); | |
| 7387 | try main_mod.deps.put(arena, "@dependencies", deps_mod); | |
| 7388 | return deps_mod; | |
| 7389 | } | |
| 7390 | ||
| 7391 | const BuildRoot = struct { | |
| 7392 | directory: Cache.Directory, | |
| 7393 | build_zig_basename: []const u8, | |
| 7394 | cleanup_build_dir: ?fs.Dir, | |
| 7395 | ||
| 7396 | fn deinit(br: *BuildRoot) void { | |
| 7397 | if (br.cleanup_build_dir) |*dir| dir.close(); | |
| 7398 | br.* = undefined; | |
| 7399 | } | |
| 7400 | }; | |
| 7401 | ||
| 7402 | const FindBuildRootOptions = struct { | |
| 7403 | build_file: ?[]const u8 = null, | |
| 7404 | cwd_path: ?[]const u8 = null, | |
| 7405 | }; | |
| 7406 | ||
| 7407 | fn findBuildRoot(arena: Allocator, options: FindBuildRootOptions) !BuildRoot { | |
| 7408 | const cwd_path = options.cwd_path orelse try process.getCwdAlloc(arena); | |
| 7409 | const build_zig_basename = if (options.build_file) |bf| | |
| 7410 | fs.path.basename(bf) | |
| 7411 | else | |
| 7412 | Package.build_zig_basename; | |
| 7413 | ||
| 7414 | if (options.build_file) |bf| { | |
| 7415 | if (fs.path.dirname(bf)) |dirname| { | |
| 7416 | const dir = fs.cwd().openDir(dirname, .{}) catch |err| { | |
| 7417 | fatal("unable to open directory to build file from argument 'build-file', '{s}': {s}", .{ dirname, @errorName(err) }); | |
| 7418 | }; | |
| 7419 | return .{ | |
| 7420 | .build_zig_basename = build_zig_basename, | |
| 7421 | .directory = .{ .path = dirname, .handle = dir }, | |
| 7422 | .cleanup_build_dir = dir, | |
| 7423 | }; | |
| 7424 | } | |
| 7425 | ||
| 7426 | return .{ | |
| 7427 | .build_zig_basename = build_zig_basename, | |
| 7428 | .directory = .{ .path = null, .handle = fs.cwd() }, | |
| 7429 | .cleanup_build_dir = null, | |
| 7430 | }; | |
| 7431 | } | |
| 7432 | // Search up parent directories until we find build.zig. | |
| 7433 | var dirname: []const u8 = cwd_path; | |
| 7434 | while (true) { | |
| 7435 | const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); | |
| 7436 | if (fs.cwd().access(joined_path, .{})) |_| { | |
| 7437 | const dir = fs.cwd().openDir(dirname, .{}) catch |err| { | |
| 7438 | fatal("unable to open directory while searching for build.zig file, '{s}': {s}", .{ dirname, @errorName(err) }); | |
| 7439 | }; | |
| 7440 | return .{ | |
| 7441 | .build_zig_basename = build_zig_basename, | |
| 7442 | .directory = .{ | |
| 7443 | .path = dirname, | |
| 7444 | .handle = dir, | |
| 7445 | }, | |
| 7446 | .cleanup_build_dir = dir, | |
| 7447 | }; | |
| 7448 | } else |err| switch (err) { | |
| 7449 | error.FileNotFound => { | |
| 7450 | dirname = fs.path.dirname(dirname) orelse { | |
| 7451 | std.log.info("initialize {s} template file with 'zig init'", .{ | |
| 7452 | Package.build_zig_basename, | |
| 7453 | }); | |
| 7454 | std.log.info("see 'zig --help' for more options", .{}); | |
| 7455 | fatal("no build.zig file found, in the current directory or any parent directories", .{}); | |
| 7456 | }; | |
| 7457 | continue; | |
| 7458 | }, | |
| 7459 | else => |e| return e, | |
| 7460 | } | |
| 7461 | } | |
| 7462 | } | |
| 7463 | ||
| 7464 | const LoadManifestOptions = struct { | |
| 7465 | root_name: []const u8, | |
| 7466 | dir: fs.Dir, | |
| 7467 | color: Color, | |
| 7468 | }; | |
| 7469 | ||
| 7470 | fn loadManifest( | |
| 7471 | gpa: Allocator, | |
| 7472 | arena: Allocator, | |
| 7473 | options: LoadManifestOptions, | |
| 7474 | ) !struct { Package.Manifest, Ast } { | |
| 7475 | const manifest_bytes = while (true) { | |
| 7476 | break options.dir.readFileAllocOptions( | |
| 7477 | arena, | |
| 7478 | Package.Manifest.basename, | |
| 7479 | Package.Manifest.max_bytes, | |
| 7480 | null, | |
| 7481 | 1, | |
| 7482 | 0, | |
| 7483 | ) catch |err| switch (err) { | |
| 7484 | error.FileNotFound => { | |
| 7485 | const fingerprint: Package.Fingerprint = .generate(options.root_name); | |
| 7486 | var templates = findTemplates(gpa, arena); | |
| 7487 | defer templates.deinit(); | |
| 7488 | templates.write(arena, options.dir, options.root_name, Package.Manifest.basename, fingerprint) catch |e| { | |
| 7489 | fatal("unable to write {s}: {s}", .{ | |
| 7490 | Package.Manifest.basename, @errorName(e), | |
| 7491 | }); | |
| 7492 | }; | |
| 7493 | continue; | |
| 7494 | }, | |
| 7495 | else => |e| fatal("unable to load {s}: {s}", .{ | |
| 7496 | Package.Manifest.basename, @errorName(e), | |
| 7497 | }), | |
| 7498 | }; | |
| 7499 | }; | |
| 7500 | var ast = try Ast.parse(gpa, manifest_bytes, .zon); | |
| 7501 | errdefer ast.deinit(gpa); | |
| 7502 | ||
| 7503 | if (ast.errors.len > 0) { | |
| 7504 | try std.zig.printAstErrorsToStderr(gpa, ast, Package.Manifest.basename, options.color); | |
| 7505 | process.exit(2); | |
| 7506 | } | |
| 7507 | ||
| 7508 | var manifest = try Package.Manifest.parse(gpa, ast, .{}); | |
| 7509 | errdefer manifest.deinit(gpa); | |
| 7510 | ||
| 7511 | if (manifest.errors.len > 0) { | |
| 7512 | var wip_errors: std.zig.ErrorBundle.Wip = undefined; | |
| 7513 | try wip_errors.init(gpa); | |
| 7514 | defer wip_errors.deinit(); | |
| 7515 | ||
| 7516 | const src_path = try wip_errors.addString(Package.Manifest.basename); | |
| 7517 | try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); | |
| 7518 | ||
| 7519 | var error_bundle = try wip_errors.toOwnedBundle(""); | |
| 7520 | defer error_bundle.deinit(gpa); | |
| 7521 | error_bundle.renderToStdErr(options.color.renderOptions()); | |
| 7522 | ||
| 7523 | process.exit(2); | |
| 7524 | } | |
| 7525 | return .{ manifest, ast }; | |
| 7526 | } | |
| 7527 | ||
| 7528 | const Templates = struct { | |
| 7529 | zig_lib_directory: Cache.Directory, | |
| 7530 | dir: fs.Dir, | |
| 7531 | buffer: std.ArrayList(u8), | |
| 7532 | ||
| 7533 | fn deinit(templates: *Templates) void { | |
| 7534 | templates.zig_lib_directory.handle.close(); | |
| 7535 | templates.dir.close(); | |
| 7536 | templates.buffer.deinit(); | |
| 7537 | templates.* = undefined; | |
| 7538 | } | |
| 7539 | ||
| 7540 | fn write( | |
| 7541 | templates: *Templates, | |
| 7542 | arena: Allocator, | |
| 7543 | out_dir: fs.Dir, | |
| 7544 | root_name: []const u8, | |
| 7545 | template_path: []const u8, | |
| 7546 | fingerprint: Package.Fingerprint, | |
| 7547 | ) !void { | |
| 7548 | if (fs.path.dirname(template_path)) |dirname| { | |
| 7549 | out_dir.makePath(dirname) catch |err| { | |
| 7550 | fatal("unable to make path '{s}': {s}", .{ dirname, @errorName(err) }); | |
| 7551 | }; | |
| 7552 | } | |
| 7553 | ||
| 7554 | const max_bytes = 10 * 1024 * 1024; | |
| 7555 | const contents = templates.dir.readFileAlloc(arena, template_path, max_bytes) catch |err| { | |
| 7556 | fatal("unable to read template file '{s}': {s}", .{ template_path, @errorName(err) }); | |
| 7557 | }; | |
| 7558 | templates.buffer.clearRetainingCapacity(); | |
| 7559 | try templates.buffer.ensureUnusedCapacity(contents.len); | |
| 7560 | var i: usize = 0; | |
| 7561 | while (i < contents.len) { | |
| 7562 | if (contents[i] == '.') { | |
| 7563 | if (std.mem.startsWith(u8, contents[i..], ".LITNAME")) { | |
| 7564 | try templates.buffer.append('.'); | |
| 7565 | try templates.buffer.appendSlice(root_name); | |
| 7566 | i += ".LITNAME".len; | |
| 7567 | continue; | |
| 7568 | } else if (std.mem.startsWith(u8, contents[i..], ".NAME")) { | |
| 7569 | try templates.buffer.appendSlice(root_name); | |
| 7570 | i += ".NAME".len; | |
| 7571 | continue; | |
| 7572 | } else if (std.mem.startsWith(u8, contents[i..], ".FINGERPRINT")) { | |
| 7573 | try templates.buffer.writer().print("0x{x}", .{fingerprint.int()}); | |
| 7574 | i += ".FINGERPRINT".len; | |
| 7575 | continue; | |
| 7576 | } else if (std.mem.startsWith(u8, contents[i..], ".ZIGVER")) { | |
| 7577 | try templates.buffer.appendSlice(build_options.version); | |
| 7578 | i += ".ZIGVER".len; | |
| 7579 | continue; | |
| 7580 | } | |
| 7581 | } | |
| 7582 | try templates.buffer.append(contents[i]); | |
| 7583 | i += 1; | |
| 7584 | } | |
| 7585 | ||
| 7586 | return out_dir.writeFile(.{ | |
| 7587 | .sub_path = template_path, | |
| 7588 | .data = templates.buffer.items, | |
| 7589 | .flags = .{ .exclusive = true }, | |
| 7590 | }); | |
| 7591 | } | |
| 7592 | }; | |
| 7593 | ||
| 7594 | fn findTemplates(gpa: Allocator, arena: Allocator) Templates { | |
| 7595 | const self_exe_path = introspect.findZigExePath(arena) catch |err| { | |
| 7596 | fatal("unable to find self exe path: {s}", .{@errorName(err)}); | |
| 7597 | }; | |
| 7598 | var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| { | |
| 7599 | fatal("unable to find zig installation directory '{s}': {s}", .{ self_exe_path, @errorName(err) }); | |
| 7600 | }; | |
| 7601 | ||
| 7602 | const s = fs.path.sep_str; | |
| 7603 | const template_sub_path = "init"; | |
| 7604 | const template_dir = zig_lib_directory.handle.openDir(template_sub_path, .{}) catch |err| { | |
| 7605 | const path = zig_lib_directory.path orelse "."; | |
| 7606 | fatal("unable to open zig project template directory '{s}{s}{s}': {s}", .{ | |
| 7607 | path, s, template_sub_path, @errorName(err), | |
| 7608 | }); | |
| 7609 | }; | |
| 7610 | ||
| 7611 | return .{ | |
| 7612 | .zig_lib_directory = zig_lib_directory, | |
| 7613 | .dir = template_dir, | |
| 7614 | .buffer = std.ArrayList(u8).init(gpa), | |
| 7615 | }; | |
| 7616 | } | |
| 7617 | ||
| 7618 | 6357 | fn parseOptimizeMode(s: []const u8) std.builtin.OptimizeMode { |
| 7619 | 6358 | return std.meta.stringToEnum(std.builtin.OptimizeMode, s) orelse |
| 7620 | 6359 | fatal("unrecognized optimization mode: '{s}'", .{s}); |
| ... | ... | @@ -7640,7 +6379,7 @@ fn handleModArg( |
| 7640 | 6379 | mod_name: []const u8, |
| 7641 | 6380 | opt_root_src_orig: ?[]const u8, |
| 7642 | 6381 | create_module: *CreateModule, |
| 7643 | mod_opts: *Package.Module.CreateOptions.Inherited, | |
| 6382 | mod_opts: *Module.CreateOptions.Inherited, | |
| 7644 | 6383 | cc_argv: *std.ArrayListUnmanaged([]const u8), |
| 7645 | 6384 | target_arch_os_abi: *?[]const u8, |
| 7646 | 6385 | target_mcpu: *?[]const u8, |