| author |  Andrew Kelley <andrew@ziglang.org> 2023-10-03 23:27:17-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2023-10-08 16:54:31-07:00 |
| log | 88bbec8f9b2f8f023a0177c204f51b8ac0aee83a |
| tree | 57d646f36e131b7ac61c244ca4b4a22dbb0d9a33 |
| parent | d06da958846330ce7c40cf9c9fde1f9799cd30dd |
rework package manager
Organize everything around a Fetch task which does a bunch of stuff in a
worker thread without touching any shared state, and then queues up
Fetch tasks for its dependencies.
This isn't the theoretical optimal package fetching performance because
CPU cores don't necessarily map 1:1 with I/O tasks, and each fetch task
contains a mixture of computations and I/O. However, it is expected for
this to significantly outperform master branch, which fetches everything
recursively with only one thread.
The logic is now a lot more linear and easy to follow. Everything that
is embarassingly parallel is done on the thread pool, and then after
everything is fetched, the worker threads are joined and the main thread
does the finishing touches of stitching together the dependencies.zig
import files. There is only one tiny little critical section and it does
not even have any error handling in it.
This also lays the groundwork for #14281 because in system mode, all
this fetching logic will be skipped, but the "finishing touches"
mentioned above still need to be done. With this branch, that logic is
separated out and no longer recursively tangled with fetching stuff.
Additionally, this branch:
* Implements inclusion directives in `build.zig.zon` for deciding which
files belong the package (#14311).
* Adds basic documentation for `build.zig.zon` files.
* Adds support for fetching dependencies with the `file://` protocol
scheme (#17364).
* Adds a workaround for a Linux/btrfs file system bug (#17282).
This commit is a work-in-progress. Still todo:
1. Hook up the CLI to the new system.
2. Restore the module table creation logic after all the fetching is
done.
3. Fix compilation errors, get the tests passing, and regression test
against real world projects.5 files changed, 1017 insertions(+), 1256 deletions(-)
CMakeLists.txt+1-1
| ... | @@ -528,7 +528,7 @@ set(ZIG_STAGE2_SOURCES | ... | @@ -528,7 +528,7 @@ set(ZIG_STAGE2_SOURCES |
| 528 | "${CMAKE_SOURCE_DIR}/src/Liveness.zig" | 528 | "${CMAKE_SOURCE_DIR}/src/Liveness.zig" |
| 529 | "${CMAKE_SOURCE_DIR}/src/Module.zig" | 529 | "${CMAKE_SOURCE_DIR}/src/Module.zig" |
| 530 | "${CMAKE_SOURCE_DIR}/src/Package.zig" | 530 | "${CMAKE_SOURCE_DIR}/src/Package.zig" |
| 531 | "${CMAKE_SOURCE_DIR}/src/Package/hash.zig" | 531 | "${CMAKE_SOURCE_DIR}/src/Package/Fetch.zig" |
| 532 | "${CMAKE_SOURCE_DIR}/src/RangeSet.zig" | 532 | "${CMAKE_SOURCE_DIR}/src/RangeSet.zig" |
| 533 | "${CMAKE_SOURCE_DIR}/src/Sema.zig" | 533 | "${CMAKE_SOURCE_DIR}/src/Sema.zig" |
| 534 | "${CMAKE_SOURCE_DIR}/src/TypedValue.zig" | 534 | "${CMAKE_SOURCE_DIR}/src/TypedValue.zig" |
src/Package.zig+3-1101
| ... | @@ -15,9 +15,9 @@ const Compilation = @import("Compilation.zig"); | ... | @@ -15,9 +15,9 @@ const Compilation = @import("Compilation.zig"); |
| 15 | const Module = @import("Module.zig"); | 15 | const Module = @import("Module.zig"); |
| 16 | const Cache = std.Build.Cache; | 16 | const Cache = std.Build.Cache; |
| 17 | const build_options = @import("build_options"); | 17 | const build_options = @import("build_options"); |
| 18 | const git = @import("git.zig"); | 18 | const Fetch = @import("Package/Fetch.zig"); |
| 19 | const computePackageHash = @import("Package/hash.zig").compute; | ||
| 20 | 19 | ||
| 20 | pub const build_zig_basename = "build.zig"; | ||
| 21 | pub const Manifest = @import("Manifest.zig"); | 21 | pub const Manifest = @import("Manifest.zig"); |
| 22 | pub const Table = std.StringHashMapUnmanaged(*Package); | 22 | pub const Table = std.StringHashMapUnmanaged(*Package); |
| 23 | 23 | ||
| ... | @@ -213,223 +213,6 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 | ... | @@ -213,223 +213,6 @@ pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 |
| 213 | return buf.toOwnedSlice(); | 213 | return buf.toOwnedSlice(); |
| 214 | } | 214 | } |
| 215 | 215 | ||
| 216 | pub const build_zig_basename = "build.zig"; | ||
| 217 | |||
| 218 | /// Fetches a package and all of its dependencies recursively. Writes the | ||
| 219 | /// corresponding datastructures for the build runner into `dependencies_source`. | ||
| 220 | pub fn fetchAndAddDependencies( | ||
| 221 | pkg: *Package, | ||
| 222 | deps_pkg: *Package, | ||
| 223 | arena: Allocator, | ||
| 224 | thread_pool: *ThreadPool, | ||
| 225 | http_client: *std.http.Client, | ||
| 226 | directory: Compilation.Directory, | ||
| 227 | global_cache_directory: Compilation.Directory, | ||
| 228 | local_cache_directory: Compilation.Directory, | ||
| 229 | dependencies_source: *std.ArrayList(u8), | ||
| 230 | error_bundle: *std.zig.ErrorBundle.Wip, | ||
| 231 | all_modules: *AllModules, | ||
| 232 | root_prog_node: *std.Progress.Node, | ||
| 233 | /// null for the root package | ||
| 234 | this_hash: ?[]const u8, | ||
| 235 | ) !void { | ||
| 236 | const max_bytes = 10 * 1024 * 1024; | ||
| 237 | const gpa = thread_pool.allocator; | ||
| 238 | const build_zig_zon_bytes = directory.handle.readFileAllocOptions( | ||
| 239 | arena, | ||
| 240 | Manifest.basename, | ||
| 241 | max_bytes, | ||
| 242 | null, | ||
| 243 | 1, | ||
| 244 | 0, | ||
| 245 | ) catch |err| switch (err) { | ||
| 246 | error.FileNotFound => { | ||
| 247 | // Handle the same as no dependencies. | ||
| 248 | if (this_hash) |hash| { | ||
| 249 | try dependencies_source.writer().print( | ||
| 250 | \\ pub const {} = struct {{ | ||
| 251 | \\ pub const build_root = "{}"; | ||
| 252 | \\ pub const build_zig = @import("{}"); | ||
| 253 | \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; | ||
| 254 | \\ }}; | ||
| 255 | \\ | ||
| 256 | , .{ | ||
| 257 | std.zig.fmtId(hash), | ||
| 258 | std.zig.fmtEscapes(pkg.root_src_directory.path.?), | ||
| 259 | std.zig.fmtEscapes(hash), | ||
| 260 | }); | ||
| 261 | } else { | ||
| 262 | try dependencies_source.writer().writeAll( | ||
| 263 | \\pub const packages = struct {}; | ||
| 264 | \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; | ||
| 265 | \\ | ||
| 266 | ); | ||
| 267 | } | ||
| 268 | return; | ||
| 269 | }, | ||
| 270 | else => |e| return e, | ||
| 271 | }; | ||
| 272 | |||
| 273 | var ast = try std.zig.Ast.parse(gpa, build_zig_zon_bytes, .zon); | ||
| 274 | defer ast.deinit(gpa); | ||
| 275 | |||
| 276 | if (ast.errors.len > 0) { | ||
| 277 | const file_path = try directory.join(arena, &.{Manifest.basename}); | ||
| 278 | try main.putAstErrorsIntoBundle(gpa, ast, file_path, error_bundle); | ||
| 279 | return error.PackageFetchFailed; | ||
| 280 | } | ||
| 281 | |||
| 282 | var manifest = try Manifest.parse(gpa, ast); | ||
| 283 | defer manifest.deinit(gpa); | ||
| 284 | |||
| 285 | if (manifest.errors.len > 0) { | ||
| 286 | const file_path = try directory.join(arena, &.{Manifest.basename}); | ||
| 287 | for (manifest.errors) |msg| { | ||
| 288 | const str = try error_bundle.addString(msg.msg); | ||
| 289 | try Report.addErrorMessage(&ast, file_path, error_bundle, 0, str, msg.tok, msg.off); | ||
| 290 | } | ||
| 291 | return error.PackageFetchFailed; | ||
| 292 | } | ||
| 293 | |||
| 294 | const report: Report = .{ | ||
| 295 | .ast = &ast, | ||
| 296 | .directory = directory, | ||
| 297 | .error_bundle = error_bundle, | ||
| 298 | }; | ||
| 299 | |||
| 300 | for (manifest.dependencies.values()) |dep| { | ||
| 301 | // If the hash is invalid, let errors happen later | ||
| 302 | // We only want to add these for progress reporting | ||
| 303 | const hash = dep.hash orelse continue; | ||
| 304 | if (hash.len != hex_multihash_len) continue; | ||
| 305 | const gop = try all_modules.getOrPut(gpa, hash[0..hex_multihash_len].*); | ||
| 306 | if (!gop.found_existing) gop.value_ptr.* = null; | ||
| 307 | } | ||
| 308 | |||
| 309 | root_prog_node.setEstimatedTotalItems(all_modules.count()); | ||
| 310 | |||
| 311 | if (this_hash == null) { | ||
| 312 | try dependencies_source.writer().writeAll("pub const packages = struct {\n"); | ||
| 313 | } | ||
| 314 | |||
| 315 | for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, *dep| { | ||
| 316 | var fetch_location = try FetchLocation.init(gpa, dep.*, directory, report); | ||
| 317 | defer fetch_location.deinit(gpa); | ||
| 318 | |||
| 319 | // Directories do not provide a hash in build.zig.zon. | ||
| 320 | // Hash the path to the module rather than its contents. | ||
| 321 | const sub_mod, const found_existing = if (fetch_location == .directory) | ||
| 322 | try getDirectoryModule(gpa, fetch_location, directory, all_modules, dep, report) | ||
| 323 | else | ||
| 324 | try getCachedPackage( | ||
| 325 | gpa, | ||
| 326 | global_cache_directory, | ||
| 327 | dep.*, | ||
| 328 | all_modules, | ||
| 329 | root_prog_node, | ||
| 330 | ) orelse .{ | ||
| 331 | try fetchAndUnpack( | ||
| 332 | fetch_location, | ||
| 333 | thread_pool, | ||
| 334 | http_client, | ||
| 335 | directory, | ||
| 336 | global_cache_directory, | ||
| 337 | dep.*, | ||
| 338 | report, | ||
| 339 | all_modules, | ||
| 340 | root_prog_node, | ||
| 341 | name, | ||
| 342 | ), | ||
| 343 | false, | ||
| 344 | }; | ||
| 345 | |||
| 346 | assert(dep.hash != null); | ||
| 347 | |||
| 348 | switch (sub_mod) { | ||
| 349 | .zig_pkg => |sub_pkg| { | ||
| 350 | if (!found_existing) { | ||
| 351 | try sub_pkg.fetchAndAddDependencies( | ||
| 352 | deps_pkg, | ||
| 353 | arena, | ||
| 354 | thread_pool, | ||
| 355 | http_client, | ||
| 356 | sub_pkg.root_src_directory, | ||
| 357 | global_cache_directory, | ||
| 358 | local_cache_directory, | ||
| 359 | dependencies_source, | ||
| 360 | error_bundle, | ||
| 361 | all_modules, | ||
| 362 | root_prog_node, | ||
| 363 | dep.hash.?, | ||
| 364 | ); | ||
| 365 | } | ||
| 366 | |||
| 367 | try pkg.add(gpa, name, sub_pkg); | ||
| 368 | if (deps_pkg.table.get(dep.hash.?)) |other_sub| { | ||
| 369 | // This should be the same package (and hence module) since it's the same hash | ||
| 370 | // TODO: dedup multiple versions of the same package | ||
| 371 | assert(other_sub == sub_pkg); | ||
| 372 | } else { | ||
| 373 | try deps_pkg.add(gpa, dep.hash.?, sub_pkg); | ||
| 374 | } | ||
| 375 | }, | ||
| 376 | .non_zig_pkg => |sub_pkg| { | ||
| 377 | if (!found_existing) { | ||
| 378 | try dependencies_source.writer().print( | ||
| 379 | \\ pub const {} = struct {{ | ||
| 380 | \\ pub const build_root = "{}"; | ||
| 381 | \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}}; | ||
| 382 | \\ }}; | ||
| 383 | \\ | ||
| 384 | , .{ | ||
| 385 | std.zig.fmtId(dep.hash.?), | ||
| 386 | std.zig.fmtEscapes(sub_pkg.root_src_directory.path.?), | ||
| 387 | }); | ||
| 388 | } | ||
| 389 | }, | ||
| 390 | } | ||
| 391 | } | ||
| 392 | |||
| 393 | if (this_hash) |hash| { | ||
| 394 | try dependencies_source.writer().print( | ||
| 395 | \\ pub const {} = struct {{ | ||
| 396 | \\ pub const build_root = "{}"; | ||
| 397 | \\ pub const build_zig = @import("{}"); | ||
| 398 | \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{ | ||
| 399 | \\ | ||
| 400 | , .{ | ||
| 401 | std.zig.fmtId(hash), | ||
| 402 | std.zig.fmtEscapes(pkg.root_src_directory.path.?), | ||
| 403 | std.zig.fmtEscapes(hash), | ||
| 404 | }); | ||
| 405 | for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { | ||
| 406 | try dependencies_source.writer().print( | ||
| 407 | " .{{ \"{}\", \"{}\" }},\n", | ||
| 408 | .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, | ||
| 409 | ); | ||
| 410 | } | ||
| 411 | try dependencies_source.writer().writeAll( | ||
| 412 | \\ }; | ||
| 413 | \\ }; | ||
| 414 | \\ | ||
| 415 | ); | ||
| 416 | } else { | ||
| 417 | try dependencies_source.writer().writeAll( | ||
| 418 | \\}; | ||
| 419 | \\ | ||
| 420 | \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ | ||
| 421 | \\ | ||
| 422 | ); | ||
| 423 | for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { | ||
| 424 | try dependencies_source.writer().print( | ||
| 425 | " .{{ \"{}\", \"{}\" }},\n", | ||
| 426 | .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) }, | ||
| 427 | ); | ||
| 428 | } | ||
| 429 | try dependencies_source.writer().writeAll("};\n"); | ||
| 430 | } | ||
| 431 | } | ||
| 432 | |||
| 433 | pub fn createFilePkg( | 216 | pub fn createFilePkg( |
| 434 | gpa: Allocator, | 217 | gpa: Allocator, |
| 435 | cache_directory: Compilation.Directory, | 218 | cache_directory: Compilation.Directory, |
| ... | @@ -450,484 +233,11 @@ pub fn createFilePkg( | ... | @@ -450,484 +233,11 @@ pub fn createFilePkg( |
| 450 | const hex_digest = hh.final(); | 233 | const hex_digest = hh.final(); |
| 451 | 234 | ||
| 452 | const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; | 235 | const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; |
| 453 | try renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); | 236 | try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); |
| 454 | 237 | ||
| 455 | return createWithDir(gpa, cache_directory, o_dir_sub_path, basename); | 238 | return createWithDir(gpa, cache_directory, o_dir_sub_path, basename); |
| 456 | } | 239 | } |
| 457 | 240 | ||
| 458 | pub const Report = struct { | ||
| 459 | ast: ?*const std.zig.Ast, | ||
| 460 | directory: Compilation.Directory, | ||
| 461 | error_bundle: *std.zig.ErrorBundle.Wip, | ||
| 462 | |||
| 463 | fn fail( | ||
| 464 | report: Report, | ||
| 465 | tok: std.zig.Ast.TokenIndex, | ||
| 466 | comptime fmt_string: []const u8, | ||
| 467 | fmt_args: anytype, | ||
| 468 | ) error{ PackageFetchFailed, OutOfMemory } { | ||
| 469 | const msg = try report.error_bundle.printString(fmt_string, fmt_args); | ||
| 470 | return failMsg(report, tok, msg); | ||
| 471 | } | ||
| 472 | |||
| 473 | fn failMsg( | ||
| 474 | report: Report, | ||
| 475 | tok: std.zig.Ast.TokenIndex, | ||
| 476 | msg: u32, | ||
| 477 | ) error{ PackageFetchFailed, OutOfMemory } { | ||
| 478 | const gpa = report.error_bundle.gpa; | ||
| 479 | |||
| 480 | const file_path = try report.directory.join(gpa, &.{Manifest.basename}); | ||
| 481 | defer gpa.free(file_path); | ||
| 482 | |||
| 483 | const eb = report.error_bundle; | ||
| 484 | |||
| 485 | if (report.ast) |ast| { | ||
| 486 | try addErrorMessage(ast, file_path, eb, 0, msg, tok, 0); | ||
| 487 | } else { | ||
| 488 | try eb.addRootErrorMessage(.{ | ||
| 489 | .msg = msg, | ||
| 490 | .src_loc = .none, | ||
| 491 | .notes_len = 0, | ||
| 492 | }); | ||
| 493 | } | ||
| 494 | |||
| 495 | return error.PackageFetchFailed; | ||
| 496 | } | ||
| 497 | |||
| 498 | fn addErrorWithNotes( | ||
| 499 | report: Report, | ||
| 500 | notes_len: u32, | ||
| 501 | msg: Manifest.ErrorMessage, | ||
| 502 | ) error{OutOfMemory}!void { | ||
| 503 | const eb = report.error_bundle; | ||
| 504 | const msg_str = try eb.addString(msg.msg); | ||
| 505 | if (report.ast) |ast| { | ||
| 506 | const gpa = eb.gpa; | ||
| 507 | const file_path = try report.directory.join(gpa, &.{Manifest.basename}); | ||
| 508 | defer gpa.free(file_path); | ||
| 509 | return addErrorMessage(ast, file_path, eb, notes_len, msg_str, msg.tok, msg.off); | ||
| 510 | } else { | ||
| 511 | return eb.addRootErrorMessage(.{ | ||
| 512 | .msg = msg_str, | ||
| 513 | .src_loc = .none, | ||
| 514 | .notes_len = notes_len, | ||
| 515 | }); | ||
| 516 | } | ||
| 517 | } | ||
| 518 | |||
| 519 | fn addErrorMessage( | ||
| 520 | ast: *const std.zig.Ast, | ||
| 521 | file_path: []const u8, | ||
| 522 | eb: *std.zig.ErrorBundle.Wip, | ||
| 523 | notes_len: u32, | ||
| 524 | msg_str: u32, | ||
| 525 | msg_tok: std.zig.Ast.TokenIndex, | ||
| 526 | msg_off: u32, | ||
| 527 | ) error{OutOfMemory}!void { | ||
| 528 | const token_starts = ast.tokens.items(.start); | ||
| 529 | const start_loc = ast.tokenLocation(0, msg_tok); | ||
| 530 | |||
| 531 | try eb.addRootErrorMessage(.{ | ||
| 532 | .msg = msg_str, | ||
| 533 | .src_loc = try eb.addSourceLocation(.{ | ||
| 534 | .src_path = try eb.addString(file_path), | ||
| 535 | .span_start = token_starts[msg_tok], | ||
| 536 | .span_end = @as(u32, @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len)), | ||
| 537 | .span_main = token_starts[msg_tok] + msg_off, | ||
| 538 | .line = @intCast(start_loc.line), | ||
| 539 | .column = @as(u32, @intCast(start_loc.column)), | ||
| 540 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | ||
| 541 | }), | ||
| 542 | .notes_len = notes_len, | ||
| 543 | }); | ||
| 544 | } | ||
| 545 | }; | ||
| 546 | |||
| 547 | pub const FetchLocation = union(enum) { | ||
| 548 | /// The relative path to a file or directory. | ||
| 549 | /// This may be a file that requires unpacking (such as a .tar.gz), | ||
| 550 | /// or the path to the root directory of a package. | ||
| 551 | file: []const u8, | ||
| 552 | directory: []const u8, | ||
| 553 | http_request: std.Uri, | ||
| 554 | git_request: std.Uri, | ||
| 555 | |||
| 556 | pub fn init( | ||
| 557 | gpa: Allocator, | ||
| 558 | dep: Manifest.Dependency, | ||
| 559 | root_dir: Compilation.Directory, | ||
| 560 | report: Report, | ||
| 561 | ) !FetchLocation { | ||
| 562 | switch (dep.location) { | ||
| 563 | .url => |url| { | ||
| 564 | const uri = std.Uri.parse(url) catch |err| switch (err) { | ||
| 565 | error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}), | ||
| 566 | else => return err, | ||
| 567 | }; | ||
| 568 | return initUri(uri, dep.location_tok, report); | ||
| 569 | }, | ||
| 570 | .path => |path| { | ||
| 571 | if (fs.path.isAbsolute(path)) { | ||
| 572 | return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{}); | ||
| 573 | } | ||
| 574 | |||
| 575 | const is_dir = isDirectory(root_dir, path) catch |err| switch (err) { | ||
| 576 | error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}), | ||
| 577 | else => return err, | ||
| 578 | }; | ||
| 579 | |||
| 580 | return if (is_dir) | ||
| 581 | .{ .directory = try gpa.dupe(u8, path) } | ||
| 582 | else | ||
| 583 | .{ .file = try gpa.dupe(u8, path) }; | ||
| 584 | }, | ||
| 585 | } | ||
| 586 | } | ||
| 587 | |||
| 588 | pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation { | ||
| 589 | if (ascii.eqlIgnoreCase(uri.scheme, "file")) { | ||
| 590 | return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{}); | ||
| 591 | } else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) { | ||
| 592 | return .{ .http_request = uri }; | ||
| 593 | } else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) { | ||
| 594 | return .{ .git_request = uri }; | ||
| 595 | } else { | ||
| 596 | return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); | ||
| 597 | } | ||
| 598 | } | ||
| 599 | |||
| 600 | pub fn deinit(f: *FetchLocation, gpa: Allocator) void { | ||
| 601 | switch (f.*) { | ||
| 602 | .file, .directory => |path| gpa.free(path), | ||
| 603 | .http_request, .git_request => {}, | ||
| 604 | } | ||
| 605 | f.* = undefined; | ||
| 606 | } | ||
| 607 | |||
| 608 | pub fn fetch( | ||
| 609 | f: FetchLocation, | ||
| 610 | gpa: Allocator, | ||
| 611 | root_dir: Compilation.Directory, | ||
| 612 | http_client: *std.http.Client, | ||
| 613 | dep_location_tok: std.zig.Ast.TokenIndex, | ||
| 614 | report: Report, | ||
| 615 | ) !ReadableResource { | ||
| 616 | switch (f) { | ||
| 617 | .file => |file| { | ||
| 618 | const owned_path = try gpa.dupe(u8, file); | ||
| 619 | errdefer gpa.free(owned_path); | ||
| 620 | return .{ | ||
| 621 | .path = owned_path, | ||
| 622 | .resource = .{ .file = try root_dir.handle.openFile(file, .{}) }, | ||
| 623 | }; | ||
| 624 | }, | ||
| 625 | .http_request => |uri| { | ||
| 626 | var h = std.http.Headers{ .allocator = gpa }; | ||
| 627 | defer h.deinit(); | ||
| 628 | |||
| 629 | var req = try http_client.request(.GET, uri, h, .{}); | ||
| 630 | errdefer req.deinit(); | ||
| 631 | |||
| 632 | try req.start(.{}); | ||
| 633 | try req.wait(); | ||
| 634 | |||
| 635 | if (req.response.status != .ok) { | ||
| 636 | return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{ | ||
| 637 | @intFromEnum(req.response.status), | ||
| 638 | req.response.status.phrase() orelse "", | ||
| 639 | }); | ||
| 640 | } | ||
| 641 | |||
| 642 | return .{ | ||
| 643 | .path = try gpa.dupe(u8, uri.path), | ||
| 644 | .resource = .{ .http_request = req }, | ||
| 645 | }; | ||
| 646 | }, | ||
| 647 | .git_request => |uri| { | ||
| 648 | var transport_uri = uri; | ||
| 649 | transport_uri.scheme = uri.scheme["git+".len..]; | ||
| 650 | var redirect_uri: []u8 = undefined; | ||
| 651 | var session: git.Session = .{ .transport = http_client, .uri = transport_uri }; | ||
| 652 | session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { | ||
| 653 | error.Redirected => { | ||
| 654 | defer gpa.free(redirect_uri); | ||
| 655 | return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri}); | ||
| 656 | }, | ||
| 657 | else => |other| return other, | ||
| 658 | }; | ||
| 659 | |||
| 660 | const want_oid = want_oid: { | ||
| 661 | const want_ref = uri.fragment orelse "HEAD"; | ||
| 662 | if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {} | ||
| 663 | |||
| 664 | const want_ref_head = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{want_ref}); | ||
| 665 | defer gpa.free(want_ref_head); | ||
| 666 | const want_ref_tag = try std.fmt.allocPrint(gpa, "refs/tags/{s}", .{want_ref}); | ||
| 667 | defer gpa.free(want_ref_tag); | ||
| 668 | |||
| 669 | var ref_iterator = try session.listRefs(gpa, .{ | ||
| 670 | .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, | ||
| 671 | .include_peeled = true, | ||
| 672 | }); | ||
| 673 | defer ref_iterator.deinit(); | ||
| 674 | while (try ref_iterator.next()) |ref| { | ||
| 675 | if (mem.eql(u8, ref.name, want_ref) or | ||
| 676 | mem.eql(u8, ref.name, want_ref_head) or | ||
| 677 | mem.eql(u8, ref.name, want_ref_tag)) | ||
| 678 | { | ||
| 679 | break :want_oid ref.peeled orelse ref.oid; | ||
| 680 | } | ||
| 681 | } | ||
| 682 | return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref}); | ||
| 683 | }; | ||
| 684 | if (uri.fragment == null) { | ||
| 685 | const notes_len = 1; | ||
| 686 | try report.addErrorWithNotes(notes_len, .{ | ||
| 687 | .tok = dep_location_tok, | ||
| 688 | .off = 0, | ||
| 689 | .msg = "url field is missing an explicit ref", | ||
| 690 | }); | ||
| 691 | const eb = report.error_bundle; | ||
| 692 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 693 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 694 | .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }), | ||
| 695 | })); | ||
| 696 | return error.PackageFetchFailed; | ||
| 697 | } | ||
| 698 | |||
| 699 | var want_oid_buf: [git.fmt_oid_length]u8 = undefined; | ||
| 700 | _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{std.fmt.fmtSliceHexLower(&want_oid)}) catch unreachable; | ||
| 701 | var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); | ||
| 702 | errdefer fetch_stream.deinit(); | ||
| 703 | |||
| 704 | return .{ | ||
| 705 | .path = try gpa.dupe(u8, &want_oid_buf), | ||
| 706 | .resource = .{ .git_fetch_stream = fetch_stream }, | ||
| 707 | }; | ||
| 708 | }, | ||
| 709 | .directory => unreachable, // Directories do not require fetching | ||
| 710 | } | ||
| 711 | } | ||
| 712 | }; | ||
| 713 | |||
| 714 | pub const ReadableResource = struct { | ||
| 715 | path: []const u8, | ||
| 716 | resource: union(enum) { | ||
| 717 | file: fs.File, | ||
| 718 | http_request: std.http.Client.Request, | ||
| 719 | git_fetch_stream: git.Session.FetchStream, | ||
| 720 | dir: fs.IterableDir, | ||
| 721 | }, | ||
| 722 | |||
| 723 | /// Unpack the package into the global cache directory. | ||
| 724 | /// If `ps` does not require unpacking (for example, if it is a directory), then no caching is performed. | ||
| 725 | /// In either case, the hash is computed and returned along with the path to the package. | ||
| 726 | pub fn unpack( | ||
| 727 | rr: *ReadableResource, | ||
| 728 | allocator: Allocator, | ||
| 729 | thread_pool: *ThreadPool, | ||
| 730 | global_cache_directory: Compilation.Directory, | ||
| 731 | dep_location_tok: std.zig.Ast.TokenIndex, | ||
| 732 | report: Report, | ||
| 733 | pkg_prog_node: *std.Progress.Node, | ||
| 734 | ) !PackageLocation { | ||
| 735 | switch (rr.resource) { | ||
| 736 | inline .file, .http_request, .git_fetch_stream, .dir => |*r, tag| { | ||
| 737 | const s = fs.path.sep_str; | ||
| 738 | const rand_int = std.crypto.random.int(u64); | ||
| 739 | const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); | ||
| 740 | |||
| 741 | const actual_hash = h: { | ||
| 742 | var tmp_directory: Compilation.Directory = d: { | ||
| 743 | const path = try global_cache_directory.join(allocator, &.{tmp_dir_sub_path}); | ||
| 744 | errdefer allocator.free(path); | ||
| 745 | |||
| 746 | const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}); | ||
| 747 | errdefer iterable_dir.close(); | ||
| 748 | |||
| 749 | break :d .{ | ||
| 750 | .path = path, | ||
| 751 | .handle = iterable_dir.dir, | ||
| 752 | }; | ||
| 753 | }; | ||
| 754 | defer tmp_directory.closeAndFree(allocator); | ||
| 755 | |||
| 756 | if (tag != .dir) { | ||
| 757 | const opt_content_length = try rr.getSize(); | ||
| 758 | |||
| 759 | var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{ | ||
| 760 | .child_reader = r.reader(), | ||
| 761 | .prog_node = pkg_prog_node, | ||
| 762 | .unit = if (opt_content_length) |content_length| unit: { | ||
| 763 | const kib = content_length / 1024; | ||
| 764 | const mib = kib / 1024; | ||
| 765 | if (mib > 0) { | ||
| 766 | pkg_prog_node.setEstimatedTotalItems(@intCast(mib)); | ||
| 767 | pkg_prog_node.setUnit("MiB"); | ||
| 768 | break :unit .mib; | ||
| 769 | } else { | ||
| 770 | pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib))); | ||
| 771 | pkg_prog_node.setUnit("KiB"); | ||
| 772 | break :unit .kib; | ||
| 773 | } | ||
| 774 | } else .any, | ||
| 775 | }; | ||
| 776 | |||
| 777 | switch (try rr.getFileType(dep_location_tok, report)) { | ||
| 778 | .tar => try unpackTarball(allocator, prog_reader.reader(), tmp_directory.handle, dep_location_tok, report), | ||
| 779 | .@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.gzip), | ||
| 780 | .@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.xz), | ||
| 781 | .git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle, dep_location_tok, report), | ||
| 782 | } | ||
| 783 | } else { | ||
| 784 | // Recursive directory copy. | ||
| 785 | var it = try r.walk(allocator); | ||
| 786 | defer it.deinit(); | ||
| 787 | while (try it.next()) |entry| { | ||
| 788 | switch (entry.kind) { | ||
| 789 | .directory => try tmp_directory.handle.makePath(entry.path), | ||
| 790 | .file => try r.dir.copyFile( | ||
| 791 | entry.path, | ||
| 792 | tmp_directory.handle, | ||
| 793 | entry.path, | ||
| 794 | .{}, | ||
| 795 | ), | ||
| 796 | .sym_link => { | ||
| 797 | var buf: [fs.MAX_PATH_BYTES]u8 = undefined; | ||
| 798 | const link_name = try r.dir.readLink(entry.path, &buf); | ||
| 799 | // TODO: if this would create a symlink to outside | ||
| 800 | // the destination directory, fail with an error instead. | ||
| 801 | try tmp_directory.handle.symLink(link_name, entry.path, .{}); | ||
| 802 | }, | ||
| 803 | else => return error.IllegalFileTypeInPackage, | ||
| 804 | } | ||
| 805 | } | ||
| 806 | } | ||
| 807 | |||
| 808 | break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); | ||
| 809 | }; | ||
| 810 | |||
| 811 | const pkg_dir_sub_path = "p" ++ s ++ Manifest.hexDigest(actual_hash); | ||
| 812 | const unpacked_path = try global_cache_directory.join(allocator, &.{pkg_dir_sub_path}); | ||
| 813 | defer allocator.free(unpacked_path); | ||
| 814 | |||
| 815 | const relative_unpacked_path = try fs.path.relative(allocator, global_cache_directory.path.?, unpacked_path); | ||
| 816 | errdefer allocator.free(relative_unpacked_path); | ||
| 817 | try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, relative_unpacked_path); | ||
| 818 | |||
| 819 | return .{ | ||
| 820 | .hash = actual_hash, | ||
| 821 | .relative_unpacked_path = relative_unpacked_path, | ||
| 822 | }; | ||
| 823 | }, | ||
| 824 | } | ||
| 825 | } | ||
| 826 | |||
| 827 | const FileType = enum { | ||
| 828 | tar, | ||
| 829 | @"tar.gz", | ||
| 830 | @"tar.xz", | ||
| 831 | git_pack, | ||
| 832 | }; | ||
| 833 | |||
| 834 | pub fn getSize(rr: ReadableResource) !?u64 { | ||
| 835 | switch (rr.resource) { | ||
| 836 | .file => |f| return (try f.metadata()).size(), | ||
| 837 | // TODO: Handle case of chunked content-length | ||
| 838 | .http_request => |req| return req.response.content_length, | ||
| 839 | .git_fetch_stream => |stream| return stream.request.response.content_length, | ||
| 840 | .dir => unreachable, | ||
| 841 | } | ||
| 842 | } | ||
| 843 | |||
| 844 | pub fn getFileType( | ||
| 845 | rr: ReadableResource, | ||
| 846 | dep_location_tok: std.zig.Ast.TokenIndex, | ||
| 847 | report: Report, | ||
| 848 | ) !FileType { | ||
| 849 | switch (rr.resource) { | ||
| 850 | .file => { | ||
| 851 | return fileTypeFromPath(rr.path) orelse | ||
| 852 | return report.fail(dep_location_tok, "unknown file type", .{}); | ||
| 853 | }, | ||
| 854 | .http_request => |req| { | ||
| 855 | const content_type = req.response.headers.getFirstValue("Content-Type") orelse | ||
| 856 | return report.fail(dep_location_tok, "missing 'Content-Type' header", .{}); | ||
| 857 | |||
| 858 | // If the response has a different content type than the URI indicates, override | ||
| 859 | // the previously assumed file type. | ||
| 860 | if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar; | ||
| 861 | |||
| 862 | return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or | ||
| 863 | ascii.eqlIgnoreCase(content_type, "application/x-gzip") or | ||
| 864 | ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) | ||
| 865 | .@"tar.gz" | ||
| 866 | else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) | ||
| 867 | .@"tar.xz" | ||
| 868 | else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) ty: { | ||
| 869 | // support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz | ||
| 870 | // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"' | ||
| 871 | const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse | ||
| 872 | return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{}); | ||
| 873 | break :ty getAttachmentType(content_disposition) orelse | ||
| 874 | return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition}); | ||
| 875 | } else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type}); | ||
| 876 | }, | ||
| 877 | .git_fetch_stream => return .git_pack, | ||
| 878 | .dir => unreachable, | ||
| 879 | } | ||
| 880 | } | ||
| 881 | |||
| 882 | fn fileTypeFromPath(file_path: []const u8) ?FileType { | ||
| 883 | if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; | ||
| 884 | if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; | ||
| 885 | if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; | ||
| 886 | return null; | ||
| 887 | } | ||
| 888 | |||
| 889 | fn getAttachmentType(content_disposition: []const u8) ?FileType { | ||
| 890 | const disposition_type_end = ascii.indexOfIgnoreCase(content_disposition, "attachment;") orelse return null; | ||
| 891 | |||
| 892 | var value_start = ascii.indexOfIgnoreCasePos(content_disposition, disposition_type_end + 1, "filename") orelse return null; | ||
| 893 | value_start += "filename".len; | ||
| 894 | if (content_disposition[value_start] == '*') { | ||
| 895 | value_start += 1; | ||
| 896 | } | ||
| 897 | if (content_disposition[value_start] != '=') return null; | ||
| 898 | value_start += 1; | ||
| 899 | |||
| 900 | var value_end = mem.indexOfPos(u8, content_disposition, value_start, ";") orelse content_disposition.len; | ||
| 901 | if (content_disposition[value_end - 1] == '\"') { | ||
| 902 | value_end -= 1; | ||
| 903 | } | ||
| 904 | return fileTypeFromPath(content_disposition[value_start..value_end]); | ||
| 905 | } | ||
| 906 | |||
| 907 | pub fn deinit(rr: *ReadableResource, gpa: Allocator) void { | ||
| 908 | gpa.free(rr.path); | ||
| 909 | switch (rr.resource) { | ||
| 910 | .file => |file| file.close(), | ||
| 911 | .http_request => |*req| req.deinit(), | ||
| 912 | .git_fetch_stream => |*stream| stream.deinit(), | ||
| 913 | .dir => |*dir| dir.close(), | ||
| 914 | } | ||
| 915 | rr.* = undefined; | ||
| 916 | } | ||
| 917 | }; | ||
| 918 | |||
| 919 | pub const PackageLocation = struct { | ||
| 920 | /// For packages that require unpacking, this is the hash of the package contents. | ||
| 921 | /// For directories, this is the hash of the absolute file path. | ||
| 922 | hash: [Manifest.Hash.digest_length]u8, | ||
| 923 | relative_unpacked_path: []const u8, | ||
| 924 | |||
| 925 | pub fn deinit(pl: *PackageLocation, allocator: Allocator) void { | ||
| 926 | allocator.free(pl.relative_unpacked_path); | ||
| 927 | pl.* = undefined; | ||
| 928 | } | ||
| 929 | }; | ||
| 930 | |||
| 931 | const hex_multihash_len = 2 * Manifest.multihash_len; | 241 | const hex_multihash_len = 2 * Manifest.multihash_len; |
| 932 | const MultiHashHexDigest = [hex_multihash_len]u8; | 242 | const MultiHashHexDigest = [hex_multihash_len]u8; |
| 933 | 243 | ||
| ... | @@ -939,411 +249,3 @@ const DependencyModule = union(enum) { | ... | @@ -939,411 +249,3 @@ const DependencyModule = union(enum) { |
| 939 | /// If the value is `null`, the package is a known dependency, but has not yet | 249 | /// If the value is `null`, the package is a known dependency, but has not yet |
| 940 | /// been fetched. | 250 | /// been fetched. |
| 941 | pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule); | 251 | pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule); |
| 942 | |||
| 943 | fn ProgressReader(comptime ReaderType: type) type { | ||
| 944 | return struct { | ||
| 945 | child_reader: ReaderType, | ||
| 946 | bytes_read: u64 = 0, | ||
| 947 | prog_node: *std.Progress.Node, | ||
| 948 | unit: enum { | ||
| 949 | kib, | ||
| 950 | mib, | ||
| 951 | any, | ||
| 952 | }, | ||
| 953 | |||
| 954 | pub const Error = ReaderType.Error; | ||
| 955 | pub const Reader = std.io.Reader(*@This(), Error, read); | ||
| 956 | |||
| 957 | pub fn read(self: *@This(), buf: []u8) Error!usize { | ||
| 958 | const amt = try self.child_reader.read(buf); | ||
| 959 | self.bytes_read += amt; | ||
| 960 | const kib = self.bytes_read / 1024; | ||
| 961 | const mib = kib / 1024; | ||
| 962 | switch (self.unit) { | ||
| 963 | .kib => self.prog_node.setCompletedItems(@intCast(kib)), | ||
| 964 | .mib => self.prog_node.setCompletedItems(@intCast(mib)), | ||
| 965 | .any => { | ||
| 966 | if (mib > 0) { | ||
| 967 | self.prog_node.setUnit("MiB"); | ||
| 968 | self.prog_node.setCompletedItems(@intCast(mib)); | ||
| 969 | } else { | ||
| 970 | self.prog_node.setUnit("KiB"); | ||
| 971 | self.prog_node.setCompletedItems(@intCast(kib)); | ||
| 972 | } | ||
| 973 | }, | ||
| 974 | } | ||
| 975 | self.prog_node.activate(); | ||
| 976 | return amt; | ||
| 977 | } | ||
| 978 | |||
| 979 | pub fn reader(self: *@This()) Reader { | ||
| 980 | return .{ .context = self }; | ||
| 981 | } | ||
| 982 | }; | ||
| 983 | } | ||
| 984 | |||
| 985 | /// Get a cached package if it exists. | ||
| 986 | /// Returns `null` if the package has not been cached | ||
| 987 | /// If the package exists in the cache, returns a pointer to the package and a | ||
| 988 | /// boolean indicating whether this package has already been seen in the build | ||
| 989 | /// (i.e. whether or not its transitive dependencies have been fetched). | ||
| 990 | fn getCachedPackage( | ||
| 991 | gpa: Allocator, | ||
| 992 | global_cache_directory: Compilation.Directory, | ||
| 993 | dep: Manifest.Dependency, | ||
| 994 | all_modules: *AllModules, | ||
| 995 | root_prog_node: *std.Progress.Node, | ||
| 996 | ) !?struct { DependencyModule, bool } { | ||
| 997 | const s = fs.path.sep_str; | ||
| 998 | // Check if the expected_hash is already present in the global package | ||
| 999 | // cache, and thereby avoid both fetching and unpacking. | ||
| 1000 | if (dep.hash) |h| { | ||
| 1001 | const hex_digest = h[0..hex_multihash_len]; | ||
| 1002 | const pkg_dir_sub_path = "p" ++ s ++ hex_digest; | ||
| 1003 | |||
| 1004 | var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { | ||
| 1005 | error.FileNotFound => return null, | ||
| 1006 | else => |e| return e, | ||
| 1007 | }; | ||
| 1008 | errdefer pkg_dir.close(); | ||
| 1009 | |||
| 1010 | // The compiler has a rule that a file must not be included in multiple modules, | ||
| 1011 | // so we must detect if a module has been created for this package and reuse it. | ||
| 1012 | const gop = try all_modules.getOrPut(gpa, hex_digest.*); | ||
| 1013 | if (gop.found_existing) { | ||
| 1014 | if (gop.value_ptr.*) |mod| { | ||
| 1015 | return .{ mod, true }; | ||
| 1016 | } | ||
| 1017 | } | ||
| 1018 | |||
| 1019 | root_prog_node.completeOne(); | ||
| 1020 | |||
| 1021 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; | ||
| 1022 | const basename = if (is_zig_mod) build_zig_basename else ""; | ||
| 1023 | const pkg = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, basename); | ||
| 1024 | |||
| 1025 | const module: DependencyModule = if (is_zig_mod) | ||
| 1026 | .{ .zig_pkg = pkg } | ||
| 1027 | else | ||
| 1028 | .{ .non_zig_pkg = pkg }; | ||
| 1029 | |||
| 1030 | try all_modules.put(gpa, hex_digest.*, module); | ||
| 1031 | return .{ module, false }; | ||
| 1032 | } | ||
| 1033 | |||
| 1034 | return null; | ||
| 1035 | } | ||
| 1036 | |||
| 1037 | fn getDirectoryModule( | ||
| 1038 | gpa: Allocator, | ||
| 1039 | fetch_location: FetchLocation, | ||
| 1040 | directory: Compilation.Directory, | ||
| 1041 | all_modules: *AllModules, | ||
| 1042 | dep: *Manifest.Dependency, | ||
| 1043 | report: Report, | ||
| 1044 | ) !struct { DependencyModule, bool } { | ||
| 1045 | assert(fetch_location == .directory); | ||
| 1046 | |||
| 1047 | if (dep.hash != null) { | ||
| 1048 | return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); | ||
| 1049 | } | ||
| 1050 | |||
| 1051 | const hash = try computePathHash(gpa, directory, fetch_location.directory); | ||
| 1052 | const hex_digest = Manifest.hexDigest(hash); | ||
| 1053 | dep.hash = try gpa.dupe(u8, &hex_digest); | ||
| 1054 | |||
| 1055 | // There is no fixed location to check for directory modules. | ||
| 1056 | // Instead, check whether it is already listed in all_modules. | ||
| 1057 | if (all_modules.get(hex_digest)) |mod| return .{ mod.?, true }; | ||
| 1058 | |||
| 1059 | var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch |err| switch (err) { | ||
| 1060 | error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}), | ||
| 1061 | else => |e| return e, | ||
| 1062 | }; | ||
| 1063 | defer pkg_dir.close(); | ||
| 1064 | |||
| 1065 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; | ||
| 1066 | const basename = if (is_zig_mod) build_zig_basename else ""; | ||
| 1067 | |||
| 1068 | const pkg = try createWithDir(gpa, directory, fetch_location.directory, basename); | ||
| 1069 | const module: DependencyModule = if (is_zig_mod) | ||
| 1070 | .{ .zig_pkg = pkg } | ||
| 1071 | else | ||
| 1072 | .{ .non_zig_pkg = pkg }; | ||
| 1073 | |||
| 1074 | try all_modules.put(gpa, hex_digest, module); | ||
| 1075 | return .{ module, false }; | ||
| 1076 | } | ||
| 1077 | |||
| 1078 | fn fetchAndUnpack( | ||
| 1079 | fetch_location: FetchLocation, | ||
| 1080 | thread_pool: *ThreadPool, | ||
| 1081 | http_client: *std.http.Client, | ||
| 1082 | directory: Compilation.Directory, | ||
| 1083 | global_cache_directory: Compilation.Directory, | ||
| 1084 | dep: Manifest.Dependency, | ||
| 1085 | report: Report, | ||
| 1086 | all_modules: *AllModules, | ||
| 1087 | root_prog_node: *std.Progress.Node, | ||
| 1088 | /// This does not have to be any form of canonical or fully-qualified name: it | ||
| 1089 | /// is only intended to be human-readable for progress reporting. | ||
| 1090 | name_for_prog: []const u8, | ||
| 1091 | ) !DependencyModule { | ||
| 1092 | assert(fetch_location != .directory); | ||
| 1093 | |||
| 1094 | const gpa = http_client.allocator; | ||
| 1095 | |||
| 1096 | var pkg_prog_node = root_prog_node.start(name_for_prog, 0); | ||
| 1097 | defer pkg_prog_node.end(); | ||
| 1098 | pkg_prog_node.activate(); | ||
| 1099 | |||
| 1100 | var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report); | ||
| 1101 | defer readable_resource.deinit(gpa); | ||
| 1102 | |||
| 1103 | var package_location = try readable_resource.unpack( | ||
| 1104 | gpa, | ||
| 1105 | thread_pool, | ||
| 1106 | global_cache_directory, | ||
| 1107 | dep.location_tok, | ||
| 1108 | report, | ||
| 1109 | &pkg_prog_node, | ||
| 1110 | ); | ||
| 1111 | defer package_location.deinit(gpa); | ||
| 1112 | |||
| 1113 | const actual_hex = Manifest.hexDigest(package_location.hash); | ||
| 1114 | if (dep.hash) |h| { | ||
| 1115 | if (!mem.eql(u8, h, &actual_hex)) { | ||
| 1116 | return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ | ||
| 1117 | h, actual_hex, | ||
| 1118 | }); | ||
| 1119 | } | ||
| 1120 | } else { | ||
| 1121 | const notes_len = 1; | ||
| 1122 | try report.addErrorWithNotes(notes_len, .{ | ||
| 1123 | .tok = dep.location_tok, | ||
| 1124 | .off = 0, | ||
| 1125 | .msg = "dependency is missing hash field", | ||
| 1126 | }); | ||
| 1127 | const eb = report.error_bundle; | ||
| 1128 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 1129 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 1130 | .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), | ||
| 1131 | })); | ||
| 1132 | return error.PackageFetchFailed; | ||
| 1133 | } | ||
| 1134 | |||
| 1135 | const build_zig_path = try fs.path.join(gpa, &.{ package_location.relative_unpacked_path, build_zig_basename }); | ||
| 1136 | defer gpa.free(build_zig_path); | ||
| 1137 | |||
| 1138 | const is_zig_mod = if (global_cache_directory.handle.access(build_zig_path, .{})) |_| true else |_| false; | ||
| 1139 | const basename = if (is_zig_mod) build_zig_basename else ""; | ||
| 1140 | const pkg = try createWithDir(gpa, global_cache_directory, package_location.relative_unpacked_path, basename); | ||
| 1141 | const module: DependencyModule = if (is_zig_mod) | ||
| 1142 | .{ .zig_pkg = pkg } | ||
| 1143 | else | ||
| 1144 | .{ .non_zig_pkg = pkg }; | ||
| 1145 | |||
| 1146 | try all_modules.put(gpa, actual_hex, module); | ||
| 1147 | return module; | ||
| 1148 | } | ||
| 1149 | |||
| 1150 | fn unpackTarballCompressed( | ||
| 1151 | gpa: Allocator, | ||
| 1152 | reader: anytype, | ||
| 1153 | out_dir: fs.Dir, | ||
| 1154 | dep_location_tok: std.zig.Ast.TokenIndex, | ||
| 1155 | report: Report, | ||
| 1156 | comptime Compression: type, | ||
| 1157 | ) !void { | ||
| 1158 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | ||
| 1159 | |||
| 1160 | var decompress = try Compression.decompress(gpa, br.reader()); | ||
| 1161 | defer decompress.deinit(); | ||
| 1162 | |||
| 1163 | return unpackTarball(gpa, decompress.reader(), out_dir, dep_location_tok, report); | ||
| 1164 | } | ||
| 1165 | |||
| 1166 | fn unpackTarball( | ||
| 1167 | gpa: Allocator, | ||
| 1168 | reader: anytype, | ||
| 1169 | out_dir: fs.Dir, | ||
| 1170 | dep_location_tok: std.zig.Ast.TokenIndex, | ||
| 1171 | report: Report, | ||
| 1172 | ) !void { | ||
| 1173 | var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa }; | ||
| 1174 | defer diagnostics.deinit(); | ||
| 1175 | |||
| 1176 | try std.tar.pipeToFileSystem(out_dir, reader, .{ | ||
| 1177 | .diagnostics = &diagnostics, | ||
| 1178 | .strip_components = 1, | ||
| 1179 | // TODO: we would like to set this to executable_bit_only, but two | ||
| 1180 | // things need to happen before that: | ||
| 1181 | // 1. the tar implementation needs to support it | ||
| 1182 | // 2. the hashing algorithm here needs to support detecting the is_executable | ||
| 1183 | // bit on Windows from the ACLs (see the isExecutable function). | ||
| 1184 | .mode_mode = .ignore, | ||
| 1185 | }); | ||
| 1186 | |||
| 1187 | if (diagnostics.errors.items.len > 0) { | ||
| 1188 | const notes_len: u32 = @intCast(diagnostics.errors.items.len); | ||
| 1189 | try report.addErrorWithNotes(notes_len, .{ | ||
| 1190 | .tok = dep_location_tok, | ||
| 1191 | .off = 0, | ||
| 1192 | .msg = "unable to unpack tarball", | ||
| 1193 | }); | ||
| 1194 | const eb = report.error_bundle; | ||
| 1195 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 1196 | for (diagnostics.errors.items, notes_start..) |item, note_i| { | ||
| 1197 | switch (item) { | ||
| 1198 | .unable_to_create_sym_link => |info| { | ||
| 1199 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 1200 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | ||
| 1201 | info.file_name, info.link_name, @errorName(info.code), | ||
| 1202 | }), | ||
| 1203 | })); | ||
| 1204 | }, | ||
| 1205 | .unsupported_file_type => |info| { | ||
| 1206 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 1207 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ | ||
| 1208 | info.file_name, @intFromEnum(info.file_type), | ||
| 1209 | }), | ||
| 1210 | })); | ||
| 1211 | }, | ||
| 1212 | } | ||
| 1213 | } | ||
| 1214 | return error.InvalidTarball; | ||
| 1215 | } | ||
| 1216 | } | ||
| 1217 | |||
| 1218 | fn unpackGitPack( | ||
| 1219 | gpa: Allocator, | ||
| 1220 | reader: anytype, | ||
| 1221 | want_oid: git.Oid, | ||
| 1222 | out_dir: fs.Dir, | ||
| 1223 | dep_location_tok: std.zig.Ast.TokenIndex, | ||
| 1224 | report: Report, | ||
| 1225 | ) !void { | ||
| 1226 | // The .git directory is used to store the packfile and associated index, but | ||
| 1227 | // we do not attempt to replicate the exact structure of a real .git | ||
| 1228 | // directory, since that isn't relevant for fetching a package. | ||
| 1229 | { | ||
| 1230 | var pack_dir = try out_dir.makeOpenPath(".git", .{}); | ||
| 1231 | defer pack_dir.close(); | ||
| 1232 | var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); | ||
| 1233 | defer pack_file.close(); | ||
| 1234 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | ||
| 1235 | try fifo.pump(reader.reader(), pack_file.writer()); | ||
| 1236 | try pack_file.sync(); | ||
| 1237 | |||
| 1238 | var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); | ||
| 1239 | defer index_file.close(); | ||
| 1240 | { | ||
| 1241 | var index_prog_node = reader.prog_node.start("Index pack", 0); | ||
| 1242 | defer index_prog_node.end(); | ||
| 1243 | index_prog_node.activate(); | ||
| 1244 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); | ||
| 1245 | try git.indexPack(gpa, pack_file, index_buffered_writer.writer()); | ||
| 1246 | try index_buffered_writer.flush(); | ||
| 1247 | try index_file.sync(); | ||
| 1248 | } | ||
| 1249 | |||
| 1250 | { | ||
| 1251 | var checkout_prog_node = reader.prog_node.start("Checkout", 0); | ||
| 1252 | defer checkout_prog_node.end(); | ||
| 1253 | checkout_prog_node.activate(); | ||
| 1254 | var repository = try git.Repository.init(gpa, pack_file, index_file); | ||
| 1255 | defer repository.deinit(); | ||
| 1256 | var diagnostics: git.Diagnostics = .{ .allocator = gpa }; | ||
| 1257 | defer diagnostics.deinit(); | ||
| 1258 | try repository.checkout(out_dir, want_oid, &diagnostics); | ||
| 1259 | |||
| 1260 | if (diagnostics.errors.items.len > 0) { | ||
| 1261 | const notes_len: u32 = @intCast(diagnostics.errors.items.len); | ||
| 1262 | try report.addErrorWithNotes(notes_len, .{ | ||
| 1263 | .tok = dep_location_tok, | ||
| 1264 | .off = 0, | ||
| 1265 | .msg = "unable to unpack packfile", | ||
| 1266 | }); | ||
| 1267 | const eb = report.error_bundle; | ||
| 1268 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 1269 | for (diagnostics.errors.items, notes_start..) |item, note_i| { | ||
| 1270 | switch (item) { | ||
| 1271 | .unable_to_create_sym_link => |info| { | ||
| 1272 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 1273 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | ||
| 1274 | info.file_name, info.link_name, @errorName(info.code), | ||
| 1275 | }), | ||
| 1276 | })); | ||
| 1277 | }, | ||
| 1278 | } | ||
| 1279 | } | ||
| 1280 | return error.InvalidGitPack; | ||
| 1281 | } | ||
| 1282 | } | ||
| 1283 | } | ||
| 1284 | |||
| 1285 | try out_dir.deleteTree(".git"); | ||
| 1286 | } | ||
| 1287 | |||
| 1288 | /// Compute the hash of a file path. | ||
| 1289 | fn computePathHash(gpa: Allocator, dir: Compilation.Directory, path: []const u8) ![Manifest.Hash.digest_length]u8 { | ||
| 1290 | const resolved_path = try std.fs.path.resolve(gpa, &.{ dir.path.?, path }); | ||
| 1291 | defer gpa.free(resolved_path); | ||
| 1292 | var hasher = Manifest.Hash.init(.{}); | ||
| 1293 | hasher.update(resolved_path); | ||
| 1294 | return hasher.finalResult(); | ||
| 1295 | } | ||
| 1296 | |||
| 1297 | fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool { | ||
| 1298 | var dir = root_dir.handle.openDir(path, .{}) catch |err| switch (err) { | ||
| 1299 | error.NotDir => return false, | ||
| 1300 | else => return err, | ||
| 1301 | }; | ||
| 1302 | defer dir.close(); | ||
| 1303 | return true; | ||
| 1304 | } | ||
| 1305 | |||
| 1306 | fn renameTmpIntoCache( | ||
| 1307 | cache_dir: fs.Dir, | ||
| 1308 | tmp_dir_sub_path: []const u8, | ||
| 1309 | dest_dir_sub_path: []const u8, | ||
| 1310 | ) !void { | ||
| 1311 | assert(dest_dir_sub_path[1] == fs.path.sep); | ||
| 1312 | var handled_missing_dir = false; | ||
| 1313 | while (true) { | ||
| 1314 | cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { | ||
| 1315 | error.FileNotFound => { | ||
| 1316 | if (handled_missing_dir) return err; | ||
| 1317 | cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { | ||
| 1318 | error.PathAlreadyExists => handled_missing_dir = true, | ||
| 1319 | else => |e| return e, | ||
| 1320 | }; | ||
| 1321 | continue; | ||
| 1322 | }, | ||
| 1323 | error.PathAlreadyExists, error.AccessDenied => { | ||
| 1324 | // Package has been already downloaded and may already be in use on the system. | ||
| 1325 | cache_dir.deleteTree(tmp_dir_sub_path) catch |del_err| { | ||
| 1326 | std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); | ||
| 1327 | }; | ||
| 1328 | }, | ||
| 1329 | else => |e| return e, | ||
| 1330 | }; | ||
| 1331 | break; | ||
| 1332 | } | ||
| 1333 | } | ||
| 1334 | |||
| 1335 | test "getAttachmentType" { | ||
| 1336 | try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); | ||
| 1337 | try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; filename*=\"stuff.tar.gz\"")); | ||
| 1338 | try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("ATTACHMENT; filename=\"stuff.tar.xz\"")); | ||
| 1339 | try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar.xz\"")); | ||
| 1340 | try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); | ||
| 1341 | |||
| 1342 | try std.testing.expect(ReadableResource.getAttachmentType("attachment FileName=\"stuff.tar.gz\"") == null); | ||
| 1343 | try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar\"") == null); | ||
| 1344 | try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName\"stuff.gz\"") == null); | ||
| 1345 | try std.testing.expect(ReadableResource.getAttachmentType("attachment; size=42") == null); | ||
| 1346 | try std.testing.expect(ReadableResource.getAttachmentType("inline; size=42") == null); | ||
| 1347 | try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\"; attachment;") == null); | ||
| 1348 | try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\";") == null); | ||
| 1349 | } |
src/Package/Fetch.zig created+1012
| ... | @@ -0,0 +1,1012 @@ | ||
| 1 | //! Represents one independent job whose responsibility is to: | ||
| 2 | //! | ||
| 3 | //! 1. Check the global zig package cache to see if the hash already exists. | ||
| 4 | //! If so, load, parse, and validate the build.zig.zon file therein, and | ||
| 5 | //! goto step 8. Likewise if the location is a relative path, treat this | ||
| 6 | //! the same as a cache hit. Otherwise, proceed. | ||
| 7 | //! 2. Fetch and unpack a URL into a temporary directory. | ||
| 8 | //! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed | ||
| 9 | //! for the file to be missing, in which case this fetched package is considered | ||
| 10 | //! to be a "naked" package. | ||
| 11 | //! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by | ||
| 12 | //! deleting excluded files. If any files had errors for files that were | ||
| 13 | //! ultimately excluded, those errors should be ignored, such as failure to | ||
| 14 | //! create symlinks that weren't supposed to be included anyway. | ||
| 15 | //! 5. Compute the package hash based on the remaining files in the temporary | ||
| 16 | //! directory. | ||
| 17 | //! 6. Rename the temporary directory into the global zig package cache | ||
| 18 | //! directory. If the hash already exists, delete the temporary directory and | ||
| 19 | //! leave the zig package cache directory untouched as it may be in use by the | ||
| 20 | //! system. This is done even if the hash is invalid, in case the package with | ||
| 21 | //! the different hash is used in the future. | ||
| 22 | //! 7. Validate the computed hash against the expected hash. If invalid, | ||
| 23 | //! this job is done. | ||
| 24 | //! 8. Spawn a new fetch job for each dependency in the manifest file. Use | ||
| 25 | //! a mutex and a hash map so that redundant jobs do not get queued up. | ||
| 26 | //! | ||
| 27 | //! All of this must be done with only referring to the state inside this struct | ||
| 28 | //! because this work will be done in a dedicated thread. | ||
| 29 | |||
| 30 | /// Try to avoid this as much as possible since arena will have less contention. | ||
| 31 | gpa: Allocator, | ||
| 32 | arena: std.heap.ArenaAllocator, | ||
| 33 | location: Location, | ||
| 34 | location_tok: std.zig.Ast.TokenIndex, | ||
| 35 | hash_tok: std.zig.Ast.TokenIndex, | ||
| 36 | global_cache: Cache.Directory, | ||
| 37 | parent_package_root: Path, | ||
| 38 | parent_manifest_ast: ?*const std.zig.Ast, | ||
| 39 | prog_node: *std.Progress.Node, | ||
| 40 | http_client: *std.http.Client, | ||
| 41 | thread_pool: *ThreadPool, | ||
| 42 | job_queue: *JobQueue, | ||
| 43 | wait_group: *WaitGroup, | ||
| 44 | |||
| 45 | // Above this are fields provided as inputs to `run`. | ||
| 46 | // Below this are fields populated by `run`. | ||
| 47 | |||
| 48 | /// This will either be relative to `global_cache`, or to the build root of | ||
| 49 | /// the root package. | ||
| 50 | package_root: Path, | ||
| 51 | error_bundle: std.zig.ErrorBundle.Wip, | ||
| 52 | manifest: ?Manifest, | ||
| 53 | manifest_ast: ?*std.zig.Ast, | ||
| 54 | actual_hash: Digest, | ||
| 55 | /// Fetch logic notices whether a package has a build.zig file and sets this flag. | ||
| 56 | has_build_zig: bool, | ||
| 57 | /// Indicates whether the task aborted due to an out-of-memory condition. | ||
| 58 | oom_flag: bool, | ||
| 59 | |||
| 60 | pub const JobQueue = struct { | ||
| 61 | mutex: std.Thread.Mutex = .{}, | ||
| 62 | }; | ||
| 63 | |||
| 64 | pub const Digest = [Manifest.Hash.digest_length]u8; | ||
| 65 | pub const MultiHashHexDigest = [hex_multihash_len]u8; | ||
| 66 | |||
| 67 | pub const Path = struct { | ||
| 68 | root_dir: Cache.Directory, | ||
| 69 | /// The path, relative to the root dir, that this `Path` represents. | ||
| 70 | /// Empty string means the root_dir is the path. | ||
| 71 | sub_path: []const u8 = "", | ||
| 72 | }; | ||
| 73 | |||
| 74 | pub const Location = union(enum) { | ||
| 75 | remote: Remote, | ||
| 76 | relative_path: []const u8, | ||
| 77 | |||
| 78 | pub const Remote = struct { | ||
| 79 | url: []const u8, | ||
| 80 | /// If this is null it means the user omitted the hash field from a dependency. | ||
| 81 | /// It will be an error but the logic should still fetch and print the discovered hash. | ||
| 82 | hash: ?[hex_multihash_len]u8, | ||
| 83 | }; | ||
| 84 | }; | ||
| 85 | |||
| 86 | pub const RunError = error{ | ||
| 87 | OutOfMemory, | ||
| 88 | /// This error code is intended to be handled by inspecting the | ||
| 89 | /// `error_bundle` field. | ||
| 90 | FetchFailed, | ||
| 91 | }; | ||
| 92 | |||
| 93 | pub fn run(f: *Fetch) RunError!void { | ||
| 94 | const eb = &f.error_bundle; | ||
| 95 | const arena = f.arena_allocator.allocator(); | ||
| 96 | |||
| 97 | // Check the global zig package cache to see if the hash already exists. If | ||
| 98 | // so, load, parse, and validate the build.zig.zon file therein, and skip | ||
| 99 | // ahead to queuing up jobs for dependencies. Likewise if the location is a | ||
| 100 | // relative path, treat this the same as a cache hit. Otherwise, proceed. | ||
| 101 | |||
| 102 | const remote = switch (f.location) { | ||
| 103 | .relative_path => |sub_path| { | ||
| 104 | if (fs.path.isAbsolute(sub_path)) return f.fail( | ||
| 105 | f.location_tok, | ||
| 106 | try eb.addString("expected path relative to build root; found absolute path"), | ||
| 107 | ); | ||
| 108 | if (f.hash_tok != 0) return f.fail( | ||
| 109 | f.hash_tok, | ||
| 110 | try eb.addString("path-based dependencies are not hashed"), | ||
| 111 | ); | ||
| 112 | f.package_root = try f.parent_package_root.join(arena, sub_path); | ||
| 113 | try loadManifest(f, f.package_root); | ||
| 114 | // Package hashes are used as unique identifiers for packages, so | ||
| 115 | // we still need one for relative paths. | ||
| 116 | const hash = h: { | ||
| 117 | var hasher = Manifest.Hash.init(.{}); | ||
| 118 | // This hash is a tuple of: | ||
| 119 | // * whether it relative to the global cache directory or to the root package | ||
| 120 | // * the relative file path from there to the build root of the package | ||
| 121 | hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle) | ||
| 122 | &package_hash_prefix_cached | ||
| 123 | else | ||
| 124 | &package_hash_prefix_project); | ||
| 125 | hasher.update(f.package_root.sub_path); | ||
| 126 | break :h hasher.finalResult(); | ||
| 127 | }; | ||
| 128 | return queueJobsForDeps(f, hash); | ||
| 129 | }, | ||
| 130 | .remote => |remote| remote, | ||
| 131 | }; | ||
| 132 | const s = fs.path.sep_str; | ||
| 133 | if (remote.hash) |expected_hash| { | ||
| 134 | const pkg_sub_path = "p" ++ s ++ expected_hash; | ||
| 135 | if (f.global_cache.handle.access(pkg_sub_path, .{})) |_| { | ||
| 136 | f.package_root = .{ | ||
| 137 | .root_dir = f.global_cache, | ||
| 138 | .sub_path = pkg_sub_path, | ||
| 139 | }; | ||
| 140 | try loadManifest(f, f.package_root); | ||
| 141 | return queueJobsForDeps(f, expected_hash); | ||
| 142 | } else |err| switch (err) { | ||
| 143 | error.FileNotFound => {}, | ||
| 144 | else => |e| { | ||
| 145 | try eb.addRootErrorMessage(.{ | ||
| 146 | .msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{ | ||
| 147 | try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e), | ||
| 148 | }), | ||
| 149 | .src_loc = .none, | ||
| 150 | .notes_len = 0, | ||
| 151 | }); | ||
| 152 | return error.FetchFailed; | ||
| 153 | }, | ||
| 154 | } | ||
| 155 | } | ||
| 156 | |||
| 157 | // Fetch and unpack the remote into a temporary directory. | ||
| 158 | |||
| 159 | const uri = std.Uri.parse(remote.url) catch |err| return f.fail( | ||
| 160 | f.location_tok, | ||
| 161 | "invalid URI: {s}", | ||
| 162 | .{@errorName(err)}, | ||
| 163 | ); | ||
| 164 | const rand_int = std.crypto.random.int(u64); | ||
| 165 | const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); | ||
| 166 | |||
| 167 | var tmp_directory: Cache.Directory = .{ | ||
| 168 | .path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}), | ||
| 169 | .handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir, | ||
| 170 | }; | ||
| 171 | defer tmp_directory.handle.close(); | ||
| 172 | |||
| 173 | var resource = try f.initResource(uri); | ||
| 174 | defer resource.deinit(); // releases more than memory | ||
| 175 | |||
| 176 | try f.unpackResource(&resource, uri.path, tmp_directory); | ||
| 177 | |||
| 178 | // Load, parse, and validate the unpacked build.zig.zon file. It is allowed | ||
| 179 | // for the file to be missing, in which case this fetched package is | ||
| 180 | // considered to be a "naked" package. | ||
| 181 | try loadManifest(f, .{ .root_dir = tmp_directory }); | ||
| 182 | |||
| 183 | // Apply the manifest's inclusion rules to the temporary directory by | ||
| 184 | // deleting excluded files. If any error occurred for files that were | ||
| 185 | // ultimately excluded, those errors should be ignored, such as failure to | ||
| 186 | // create symlinks that weren't supposed to be included anyway. | ||
| 187 | |||
| 188 | // Empty directories have already been omitted by `unpackResource`. | ||
| 189 | |||
| 190 | const filter: Filter = .{ | ||
| 191 | .include_paths = if (f.manifest) |m| m.paths else .{}, | ||
| 192 | }; | ||
| 193 | |||
| 194 | // Compute the package hash based on the remaining files in the temporary | ||
| 195 | // directory. | ||
| 196 | |||
| 197 | if (builtin.os.tag == .linux and f.work_around_btrfs_bug) { | ||
| 198 | // https://github.com/ziglang/zig/issues/17095 | ||
| 199 | tmp_directory.handle.close(); | ||
| 200 | const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch | ||
| 201 | @panic("btrfs workaround failed"); | ||
| 202 | tmp_directory.handle = iterable_dir.dir; | ||
| 203 | } | ||
| 204 | |||
| 205 | f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter); | ||
| 206 | |||
| 207 | // Rename the temporary directory into the global zig package cache | ||
| 208 | // directory. If the hash already exists, delete the temporary directory | ||
| 209 | // and leave the zig package cache directory untouched as it may be in use | ||
| 210 | // by the system. This is done even if the hash is invalid, in case the | ||
| 211 | // package with the different hash is used in the future. | ||
| 212 | |||
| 213 | const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash); | ||
| 214 | try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path); | ||
| 215 | |||
| 216 | // Validate the computed hash against the expected hash. If invalid, this | ||
| 217 | // job is done. | ||
| 218 | |||
| 219 | const actual_hex = Manifest.hexDigest(f.actual_hash); | ||
| 220 | if (remote.hash) |declared_hash| { | ||
| 221 | if (!std.mem.eql(u8, declared_hash, &actual_hex)) { | ||
| 222 | return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{ | ||
| 223 | declared_hash, actual_hex, | ||
| 224 | }); | ||
| 225 | } | ||
| 226 | } else { | ||
| 227 | const notes_len = 1; | ||
| 228 | try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field"); | ||
| 229 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 230 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 231 | .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), | ||
| 232 | })); | ||
| 233 | return error.PackageFetchFailed; | ||
| 234 | } | ||
| 235 | |||
| 236 | // Spawn a new fetch job for each dependency in the manifest file. Use | ||
| 237 | // a mutex and a hash map so that redundant jobs do not get queued up. | ||
| 238 | return queueJobsForDeps(f, .{ .hash = f.actual_hash }); | ||
| 239 | } | ||
| 240 | |||
| 241 | /// This function populates `f.manifest` or leaves it `null`. | ||
| 242 | fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { | ||
| 243 | const eb = &f.error_bundle; | ||
| 244 | const arena = f.arena_allocator.allocator(); | ||
| 245 | const manifest_bytes = pkg_root.readFileAllocOptions( | ||
| 246 | arena, | ||
| 247 | Manifest.basename, | ||
| 248 | Manifest.max_bytes, | ||
| 249 | null, | ||
| 250 | 1, | ||
| 251 | 0, | ||
| 252 | ) catch |err| switch (err) { | ||
| 253 | error.FileNotFound => return, | ||
| 254 | else => |e| { | ||
| 255 | const file_path = try pkg_root.join(arena, .{Manifest.basename}); | ||
| 256 | try eb.addRootErrorMessage(.{ | ||
| 257 | .msg = try eb.printString("unable to load package manifest '{s}': {s}", .{ | ||
| 258 | file_path, @errorName(e), | ||
| 259 | }), | ||
| 260 | .src_loc = .none, | ||
| 261 | .notes_len = 0, | ||
| 262 | }); | ||
| 263 | }, | ||
| 264 | }; | ||
| 265 | |||
| 266 | var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon); | ||
| 267 | f.manifest_ast = ast; | ||
| 268 | |||
| 269 | if (ast.errors.len > 0) { | ||
| 270 | const file_path = try pkg_root.join(arena, .{Manifest.basename}); | ||
| 271 | try main.putAstErrorsIntoBundle(arena, ast, file_path, eb); | ||
| 272 | return error.PackageFetchFailed; | ||
| 273 | } | ||
| 274 | |||
| 275 | f.manifest = try Manifest.parse(arena, ast); | ||
| 276 | |||
| 277 | if (f.manifest.errors.len > 0) { | ||
| 278 | const file_path = try pkg_root.join(arena, .{Manifest.basename}); | ||
| 279 | const token_starts = ast.tokens.items(.start); | ||
| 280 | |||
| 281 | for (f.manifest.errors) |msg| { | ||
| 282 | const start_loc = ast.tokenLocation(0, msg.tok); | ||
| 283 | |||
| 284 | try eb.addRootErrorMessage(.{ | ||
| 285 | .msg = try eb.addString(msg.msg), | ||
| 286 | .src_loc = try eb.addSourceLocation(.{ | ||
| 287 | .src_path = try eb.addString(file_path), | ||
| 288 | .span_start = token_starts[msg.tok], | ||
| 289 | .span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), | ||
| 290 | .span_main = token_starts[msg.tok] + msg.off, | ||
| 291 | .line = @intCast(start_loc.line), | ||
| 292 | .column = @intCast(start_loc.column), | ||
| 293 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | ||
| 294 | }), | ||
| 295 | .notes_len = 0, | ||
| 296 | }); | ||
| 297 | } | ||
| 298 | return error.PackageFetchFailed; | ||
| 299 | } | ||
| 300 | } | ||
| 301 | |||
| 302 | fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { | ||
| 303 | // If the package does not have a build.zig.zon file then there are no dependencies. | ||
| 304 | const manifest = f.manifest orelse return; | ||
| 305 | |||
| 306 | const new_fetches = nf: { | ||
| 307 | // Grab the new tasks into a temporary buffer so we can unlock that mutex | ||
| 308 | // as fast as possible. | ||
| 309 | // This overallocates any fetches that get skipped by the `continue` in the | ||
| 310 | // loop below. | ||
| 311 | const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count()); | ||
| 312 | var new_fetch_index: usize = 0; | ||
| 313 | |||
| 314 | f.job_queue.lock(); | ||
| 315 | defer f.job_queue.unlock(); | ||
| 316 | |||
| 317 | // It is impossible for there to be a collision here. Consider all three cases: | ||
| 318 | // * Correct hash is provided by manifest. | ||
| 319 | // - Redundant jobs are skipped in the loop below. | ||
| 320 | // * Incorrect has is provided by manifest. | ||
| 321 | // - Hash mismatch error emitted; `queueJobsForDeps` is not called. | ||
| 322 | // * Hash is not provided by manifest. | ||
| 323 | // - Hash missing error emitted; `queueJobsForDeps` is not called. | ||
| 324 | try f.job_queue.finish(hash, f, new_fetches.len); | ||
| 325 | |||
| 326 | for (manifest.dependencies.values()) |dep| { | ||
| 327 | const location: Location = switch (dep.location) { | ||
| 328 | .url => |url| .{ .remote = .{ | ||
| 329 | .url = url, | ||
| 330 | .hash = if (dep.hash) |h| h[0..hex_multihash_len].* else null, | ||
| 331 | } }, | ||
| 332 | .path => |path| .{ .relative_path = path }, | ||
| 333 | }; | ||
| 334 | const new_fetch = &new_fetches[new_fetch_index]; | ||
| 335 | const already_done = f.job_queue.add(location, new_fetch); | ||
| 336 | if (already_done) continue; | ||
| 337 | new_fetch_index += 1; | ||
| 338 | |||
| 339 | new_fetch.* = .{ | ||
| 340 | .gpa = f.gpa, | ||
| 341 | .arena = std.heap.ArenaAllocator.init(f.gpa), | ||
| 342 | .location = location, | ||
| 343 | .location_tok = dep.location_tok, | ||
| 344 | .hash_tok = dep.hash_tok, | ||
| 345 | .global_cache = f.global_cache, | ||
| 346 | .parent_package_root = f.package_root, | ||
| 347 | .parent_manifest_ast = f.manifest_ast.?, | ||
| 348 | .prog_node = f.prog_node, | ||
| 349 | .http_client = f.http_client, | ||
| 350 | .thread_pool = f.thread_pool, | ||
| 351 | .job_queue = f.job_queue, | ||
| 352 | .wait_group = f.wait_group, | ||
| 353 | |||
| 354 | .package_root = undefined, | ||
| 355 | .error_bundle = .{}, | ||
| 356 | .manifest = null, | ||
| 357 | .manifest_ast = null, | ||
| 358 | .actual_hash = undefined, | ||
| 359 | .has_build_zig = false, | ||
| 360 | }; | ||
| 361 | } | ||
| 362 | |||
| 363 | break :nf new_fetches[0..new_fetch_index]; | ||
| 364 | }; | ||
| 365 | |||
| 366 | // Now it's time to give tasks to the thread pool. | ||
| 367 | for (new_fetches) |new_fetch| { | ||
| 368 | f.wait_group.start(); | ||
| 369 | f.thread_pool.spawn(workerRun, .{f}) catch |err| switch (err) { | ||
| 370 | error.OutOfMemory => { | ||
| 371 | new_fetch.oom_flag = true; | ||
| 372 | f.wait_group.finish(); | ||
| 373 | continue; | ||
| 374 | }, | ||
| 375 | }; | ||
| 376 | } | ||
| 377 | } | ||
| 378 | |||
| 379 | fn workerRun(f: *Fetch) void { | ||
| 380 | defer f.wait_group.finish(); | ||
| 381 | run(f) catch |err| switch (err) { | ||
| 382 | error.OutOfMemory => f.oom_flag = true, | ||
| 383 | error.FetchFailed => {}, // See `error_bundle`. | ||
| 384 | }; | ||
| 385 | } | ||
| 386 | |||
| 387 | fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void { | ||
| 388 | const ast = f.parent_manifest_ast; | ||
| 389 | const token_starts = ast.tokens.items(.start); | ||
| 390 | const start_loc = ast.tokenLocation(0, msg_tok); | ||
| 391 | const eb = &f.error_bundle; | ||
| 392 | const file_path = try f.parent_package_root.join(f.arena, Manifest.basename); | ||
| 393 | const msg_off = 0; | ||
| 394 | |||
| 395 | try eb.addRootErrorMessage(.{ | ||
| 396 | .msg = msg_str, | ||
| 397 | .src_loc = try eb.addSourceLocation(.{ | ||
| 398 | .src_path = try eb.addString(file_path), | ||
| 399 | .span_start = token_starts[msg_tok], | ||
| 400 | .span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len), | ||
| 401 | .span_main = token_starts[msg_tok] + msg_off, | ||
| 402 | .line = @intCast(start_loc.line), | ||
| 403 | .column = @intCast(start_loc.column), | ||
| 404 | .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), | ||
| 405 | }), | ||
| 406 | .notes_len = 0, | ||
| 407 | }); | ||
| 408 | |||
| 409 | return error.FetchFailed; | ||
| 410 | } | ||
| 411 | |||
| 412 | const Resource = union(enum) { | ||
| 413 | file: fs.File, | ||
| 414 | http_request: std.http.Client.Request, | ||
| 415 | git_fetch_stream: git.Session.FetchStream, | ||
| 416 | dir: fs.IterableDir, | ||
| 417 | }; | ||
| 418 | |||
| 419 | const FileType = enum { | ||
| 420 | tar, | ||
| 421 | @"tar.gz", | ||
| 422 | @"tar.xz", | ||
| 423 | git_pack, | ||
| 424 | |||
| 425 | fn fromPath(file_path: []const u8) ?FileType { | ||
| 426 | if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; | ||
| 427 | if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; | ||
| 428 | if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; | ||
| 429 | return null; | ||
| 430 | } | ||
| 431 | |||
| 432 | /// Parameter is a content-disposition header value. | ||
| 433 | fn fromContentDisposition(cd_header: []const u8) ?FileType { | ||
| 434 | const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse | ||
| 435 | return null; | ||
| 436 | |||
| 437 | var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse | ||
| 438 | return null; | ||
| 439 | value_start += "filename".len; | ||
| 440 | if (cd_header[value_start] == '*') { | ||
| 441 | value_start += 1; | ||
| 442 | } | ||
| 443 | if (cd_header[value_start] != '=') return null; | ||
| 444 | value_start += 1; | ||
| 445 | |||
| 446 | var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; | ||
| 447 | if (cd_header[value_end - 1] == '\"') { | ||
| 448 | value_end -= 1; | ||
| 449 | } | ||
| 450 | return fromPath(cd_header[value_start..value_end]); | ||
| 451 | } | ||
| 452 | |||
| 453 | test fromContentDisposition { | ||
| 454 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); | ||
| 455 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); | ||
| 456 | try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); | ||
| 457 | try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); | ||
| 458 | try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); | ||
| 459 | |||
| 460 | try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); | ||
| 461 | try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null); | ||
| 462 | try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); | ||
| 463 | try std.testing.expect(fromContentDisposition("attachment; size=42") == null); | ||
| 464 | try std.testing.expect(fromContentDisposition("inline; size=42") == null); | ||
| 465 | try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); | ||
| 466 | try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); | ||
| 467 | } | ||
| 468 | }; | ||
| 469 | |||
| 470 | fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { | ||
| 471 | const gpa = f.gpa; | ||
| 472 | const arena = f.arena_allocator.allocator(); | ||
| 473 | const eb = &f.error_bundle; | ||
| 474 | |||
| 475 | if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{ | ||
| 476 | .file = try f.parent_package_root.openFile(uri.path, .{}), | ||
| 477 | }; | ||
| 478 | |||
| 479 | if (ascii.eqlIgnoreCase(uri.scheme, "http") or | ||
| 480 | ascii.eqlIgnoreCase(uri.scheme, "https")) | ||
| 481 | { | ||
| 482 | var h = std.http.Headers{ .allocator = gpa }; | ||
| 483 | defer h.deinit(); | ||
| 484 | |||
| 485 | var req = try f.http_client.request(.GET, uri, h, .{}); | ||
| 486 | errdefer req.deinit(); // releases more than memory | ||
| 487 | |||
| 488 | try req.start(.{}); | ||
| 489 | try req.wait(); | ||
| 490 | |||
| 491 | if (req.response.status != .ok) { | ||
| 492 | return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{ | ||
| 493 | @intFromEnum(req.response.status), req.response.status.phrase() orelse "", | ||
| 494 | }); | ||
| 495 | } | ||
| 496 | |||
| 497 | return .{ .http_request = req }; | ||
| 498 | } | ||
| 499 | |||
| 500 | if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or | ||
| 501 | ascii.eqlIgnoreCase(uri.scheme, "git+https")) | ||
| 502 | { | ||
| 503 | var transport_uri = uri; | ||
| 504 | transport_uri.scheme = uri.scheme["git+".len..]; | ||
| 505 | var redirect_uri: []u8 = undefined; | ||
| 506 | var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri }; | ||
| 507 | session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { | ||
| 508 | error.Redirected => { | ||
| 509 | defer gpa.free(redirect_uri); | ||
| 510 | return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri}); | ||
| 511 | }, | ||
| 512 | else => |other| return other, | ||
| 513 | }; | ||
| 514 | |||
| 515 | const want_oid = want_oid: { | ||
| 516 | const want_ref = uri.fragment orelse "HEAD"; | ||
| 517 | if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {} | ||
| 518 | |||
| 519 | const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); | ||
| 520 | const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); | ||
| 521 | |||
| 522 | var ref_iterator = try session.listRefs(gpa, .{ | ||
| 523 | .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, | ||
| 524 | .include_peeled = true, | ||
| 525 | }); | ||
| 526 | defer ref_iterator.deinit(); | ||
| 527 | while (try ref_iterator.next()) |ref| { | ||
| 528 | if (std.mem.eql(u8, ref.name, want_ref) or | ||
| 529 | std.mem.eql(u8, ref.name, want_ref_head) or | ||
| 530 | std.mem.eql(u8, ref.name, want_ref_tag)) | ||
| 531 | { | ||
| 532 | break :want_oid ref.peeled orelse ref.oid; | ||
| 533 | } | ||
| 534 | } | ||
| 535 | return f.fail(f.location_tok, "ref not found: {s}", .{want_ref}); | ||
| 536 | }; | ||
| 537 | if (uri.fragment == null) { | ||
| 538 | const notes_len = 1; | ||
| 539 | try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref"); | ||
| 540 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 541 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 542 | .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ | ||
| 543 | uri, std.fmt.fmtSliceHexLower(&want_oid), | ||
| 544 | }), | ||
| 545 | })); | ||
| 546 | return error.PackageFetchFailed; | ||
| 547 | } | ||
| 548 | |||
| 549 | var want_oid_buf: [git.fmt_oid_length]u8 = undefined; | ||
| 550 | _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{ | ||
| 551 | std.fmt.fmtSliceHexLower(&want_oid), | ||
| 552 | }) catch unreachable; | ||
| 553 | var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); | ||
| 554 | errdefer fetch_stream.deinit(); | ||
| 555 | |||
| 556 | return .{ .git_fetch_stream = fetch_stream }; | ||
| 557 | } | ||
| 558 | |||
| 559 | return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); | ||
| 560 | } | ||
| 561 | |||
| 562 | fn unpackResource( | ||
| 563 | f: *Fetch, | ||
| 564 | resource: *Resource, | ||
| 565 | uri_path: []const u8, | ||
| 566 | tmp_directory: Cache.Directory, | ||
| 567 | ) RunError!void { | ||
| 568 | const file_type = switch (resource.*) { | ||
| 569 | .file => FileType.fromPath(uri_path) orelse | ||
| 570 | return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}), | ||
| 571 | |||
| 572 | .http_request => |req| ft: { | ||
| 573 | // Content-Type takes first precedence. | ||
| 574 | const content_type = req.response.headers.getFirstValue("Content-Type") orelse | ||
| 575 | return f.fail(f.location_tok, "missing 'Content-Type' header", .{}); | ||
| 576 | |||
| 577 | if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) | ||
| 578 | return .tar; | ||
| 579 | |||
| 580 | if (ascii.eqlIgnoreCase(content_type, "application/gzip") or | ||
| 581 | ascii.eqlIgnoreCase(content_type, "application/x-gzip") or | ||
| 582 | ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) | ||
| 583 | { | ||
| 584 | return .@"tar.gz"; | ||
| 585 | } | ||
| 586 | |||
| 587 | if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) | ||
| 588 | return .@"tar.xz"; | ||
| 589 | |||
| 590 | if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { | ||
| 591 | return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{ | ||
| 592 | content_type, | ||
| 593 | }); | ||
| 594 | } | ||
| 595 | |||
| 596 | // Next, the filename from 'content-disposition: attachment' takes precedence. | ||
| 597 | if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| { | ||
| 598 | break :ft FileType.fromContentDisposition(cd_header) orelse | ||
| 599 | return f.fail( | ||
| 600 | f.location_tok, | ||
| 601 | "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", | ||
| 602 | .{cd_header}, | ||
| 603 | ); | ||
| 604 | } | ||
| 605 | |||
| 606 | // Finally, the path from the URI is used. | ||
| 607 | break :ft FileType.fromPath(uri_path) orelse | ||
| 608 | return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}); | ||
| 609 | }, | ||
| 610 | .git_fetch_stream => return .git_pack, | ||
| 611 | .dir => |dir| { | ||
| 612 | try f.recursiveDirectoryCopy(dir, tmp_directory.handle); | ||
| 613 | return; | ||
| 614 | }, | ||
| 615 | }; | ||
| 616 | |||
| 617 | switch (file_type) { | ||
| 618 | .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), | ||
| 619 | .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), | ||
| 620 | .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), | ||
| 621 | .git_pack => try unpackGitPack(f, tmp_directory.handle, resource), | ||
| 622 | } | ||
| 623 | } | ||
| 624 | |||
| 625 | fn unpackTarballCompressed( | ||
| 626 | f: *Fetch, | ||
| 627 | out_dir: fs.Dir, | ||
| 628 | resource: *Resource, | ||
| 629 | comptime Compression: type, | ||
| 630 | ) RunError!void { | ||
| 631 | const gpa = f.gpa; | ||
| 632 | const reader = resource.reader(); | ||
| 633 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | ||
| 634 | |||
| 635 | var decompress = try Compression.decompress(gpa, br.reader()); | ||
| 636 | defer decompress.deinit(); | ||
| 637 | |||
| 638 | return unpackTarball(f, out_dir, decompress.reader()); | ||
| 639 | } | ||
| 640 | |||
| 641 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { | ||
| 642 | const eb = &f.error_bundle; | ||
| 643 | |||
| 644 | var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa }; | ||
| 645 | defer diagnostics.deinit(); | ||
| 646 | |||
| 647 | try std.tar.pipeToFileSystem(out_dir, reader, .{ | ||
| 648 | .diagnostics = &diagnostics, | ||
| 649 | .strip_components = 1, | ||
| 650 | // TODO: we would like to set this to executable_bit_only, but two | ||
| 651 | // things need to happen before that: | ||
| 652 | // 1. the tar implementation needs to support it | ||
| 653 | // 2. the hashing algorithm here needs to support detecting the is_executable | ||
| 654 | // bit on Windows from the ACLs (see the isExecutable function). | ||
| 655 | .mode_mode = .ignore, | ||
| 656 | .filter = .{ .exclude_empty_directories = true }, | ||
| 657 | }); | ||
| 658 | |||
| 659 | if (diagnostics.errors.items.len > 0) { | ||
| 660 | const notes_len: u32 = @intCast(diagnostics.errors.items.len); | ||
| 661 | try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball"); | ||
| 662 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 663 | for (diagnostics.errors.items, notes_start..) |item, note_i| { | ||
| 664 | switch (item) { | ||
| 665 | .unable_to_create_sym_link => |info| { | ||
| 666 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 667 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | ||
| 668 | info.file_name, info.link_name, @errorName(info.code), | ||
| 669 | }), | ||
| 670 | })); | ||
| 671 | }, | ||
| 672 | .unsupported_file_type => |info| { | ||
| 673 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 674 | .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ | ||
| 675 | info.file_name, @intFromEnum(info.file_type), | ||
| 676 | }), | ||
| 677 | })); | ||
| 678 | }, | ||
| 679 | } | ||
| 680 | } | ||
| 681 | return error.InvalidTarball; | ||
| 682 | } | ||
| 683 | } | ||
| 684 | |||
| 685 | fn unpackGitPack( | ||
| 686 | f: *Fetch, | ||
| 687 | out_dir: fs.Dir, | ||
| 688 | resource: *Resource, | ||
| 689 | want_oid: git.Oid, | ||
| 690 | ) !void { | ||
| 691 | const eb = &f.error_bundle; | ||
| 692 | const gpa = f.gpa; | ||
| 693 | const reader = resource.reader(); | ||
| 694 | // The .git directory is used to store the packfile and associated index, but | ||
| 695 | // we do not attempt to replicate the exact structure of a real .git | ||
| 696 | // directory, since that isn't relevant for fetching a package. | ||
| 697 | { | ||
| 698 | var pack_dir = try out_dir.makeOpenPath(".git", .{}); | ||
| 699 | defer pack_dir.close(); | ||
| 700 | var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); | ||
| 701 | defer pack_file.close(); | ||
| 702 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | ||
| 703 | try fifo.pump(reader.reader(), pack_file.writer()); | ||
| 704 | try pack_file.sync(); | ||
| 705 | |||
| 706 | var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); | ||
| 707 | defer index_file.close(); | ||
| 708 | { | ||
| 709 | var index_prog_node = reader.prog_node.start("Index pack", 0); | ||
| 710 | defer index_prog_node.end(); | ||
| 711 | index_prog_node.activate(); | ||
| 712 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); | ||
| 713 | try git.indexPack(gpa, pack_file, index_buffered_writer.writer()); | ||
| 714 | try index_buffered_writer.flush(); | ||
| 715 | try index_file.sync(); | ||
| 716 | } | ||
| 717 | |||
| 718 | { | ||
| 719 | var checkout_prog_node = reader.prog_node.start("Checkout", 0); | ||
| 720 | defer checkout_prog_node.end(); | ||
| 721 | checkout_prog_node.activate(); | ||
| 722 | var repository = try git.Repository.init(gpa, pack_file, index_file); | ||
| 723 | defer repository.deinit(); | ||
| 724 | var diagnostics: git.Diagnostics = .{ .allocator = gpa }; | ||
| 725 | defer diagnostics.deinit(); | ||
| 726 | try repository.checkout(out_dir, want_oid, &diagnostics); | ||
| 727 | |||
| 728 | if (diagnostics.errors.items.len > 0) { | ||
| 729 | const notes_len: u32 = @intCast(diagnostics.errors.items.len); | ||
| 730 | try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile"); | ||
| 731 | const notes_start = try eb.reserveNotes(notes_len); | ||
| 732 | for (diagnostics.errors.items, notes_start..) |item, note_i| { | ||
| 733 | switch (item) { | ||
| 734 | .unable_to_create_sym_link => |info| { | ||
| 735 | eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ | ||
| 736 | .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ | ||
| 737 | info.file_name, info.link_name, @errorName(info.code), | ||
| 738 | }), | ||
| 739 | })); | ||
| 740 | }, | ||
| 741 | } | ||
| 742 | } | ||
| 743 | return error.InvalidGitPack; | ||
| 744 | } | ||
| 745 | } | ||
| 746 | } | ||
| 747 | |||
| 748 | try out_dir.deleteTree(".git"); | ||
| 749 | } | ||
| 750 | |||
| 751 | fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void { | ||
| 752 | // Recursive directory copy. | ||
| 753 | var it = try dir.walk(f.gpa); | ||
| 754 | defer it.deinit(); | ||
| 755 | while (try it.next()) |entry| { | ||
| 756 | switch (entry.kind) { | ||
| 757 | .directory => {}, // omit empty directories | ||
| 758 | .file => { | ||
| 759 | dir.dir.copyFile( | ||
| 760 | entry.path, | ||
| 761 | tmp_dir, | ||
| 762 | entry.path, | ||
| 763 | .{}, | ||
| 764 | ) catch |err| switch (err) { | ||
| 765 | error.FileNotFound => { | ||
| 766 | if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.makePath(dirname); | ||
| 767 | try dir.dir.copyFile(entry.path, tmp_dir, entry.path, .{}); | ||
| 768 | }, | ||
| 769 | else => |e| return e, | ||
| 770 | }; | ||
| 771 | }, | ||
| 772 | .sym_link => { | ||
| 773 | var buf: [fs.MAX_PATH_BYTES]u8 = undefined; | ||
| 774 | const link_name = try dir.dir.readLink(entry.path, &buf); | ||
| 775 | // TODO: if this would create a symlink to outside | ||
| 776 | // the destination directory, fail with an error instead. | ||
| 777 | try tmp_dir.symLink(link_name, entry.path, .{}); | ||
| 778 | }, | ||
| 779 | else => return error.IllegalFileTypeInPackage, | ||
| 780 | } | ||
| 781 | } | ||
| 782 | } | ||
| 783 | |||
| 784 | pub fn renameTmpIntoCache( | ||
| 785 | cache_dir: fs.Dir, | ||
| 786 | tmp_dir_sub_path: []const u8, | ||
| 787 | dest_dir_sub_path: []const u8, | ||
| 788 | ) !void { | ||
| 789 | assert(dest_dir_sub_path[1] == fs.path.sep); | ||
| 790 | var handled_missing_dir = false; | ||
| 791 | while (true) { | ||
| 792 | cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { | ||
| 793 | error.FileNotFound => { | ||
| 794 | if (handled_missing_dir) return err; | ||
| 795 | cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { | ||
| 796 | error.PathAlreadyExists => handled_missing_dir = true, | ||
| 797 | else => |e| return e, | ||
| 798 | }; | ||
| 799 | continue; | ||
| 800 | }, | ||
| 801 | error.PathAlreadyExists, error.AccessDenied => { | ||
| 802 | // Package has been already downloaded and may already be in use on the system. | ||
| 803 | cache_dir.deleteTree(tmp_dir_sub_path) catch { | ||
| 804 | // Garbage files leftover in zig-cache/tmp/ is, as they say | ||
| 805 | // on Star Trek, "operating within normal parameters". | ||
| 806 | }; | ||
| 807 | }, | ||
| 808 | else => |e| return e, | ||
| 809 | }; | ||
| 810 | break; | ||
| 811 | } | ||
| 812 | } | ||
| 813 | |||
| 814 | /// Assumes that files not included in the package have already been filtered | ||
| 815 | /// prior to calling this function. This ensures that files not protected by | ||
| 816 | /// the hash are not present on the file system. Empty directories are *not | ||
| 817 | /// hashed* and must not be present on the file system when calling this | ||
| 818 | /// function. | ||
| 819 | fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest { | ||
| 820 | // All the path name strings need to be in memory for sorting. | ||
| 821 | const arena = f.arena_allocator.allocator(); | ||
| 822 | const gpa = f.gpa; | ||
| 823 | |||
| 824 | // Collect all files, recursively, then sort. | ||
| 825 | var all_files = std.ArrayList(*HashedFile).init(gpa); | ||
| 826 | defer all_files.deinit(); | ||
| 827 | |||
| 828 | var walker = try pkg_dir.walk(gpa); | ||
| 829 | defer walker.deinit(); | ||
| 830 | |||
| 831 | { | ||
| 832 | // The final hash will be a hash of each file hashed independently. This | ||
| 833 | // allows hashing in parallel. | ||
| 834 | var wait_group: WaitGroup = .{}; | ||
| 835 | // `computeHash` is called from a worker thread so there must not be | ||
| 836 | // any waiting without working or a deadlock could occur. | ||
| 837 | defer wait_group.waitAndWork(); | ||
| 838 | |||
| 839 | while (try walker.next()) |entry| { | ||
| 840 | _ = filter; // TODO: apply filter rules here | ||
| 841 | |||
| 842 | const kind: HashedFile.Kind = switch (entry.kind) { | ||
| 843 | .directory => continue, | ||
| 844 | .file => .file, | ||
| 845 | .sym_link => .sym_link, | ||
| 846 | else => return error.IllegalFileTypeInPackage, | ||
| 847 | }; | ||
| 848 | |||
| 849 | if (std.mem.eql(u8, entry.path, build_zig_basename)) | ||
| 850 | f.has_build_zig = true; | ||
| 851 | |||
| 852 | const hashed_file = try arena.create(HashedFile); | ||
| 853 | const fs_path = try arena.dupe(u8, entry.path); | ||
| 854 | hashed_file.* = .{ | ||
| 855 | .fs_path = fs_path, | ||
| 856 | .normalized_path = try normalizePath(arena, fs_path), | ||
| 857 | .kind = kind, | ||
| 858 | .hash = undefined, // to be populated by the worker | ||
| 859 | .failure = undefined, // to be populated by the worker | ||
| 860 | }; | ||
| 861 | wait_group.start(); | ||
| 862 | try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); | ||
| 863 | |||
| 864 | try all_files.append(hashed_file); | ||
| 865 | } | ||
| 866 | } | ||
| 867 | |||
| 868 | std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); | ||
| 869 | |||
| 870 | var hasher = Manifest.Hash.init(.{}); | ||
| 871 | var any_failures = false; | ||
| 872 | const eb = &f.error_bundle; | ||
| 873 | for (all_files.items) |hashed_file| { | ||
| 874 | hashed_file.failure catch |err| { | ||
| 875 | any_failures = true; | ||
| 876 | try eb.addRootErrorMessage(.{ | ||
| 877 | .msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}), | ||
| 878 | .src_loc = try eb.addSourceLocation(.{ | ||
| 879 | .src_path = try eb.addString(hashed_file.fs_path), | ||
| 880 | .span_start = 0, | ||
| 881 | .span_end = 0, | ||
| 882 | .span_main = 0, | ||
| 883 | }), | ||
| 884 | .notes_len = 0, | ||
| 885 | }); | ||
| 886 | }; | ||
| 887 | hasher.update(&hashed_file.hash); | ||
| 888 | } | ||
| 889 | if (any_failures) return error.FetchFailed; | ||
| 890 | return hasher.finalResult(); | ||
| 891 | } | ||
| 892 | |||
| 893 | fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { | ||
| 894 | defer wg.finish(); | ||
| 895 | hashed_file.failure = hashFileFallible(dir, hashed_file); | ||
| 896 | } | ||
| 897 | |||
| 898 | fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { | ||
| 899 | var buf: [8000]u8 = undefined; | ||
| 900 | var hasher = Manifest.Hash.init(.{}); | ||
| 901 | hasher.update(hashed_file.normalized_path); | ||
| 902 | switch (hashed_file.kind) { | ||
| 903 | .file => { | ||
| 904 | var file = try dir.openFile(hashed_file.fs_path, .{}); | ||
| 905 | defer file.close(); | ||
| 906 | hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) }); | ||
| 907 | while (true) { | ||
| 908 | const bytes_read = try file.read(&buf); | ||
| 909 | if (bytes_read == 0) break; | ||
| 910 | hasher.update(buf[0..bytes_read]); | ||
| 911 | } | ||
| 912 | }, | ||
| 913 | .sym_link => { | ||
| 914 | const link_name = try dir.readLink(hashed_file.fs_path, &buf); | ||
| 915 | hasher.update(link_name); | ||
| 916 | }, | ||
| 917 | } | ||
| 918 | hasher.final(&hashed_file.hash); | ||
| 919 | } | ||
| 920 | |||
| 921 | fn isExecutable(file: fs.File) !bool { | ||
| 922 | if (builtin.os.tag == .windows) { | ||
| 923 | // TODO check the ACL on Windows. | ||
| 924 | // Until this is implemented, this could be a false negative on | ||
| 925 | // Windows, which is why we do not yet set executable_bit_only above | ||
| 926 | // when unpacking the tarball. | ||
| 927 | return false; | ||
| 928 | } else { | ||
| 929 | const stat = try file.stat(); | ||
| 930 | return (stat.mode & std.os.S.IXUSR) != 0; | ||
| 931 | } | ||
| 932 | } | ||
| 933 | |||
| 934 | const HashedFile = struct { | ||
| 935 | fs_path: []const u8, | ||
| 936 | normalized_path: []const u8, | ||
| 937 | hash: Digest, | ||
| 938 | failure: Error!void, | ||
| 939 | kind: Kind, | ||
| 940 | |||
| 941 | const Error = | ||
| 942 | fs.File.OpenError || | ||
| 943 | fs.File.ReadError || | ||
| 944 | fs.File.StatError || | ||
| 945 | fs.Dir.ReadLinkError; | ||
| 946 | |||
| 947 | const Kind = enum { file, sym_link }; | ||
| 948 | |||
| 949 | fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { | ||
| 950 | _ = context; | ||
| 951 | return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); | ||
| 952 | } | ||
| 953 | }; | ||
| 954 | |||
| 955 | /// Make a file system path identical independently of operating system path inconsistencies. | ||
| 956 | /// This converts backslashes into forward slashes. | ||
| 957 | fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { | ||
| 958 | const canonical_sep = '/'; | ||
| 959 | |||
| 960 | if (fs.path.sep == canonical_sep) | ||
| 961 | return fs_path; | ||
| 962 | |||
| 963 | const normalized = try arena.dupe(u8, fs_path); | ||
| 964 | for (normalized) |*byte| { | ||
| 965 | switch (byte.*) { | ||
| 966 | fs.path.sep => byte.* = canonical_sep, | ||
| 967 | else => continue, | ||
| 968 | } | ||
| 969 | } | ||
| 970 | return normalized; | ||
| 971 | } | ||
| 972 | |||
| 973 | pub const Filter = struct { | ||
| 974 | include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, | ||
| 975 | |||
| 976 | /// sub_path is relative to the tarball root. | ||
| 977 | pub fn includePath(self: Filter, sub_path: []const u8) bool { | ||
| 978 | if (self.include_paths.count() == 0) return true; | ||
| 979 | if (self.include_paths.contains("")) return true; | ||
| 980 | if (self.include_paths.contains(sub_path)) return true; | ||
| 981 | |||
| 982 | // Check if any included paths are parent directories of sub_path. | ||
| 983 | var dirname = sub_path; | ||
| 984 | while (std.fs.path.dirname(sub_path)) |next_dirname| { | ||
| 985 | if (self.include_paths.contains(sub_path)) return true; | ||
| 986 | dirname = next_dirname; | ||
| 987 | } | ||
| 988 | |||
| 989 | return false; | ||
| 990 | } | ||
| 991 | }; | ||
| 992 | |||
| 993 | const build_zig_basename = @import("../Package.zig").build_zig_basename; | ||
| 994 | const hex_multihash_len = 2 * Manifest.multihash_len; | ||
| 995 | |||
| 996 | // These are random bytes. | ||
| 997 | const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; | ||
| 998 | const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; | ||
| 999 | |||
| 1000 | const builtin = @import("builtin"); | ||
| 1001 | const std = @import("std"); | ||
| 1002 | const fs = std.fs; | ||
| 1003 | const assert = std.debug.assert; | ||
| 1004 | const ascii = std.ascii; | ||
| 1005 | const Allocator = std.mem.Allocator; | ||
| 1006 | const Cache = std.Build.Cache; | ||
| 1007 | const ThreadPool = std.Thread.Pool; | ||
| 1008 | const WaitGroup = std.Thread.WaitGroup; | ||
| 1009 | const Manifest = @import("../Manifest.zig"); | ||
| 1010 | const Fetch = @This(); | ||
| 1011 | const main = @import("../main.zig"); | ||
| 1012 | const git = @import("../git.zig"); | ||
src/Package/hash.zig deleted-153
| ... | @@ -1,153 +0,0 @@ | ||
| 1 | const builtin = @import("builtin"); | ||
| 2 | const std = @import("std"); | ||
| 3 | const fs = std.fs; | ||
| 4 | const ThreadPool = std.Thread.Pool; | ||
| 5 | const WaitGroup = std.Thread.WaitGroup; | ||
| 6 | const Allocator = std.mem.Allocator; | ||
| 7 | |||
| 8 | const Hash = @import("../Manifest.zig").Hash; | ||
| 9 | |||
| 10 | pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 { | ||
| 11 | const gpa = thread_pool.allocator; | ||
| 12 | |||
| 13 | // We'll use an arena allocator for the path name strings since they all | ||
| 14 | // need to be in memory for sorting. | ||
| 15 | var arena_instance = std.heap.ArenaAllocator.init(gpa); | ||
| 16 | defer arena_instance.deinit(); | ||
| 17 | const arena = arena_instance.allocator(); | ||
| 18 | |||
| 19 | // TODO: delete files not included in the package prior to computing the package hash. | ||
| 20 | // for example, if the ini file has directives to include/not include certain files, | ||
| 21 | // apply those rules directly to the filesystem right here. This ensures that files | ||
| 22 | // not protected by the hash are not present on the file system. | ||
| 23 | |||
| 24 | // Collect all files, recursively, then sort. | ||
| 25 | var all_files = std.ArrayList(*HashedFile).init(gpa); | ||
| 26 | defer all_files.deinit(); | ||
| 27 | |||
| 28 | var walker = try pkg_dir.walk(gpa); | ||
| 29 | defer walker.deinit(); | ||
| 30 | |||
| 31 | { | ||
| 32 | // The final hash will be a hash of each file hashed independently. This | ||
| 33 | // allows hashing in parallel. | ||
| 34 | var wait_group: WaitGroup = .{}; | ||
| 35 | defer wait_group.wait(); | ||
| 36 | |||
| 37 | while (try walker.next()) |entry| { | ||
| 38 | const kind: HashedFile.Kind = switch (entry.kind) { | ||
| 39 | .directory => continue, | ||
| 40 | .file => .file, | ||
| 41 | .sym_link => .sym_link, | ||
| 42 | else => return error.IllegalFileTypeInPackage, | ||
| 43 | }; | ||
| 44 | const hashed_file = try arena.create(HashedFile); | ||
| 45 | const fs_path = try arena.dupe(u8, entry.path); | ||
| 46 | hashed_file.* = .{ | ||
| 47 | .fs_path = fs_path, | ||
| 48 | .normalized_path = try normalizePath(arena, fs_path), | ||
| 49 | .kind = kind, | ||
| 50 | .hash = undefined, // to be populated by the worker | ||
| 51 | .failure = undefined, // to be populated by the worker | ||
| 52 | }; | ||
| 53 | wait_group.start(); | ||
| 54 | try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); | ||
| 55 | |||
| 56 | try all_files.append(hashed_file); | ||
| 57 | } | ||
| 58 | } | ||
| 59 | |||
| 60 | std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); | ||
| 61 | |||
| 62 | var hasher = Hash.init(.{}); | ||
| 63 | var any_failures = false; | ||
| 64 | for (all_files.items) |hashed_file| { | ||
| 65 | hashed_file.failure catch |err| { | ||
| 66 | any_failures = true; | ||
| 67 | std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) }); | ||
| 68 | }; | ||
| 69 | hasher.update(&hashed_file.hash); | ||
| 70 | } | ||
| 71 | if (any_failures) return error.PackageHashUnavailable; | ||
| 72 | return hasher.finalResult(); | ||
| 73 | } | ||
| 74 | |||
| 75 | const HashedFile = struct { | ||
| 76 | fs_path: []const u8, | ||
| 77 | normalized_path: []const u8, | ||
| 78 | hash: [Hash.digest_length]u8, | ||
| 79 | failure: Error!void, | ||
| 80 | kind: Kind, | ||
| 81 | |||
| 82 | const Error = | ||
| 83 | fs.File.OpenError || | ||
| 84 | fs.File.ReadError || | ||
| 85 | fs.File.StatError || | ||
| 86 | fs.Dir.ReadLinkError; | ||
| 87 | |||
| 88 | const Kind = enum { file, sym_link }; | ||
| 89 | |||
| 90 | fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { | ||
| 91 | _ = context; | ||
| 92 | return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); | ||
| 93 | } | ||
| 94 | }; | ||
| 95 | |||
| 96 | /// Make a file system path identical independently of operating system path inconsistencies. | ||
| 97 | /// This converts backslashes into forward slashes. | ||
| 98 | fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { | ||
| 99 | const canonical_sep = '/'; | ||
| 100 | |||
| 101 | if (fs.path.sep == canonical_sep) | ||
| 102 | return fs_path; | ||
| 103 | |||
| 104 | const normalized = try arena.dupe(u8, fs_path); | ||
| 105 | for (normalized) |*byte| { | ||
| 106 | switch (byte.*) { | ||
| 107 | fs.path.sep => byte.* = canonical_sep, | ||
| 108 | else => continue, | ||
| 109 | } | ||
| 110 | } | ||
| 111 | return normalized; | ||
| 112 | } | ||
| 113 | |||
| 114 | fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { | ||
| 115 | defer wg.finish(); | ||
| 116 | hashed_file.failure = hashFileFallible(dir, hashed_file); | ||
| 117 | } | ||
| 118 | |||
| 119 | fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { | ||
| 120 | var buf: [8000]u8 = undefined; | ||
| 121 | var hasher = Hash.init(.{}); | ||
| 122 | hasher.update(hashed_file.normalized_path); | ||
| 123 | switch (hashed_file.kind) { | ||
| 124 | .file => { | ||
| 125 | var file = try dir.openFile(hashed_file.fs_path, .{}); | ||
| 126 | defer file.close(); | ||
| 127 | hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) }); | ||
| 128 | while (true) { | ||
| 129 | const bytes_read = try file.read(&buf); | ||
| 130 | if (bytes_read == 0) break; | ||
| 131 | hasher.update(buf[0..bytes_read]); | ||
| 132 | } | ||
| 133 | }, | ||
| 134 | .sym_link => { | ||
| 135 | const link_name = try dir.readLink(hashed_file.fs_path, &buf); | ||
| 136 | hasher.update(link_name); | ||
| 137 | }, | ||
| 138 | } | ||
| 139 | hasher.final(&hashed_file.hash); | ||
| 140 | } | ||
| 141 | |||
| 142 | fn isExecutable(file: fs.File) !bool { | ||
| 143 | if (builtin.os.tag == .windows) { | ||
| 144 | // TODO check the ACL on Windows. | ||
| 145 | // Until this is implemented, this could be a false negative on | ||
| 146 | // Windows, which is why we do not yet set executable_bit_only above | ||
| 147 | // when unpacking the tarball. | ||
| 148 | return false; | ||
| 149 | } else { | ||
| 150 | const stat = try file.stat(); | ||
| 151 | return (stat.mode & std.os.S.IXUSR) != 0; | ||
| 152 | } | ||
| 153 | } | ||
src/main.zig+1-1
| ... | @@ -4714,7 +4714,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi | ... | @@ -4714,7 +4714,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi |
| 4714 | defer if (cleanup_build_dir) |*dir| dir.close(); | 4714 | defer if (cleanup_build_dir) |*dir| dir.close(); |
| 4715 | 4715 | ||
| 4716 | const cwd_path = try process.getCwdAlloc(arena); | 4716 | const cwd_path = try process.getCwdAlloc(arena); |
| 4717 | const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else "build.zig"; | 4717 | const build_zig_basename = if (build_file) |bf| fs.path.basename(bf) else Package.build_zig_basename; |
| 4718 | const build_directory: Compilation.Directory = blk: { | 4718 | const build_directory: Compilation.Directory = blk: { |
| 4719 | if (build_file) |bf| { | 4719 | if (build_file) |bf| { |
| 4720 | if (fs.path.dirname(bf)) |dirname| { | 4720 | if (fs.path.dirname(bf)) |dirname| { |