Commit c737c2efe350db5c9b30a2553d294b6642860378 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-06-30 21:00:20+02:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-06-30 21:00:20+02:00
log	c737c2efe350db5c9b30a2553d294b6642860378
tree	e21e672174eb13e6d0abc304ca4b0fb94f86359e
parent	f6d22629aa6a7fbc5a2f285c335e1580bd4a3e24
parent	779b6cc63cbf1ab0f132996506a238eae72dcbf9

Merge pull request 'move all package management functionality from compiler to build system' (#35917) from build-system into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/35917

105 files changed, 9147 insertions(+), 9173 deletions(-)

CMakeLists.txt
build.zig
build.zig.zon
ci
lib
- compiler
  - Maker.zig
  - Maker
    Fetch.zig
    Fetch
    git.zig
    git
    testdata
    testrepo-sha1.idx
    testrepo-sha1.pack
    testrepo-sha256.idx
    testrepo-sha256.pack
    Fuzz.zig
    Graph.zig
    Package.zig
    Package
    Manifest.zig
    ScannedConfig.zig
    Step.zig
    Step
    Compile.zig
    ObjCopy.zig
    Run.zig
    TranslateC.zig
    UpdateSourceFiles.zig
    WriteFile.zig
    WebServer.zig
  - configurer.zig
  - libc.zig
  - resinator
    main.zig
  - std-docs.zig
- std
  - Build.zig
  - Build
    Cache.zig
    Configuration.zig
    Step
    ConfigHeader.zig
    TranslateC.zig
  - Progress.zig
  - zig.zig
  - zig
    LibCDirs.zig
    Server.zig
src
- Builtin.zig
- Compilation.zig
- Compilation
  - Config.zig
- Module.zig
- Package.zig
- Package
  - Fetch.zig
  - Fetch
    git.zig
    git
    testdata
    testrepo-sha1.idx
    testrepo-sha1.pack
    testrepo-sha256.idx
    testrepo-sha256.pack
  - Manifest.zig
  - Module.zig
- Sema.zig
- Zcu.zig
- Zcu
  - PerThread.zig
- codegen
  - aarch64
    Select.zig
  - c.zig
  - llvm.zig
  - llvm
    FuncGen.zig
  - riscv64
    CodeGen.zig
  - x86_64
    CodeGen.zig
- dev.zig
- introspect.zig
- libs
- link.zig
- link
- main.zig
- print_env.zig
- print_targets.zig
test
- src
  - Cases.zig
- tests.zig

CMakeLists.txt-8

...	...	@@ -335,11 +335,6 @@ set(ZIG_STAGE2_SOURCES
335	335	src/Compilation.zig
336	336	src/Compilation/Config.zig
337	337	src/InternPool.zig
338		src/Package.zig
339		src/Package/Fetch.zig
340		src/Package/Fetch/git.zig
341		src/Package/Manifest.zig
342		src/Package/Module.zig
343	338	src/RangeSet.zig
344	339	src/Sema.zig
345	340	src/Sema/reinterpret.zig
...	...	@@ -369,7 +364,6 @@ set(ZIG_STAGE2_SOURCES
369	364	src/libs/glibc.zig
370	365	src/libs/netbsd.zig
371	366	src/libs/openbsd.zig
372		src/introspect.zig
373	367	src/libs/libcxx.zig
374	368	src/libs/libtsan.zig
375	369	src/libs/libunwind.zig
...	...	@@ -735,8 +729,6 @@ endif()
735	729
736	730
737	731	set(ZIG_BUILD_ARGS
738		--zig-lib-dir "${PROJECT_SOURCE_DIR}/lib"
739
740	732	"-Dversion-string=${RESOLVED_ZIG_VERSION}"
741	733	"-Dtarget=${ZIG_TARGET_TRIPLE}"
742	734	"-Dcpu=${ZIG_TARGET_MCPU}"

build.zig+5-3

...	...	@@ -175,6 +175,9 @@ pub fn build(b: *std.Build) !void {
175	175	".tar",
176	176	// exclude files from lib/std/zip/testdata
177	177	".zip",
	178	// exclude files from lib/compiler/Maker/Fetch/git/testdata
	179	".idx",
	180	".pack",
178	181	// others
179	182	"README.md",
180	183	},
...	...	@@ -264,9 +267,8 @@ pub fn build(b: *std.Build) !void {
264	267	std.process.exit(1);
265	268	}
266	269
267		// Ensure git version changes get picked up
268		// https://codeberg.org/ziglang/zig/issues/35473
269		b.graph.poisonCache();
	270	// Ensure git version changes get picked up.
	271	b.dependOnFileContents(b.path(".git/HEAD"));
270	272
271	273	const version_string = b.fmt("{d}.{d}.{d}", .{ zig_version.major, zig_version.minor, zig_version.patch });
272	274

build.zig.zon-3

...	...	@@ -7,9 +7,6 @@
7	7	.standalone_test_cases = .{
8	8	.path = "test/standalone",
9	9	},
10		.link_test_cases = .{
11		.path = "test/link",
12		},
13	10	},
14	11	.paths = .{""},
15	12	.fingerprint = 0xc1ce108124179e16,

ci/aarch64-freebsd-debug.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-debug/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 4m
50	52
51	53	stage3-debug/bin/zig build \

ci/aarch64-freebsd-release.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-release/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 4m
50	52
51	53	# Ensure that stage3 and stage4 are byte-for-byte identical.

ci/aarch64-linux-debug.sh+3-1

...	...	@@ -42,6 +42,9 @@ unset CXX
42	42
43	43	ninja install
44	44
	45	# Must be done after zig cc is finished.
	46	export ZIG_LIB_DIR="$PWD/../lib"
	47
45	48	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
46	49	stage3-debug/bin/zig build test docs \
47	50	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -49,7 +52,6 @@ stage3-debug/bin/zig build test docs \
49	52	-Dskip-non-native \
50	53	-Dtarget=native-native-musl \
51	54	--search-prefix "$PREFIX" \
52		--zig-lib-dir "$PWD/../lib" \
53	55	-Denable-superhtml \
54	56	--test-timeout 3m
55	57

ci/aarch64-linux-release.sh+3-1

...	...	@@ -42,6 +42,9 @@ unset CXX
42	42
43	43	ninja install
44	44
	45	# Must be done after zig cc is finished.
	46	export ZIG_LIB_DIR="$PWD/../lib"
	47
45	48	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
46	49	stage3-release/bin/zig build test docs \
47	50	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -49,7 +52,6 @@ stage3-release/bin/zig build test docs \
49	52	-Dskip-non-native \
50	53	-Dtarget=native-native-musl \
51	54	--search-prefix "$PREFIX" \
52		--zig-lib-dir "$PWD/../lib" \
53	55	-Denable-superhtml \
54	56	--test-timeout 3m
55	57

ci/aarch64-macos-debug.sh+3-1

...	...	@@ -43,9 +43,11 @@ cmake .. \
43	43
44	44	ninja install
45	45
	46	# Must be done after zig cc is finished.
	47	export ZIG_LIB_DIR="$PWD/../lib"
	48
46	49	stage3-debug/bin/zig build test docs \
47	50	--maxrss ${ZSF_MAX_RSS:-0} \
48		--zig-lib-dir "$PWD/../lib" \
49	51	-Denable-macos-sdk \
50	52	-Dstatic-llvm \
51	53	-Dskip-spirv \

ci/aarch64-macos-release.sh+3-1

...	...	@@ -43,8 +43,10 @@ cmake .. \
43	43
44	44	ninja install
45	45
	46	# Must be done after zig cc is finished.
	47	export ZIG_LIB_DIR="$PWD/../lib"
	48
46	49	stage3-release/bin/zig build test docs \
47		--zig-lib-dir "$PWD/../lib" \
48	50	-Denable-macos-sdk \
49	51	-Dstatic-llvm \
50	52	-Dskip-spirv \

ci/aarch64-netbsd-debug.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-debug/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 4m
50	52
51	53	stage3-debug/bin/zig build \

ci/aarch64-netbsd-release.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-release/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 4m
50	52
51	53	# Ensure that stage3 and stage4 are byte-for-byte identical.

ci/aarch64-windows.ps1+3-2

...	...	@@ -4,7 +4,6 @@ $MCPU = "baseline"
4	4	$ZIG_LLVM_CLANG_LLD_URL = "https://ziglang.org/deps/$ZIG_LLVM_CLANG_LLD_NAME.zip"
5	5	$PREFIX_PATH = "$(Get-Location)\..\$ZIG_LLVM_CLANG_LLD_NAME"
6	6	$ZIG = "$PREFIX_PATH\bin\zig.exe"
7		$ZIG_LIB_DIR = "$(Get-Location)\lib"
8	7	$ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 }
9	8
10	9	if (!(Test-Path "..\$ZIG_LLVM_CLANG_LLD_NAME.zip")) {
...	...	@@ -53,10 +52,12 @@ CheckLastExitCode
53	52	ninja install
54	53	CheckLastExitCode
55	54
	55	# Must be done after zig cc is finished.
	56	$Env:ZIG_LIB_DIR="$(Get-Location)\..\lib"
	57
56	58	Write-Output "Main test suite..."
57	59	& "stage3-release\bin\zig.exe" build test docs `
58	60	--maxrss $ZSF_MAX_RSS `
59		--zig-lib-dir "$ZIG_LIB_DIR" `
60	61	--search-prefix "$PREFIX_PATH" `
61	62	-Dstatic-llvm `
62	63	-Dskip-non-native `

ci/loongarch64-linux-debug.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
44	47	stage3-debug/bin/zig build test docs \
45	48	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -47,7 +50,6 @@ stage3-debug/bin/zig build test docs \
47	50	-Dskip-non-native \
48	51	-Dtarget=native-native-musl \
49	52	--search-prefix "$PREFIX" \
50		--zig-lib-dir "$PWD/../lib" \
51	53	--test-timeout 4m
52	54
53	55	stage3-debug/bin/zig build \

ci/loongarch64-linux-release.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
44	47	stage3-release/bin/zig build test docs \
45	48	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -47,7 +50,6 @@ stage3-release/bin/zig build test docs \
47	50	-Dskip-non-native \
48	51	-Dtarget=native-native-musl \
49	52	--search-prefix "$PREFIX" \
50		--zig-lib-dir "$PWD/../lib" \
51	53	--test-timeout 4m
52	54
53	55	# Ensure that stage3 and stage4 are byte-for-byte identical.

ci/powerpc64le-linux-debug.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
44	47	stage3-debug/bin/zig build test docs \
45	48	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -48,7 +51,6 @@ stage3-debug/bin/zig build test docs \
48	51	-Dtarget=native-native-musl \
49	52	-Dcpu=native+longcall \
50	53	--search-prefix "$PREFIX" \
51		--zig-lib-dir "$PWD/../lib" \
52	54	--test-timeout 4m
53	55
54	56	stage3-debug/bin/zig build \

ci/powerpc64le-linux-release.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
44	47	stage3-release/bin/zig build test docs \
45	48	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -48,7 +51,6 @@ stage3-release/bin/zig build test docs \
48	51	-Dtarget=native-native-musl \
49	52	-Dcpu=native+longcall \
50	53	--search-prefix "$PREFIX" \
51		--zig-lib-dir "$PWD/../lib" \
52	54	--test-timeout 4m
53	55
54	56	# Ensure that stage3 and stage4 are byte-for-byte identical.

ci/riscv64-linux-debug.sh+3-1

...	...	@@ -42,6 +42,9 @@ unset CXX
42	42
43	43	ninja install
44	44
	45	# Must be done after zig cc is finished.
	46	export ZIG_LIB_DIR="$PWD/../lib"
	47
45	48	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
46	49	stage3-debug/bin/zig build test-modules test-c-abi \
47	50	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -50,5 +53,4 @@ stage3-debug/bin/zig build test-modules test-c-abi \
50	53	-Dskip-single-threaded \
51	54	-Dtarget=native-native-musl \
52	55	--search-prefix "$PREFIX" \
53		--zig-lib-dir "$PWD/../lib" \
54	56	--test-timeout 4m

ci/riscv64-linux-release.sh+3-1

...	...	@@ -42,6 +42,9 @@ unset CXX
42	42
43	43	ninja install
44	44
	45	# Must be done after zig cc is finished.
	46	export ZIG_LIB_DIR="$PWD/../lib"
	47
45	48	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
46	49	stage3-release/bin/zig build test-modules test-c-abi \
47	50	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -50,5 +53,4 @@ stage3-release/bin/zig build test-modules test-c-abi \
50	53	-Dskip-single-threaded \
51	54	-Dtarget=native-native-musl \
52	55	--search-prefix "$PREFIX" \
53		--zig-lib-dir "$PWD/../lib" \
54	56	--test-timeout 4m

ci/s390x-linux-debug.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
44	47	stage3-debug/bin/zig build test docs \
45	48	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -47,7 +50,6 @@ stage3-debug/bin/zig build test docs \
47	50	-Dskip-non-native \
48	51	-Dtarget=native-native-musl \
49	52	--search-prefix "$PREFIX" \
50		--zig-lib-dir "$PWD/../lib" \
51	53	--test-timeout 4m
52	54
53	55	stage3-debug/bin/zig build \

ci/s390x-linux-release.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
44	47	stage3-release/bin/zig build test docs \
45	48	--maxrss ${ZSF_MAX_RSS:-0} \
...	...	@@ -47,7 +50,6 @@ stage3-release/bin/zig build test docs \
47	50	-Dskip-non-native \
48	51	-Dtarget=native-native-musl \
49	52	--search-prefix "$PREFIX" \
50		--zig-lib-dir "$PWD/../lib" \
51	53	--test-timeout 4m
52	54
53	55	# Ensure that stage3 and stage4 are byte-for-byte identical.

ci/x86_64-freebsd-debug.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-debug/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
...	...	@@ -51,7 +54,6 @@ stage3-debug/bin/zig build test docs \
51	54	-Dskip-windows \
52	55	-Dskip-darwin \
53	56	--search-prefix "$PREFIX" \
54		--zig-lib-dir "$PWD/../lib" \
55	57	--test-timeout 2m
56	58
57	59	stage3-debug/bin/zig build \

ci/x86_64-freebsd-release.sh+3-1

...	...	@@ -40,6 +40,9 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-release/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
...	...	@@ -51,7 +54,6 @@ stage3-release/bin/zig build test docs \
51	54	-Dskip-windows \
52	55	-Dskip-darwin \
53	56	--search-prefix "$PREFIX" \
54		--zig-lib-dir "$PWD/../lib" \
55	57	--test-timeout 2m
56	58
57	59	# Ensure that the fuzzer at least compiles.

ci/x86_64-linux-debug-llvm.sh+3-1

...	...	@@ -43,6 +43,9 @@ unset CXX
43	43
44	44	ninja install
45	45
	46	# Must be done after zig cc is finished.
	47	export ZIG_LIB_DIR="$PWD/../lib"
	48
46	49	# simultaneously test building self-hosted without LLVM and with 32-bit arm
47	50	stage3-debug/bin/zig build \
48	51	-Dtarget=arm-linux-musleabihf \
...	...	@@ -63,7 +66,6 @@ stage3-debug/bin/zig build test docs \
63	66	-Dskip-darwin \
64	67	-Dtarget=native-native-musl \
65	68	--search-prefix "$PREFIX" \
66		--zig-lib-dir "$PWD/../lib" \
67	69	-Denable-superhtml \
68	70	--test-timeout 12m
69	71

ci/x86_64-linux-debug.sh+4-2

...	...	@@ -42,13 +42,16 @@ unset CXX
42	42
43	43	ninja install
44	44
	45	# Must be done after zig cc is finished.
	46	export ZIG_LIB_DIR="$PWD/../lib"
	47	export ZIG_DEBUG_MAKER=1
	48
45	49	# simultaneously test building self-hosted without LLVM and with 32-bit arm
46	50	stage3-debug/bin/zig build \
47	51	-Dtarget=arm-linux-musleabihf \
48	52	-Dno-lib
49	53
50	54	stage3-debug/bin/zig build test docs \
51		--maker-opt=Debug \
52	55	--maxrss ${ZSF_MAX_RSS:-0} \
53	56	-Dlldb=$HOME/deps/lldb-zig/Debug-7c1090fd46/bin/lldb \
54	57	-fqemu \
...	...	@@ -63,7 +66,6 @@ stage3-debug/bin/zig build test docs \
63	66	-Dskip-llvm \
64	67	-Dtarget=native-native-musl \
65	68	--search-prefix "$PREFIX" \
66		--zig-lib-dir "$PWD/../lib" \
67	69	-Denable-superhtml \
68	70	--test-timeout 10m
69	71

ci/x86_64-linux-release.sh+7-3

...	...	@@ -48,6 +48,9 @@ unset CXX
48	48
49	49	ninja install
50	50
	51	# Must not be set while using the other `zig cc` which has its own zig lib dir.
	52	export ZIG_LIB_DIR="$PWD/../lib"
	53
51	54	# Covers several things:
52	55	# 1. building the compiler without LLVM
53	56	# 2. 32-bit
...	...	@@ -66,7 +69,6 @@ stage3-release/bin/zig build test docs \
66	69	-Dstatic-llvm \
67	70	-Dtarget=native-native-musl \
68	71	--search-prefix "$PREFIX" \
69		--zig-lib-dir "$PWD/../lib" \
70	72	-Denable-superhtml \
71	73	--test-timeout 12m
72	74
...	...	@@ -97,6 +99,7 @@ cd ../build-new
97	99
98	100	export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
99	101	export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
	102	unset ZIG_LIB_DIR
100	103
101	104	cmake .. \
102	105	-DCMAKE_PREFIX_PATH="$PREFIX" \
...	...	@@ -115,11 +118,12 @@ unset CXX
115	118
116	119	ninja install
117	120
	121	export ZIG_LIB_DIR="$PWD/../lib"
	122
118	123	stage3/bin/zig test ../test/behavior.zig
119	124	stage3/bin/zig build -p stage4 \
120	125	-Dstatic-llvm \
121	126	-Dtarget=native-native-musl \
122	127	-Dno-lib \
123		--search-prefix "$PREFIX" \
124		--zig-lib-dir "$PWD/../lib"
	128	--search-prefix "$PREFIX"
125	129	stage4/bin/zig test ../test/behavior.zig

ci/x86_64-netbsd-debug.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-debug/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 2m
50	52
51	53	stage3-debug/bin/zig build \

ci/x86_64-netbsd-release.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-release/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 2m
50	52
51	53	# Ensure that the fuzzer at least compiles.

ci/x86_64-openbsd-debug.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-debug/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 2m
50	52
51	53	stage3-debug/bin/zig build \

ci/x86_64-openbsd-release.sh+3-1

...	...	@@ -40,12 +40,14 @@ unset CXX
40	40
41	41	ninja install
42	42
	43	# Must be done after zig cc is finished.
	44	export ZIG_LIB_DIR="$PWD/../lib"
	45
43	46	stage3-release/bin/zig build test docs \
44	47	--maxrss ${ZSF_MAX_RSS:-0} \
45	48	-Dstatic-llvm \
46	49	-Dskip-non-native \
47	50	--search-prefix "$PREFIX" \
48		--zig-lib-dir "$PWD/../lib" \
49	51	--test-timeout 2m
50	52
51	53	# Ensure that the fuzzer at least compiles.

ci/x86_64-windows-debug.ps1+3-4

...	...	@@ -2,7 +2,6 @@ $TARGET = "x86_64-windows-gnu"
2	2	$MCPU = "baseline"
3	3	$PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
4	4	$ZIG = "$PREFIX_PATH\bin\zig.exe"
5		$ZIG_LIB_DIR = "$(Get-Location)\lib"
6	5	$ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 }
7	6
8	7	function CheckLastExitCode {
...	...	@@ -42,10 +41,12 @@ CheckLastExitCode
42	41	ninja install
43	42	CheckLastExitCode
44	43
	44	# Must be done after zig cc is finished.
	45	$Env:ZIG_LIB_DIR="$(Get-Location)\..\lib"
	46
45	47	Write-Output "Main test suite..."
46	48	stage3-debug\bin\zig build test docs `
47	49	--maxrss $ZSF_MAX_RSS `
48		--zig-lib-dir "$ZIG_LIB_DIR" `
49	50	--search-prefix "$PREFIX_PATH" `
50	51	-Dstatic-llvm `
51	52	-Dskip-non-native `
...	...	@@ -56,7 +57,6 @@ CheckLastExitCode
56	57
57	58	Write-Output "Build x86_64-windows-msvc behavior tests using the C backend..."
58	59	stage3-debug\bin\zig build-obj `
59		--zig-lib-dir "$ZIG_LIB_DIR" `
60	60	-ofmt=c `
61	61	-OReleaseSmall `
62	62	--name compiler_rt `
...	...	@@ -67,7 +67,6 @@ stage3-debug\bin\zig build-obj `
67	67	CheckLastExitCode
68	68
69	69	stage3-debug\bin\zig test `
70		--zig-lib-dir "$ZIG_LIB_DIR" `
71	70	-ofmt=c `
72	71	-femit-bin="behavior-x86_64-windows-msvc.c" `
73	72	--test-no-exec `

ci/x86_64-windows-release.ps1+3-4

...	...	@@ -2,7 +2,6 @@ $TARGET = "x86_64-windows-gnu"
2	2	$MCPU = "baseline"
3	3	$PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
4	4	$ZIG = "$PREFIX_PATH\bin\zig.exe"
5		$ZIG_LIB_DIR = "$(Get-Location)\lib"
6	5	$ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 }
7	6
8	7	function CheckLastExitCode {
...	...	@@ -42,10 +41,12 @@ CheckLastExitCode
42	41	ninja install
43	42	CheckLastExitCode
44	43
	44	# Must be done after zig cc is finished.
	45	$Env:ZIG_LIB_DIR="$(Get-Location)\..\lib"
	46
45	47	Write-Output "Main test suite..."
46	48	stage3-release\bin\zig.exe build test docs `
47	49	--maxrss $ZSF_MAX_RSS `
48		--zig-lib-dir "$ZIG_LIB_DIR" `
49	50	--search-prefix "$PREFIX_PATH" `
50	51	-Dstatic-llvm `
51	52	-Dskip-non-native `
...	...	@@ -82,7 +83,6 @@ CheckLastExitCode
82	83
83	84	Write-Output "Build x86_64-windows-msvc behavior tests using the C backend..."
84	85	stage3-release\bin\zig.exe build-obj `
85		--zig-lib-dir "$ZIG_LIB_DIR" `
86	86	-ofmt=c `
87	87	-OReleaseSmall `
88	88	--name compiler_rt `
...	...	@@ -93,7 +93,6 @@ stage3-release\bin\zig.exe build-obj `
93	93	CheckLastExitCode
94	94
95	95	stage3-release\bin\zig.exe test `
96		--zig-lib-dir "$ZIG_LIB_DIR" `
97	96	-ofmt=c `
98	97	-femit-bin="behavior-x86_64-windows-msvc.c" `
99	98	--test-no-exec `

lib/compiler/Maker.zig+1946-299

...	...	@@ -1,5 +1,7 @@
1	1	const Maker = @This();
	2
2	3	const builtin = @import("builtin");
	4	const native_os = builtin.os.tag;
3	5
4	6	const std = @import("std");
5	7	const Allocator = std.mem.Allocator;
...	...	@@ -17,6 +19,9 @@ const log = std.log;
17	19	const mem = std.mem;
18	20	const process = std.process;
19	21	const Color = std.zig.Color;
	22	const EnvVar = std.zig.EnvVar;
	23	const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename;
	24	const stringToEnum = std.meta.stringToEnum;
20	25
21	26	const Fuzz = @import("Maker/Fuzz.zig");
22	27	const Graph = @import("Maker/Graph.zig");
...	...	@@ -25,10 +30,11 @@ const Watch = @import("Maker/Watch.zig");
25	30	const WebServer = @import("Maker/WebServer.zig");
26	31	const ScannedConfig = @import("Maker/ScannedConfig.zig");
27	32	const PkgConfig = @import("Maker/PkgConfig.zig");
	33	const Fetch = @import("Maker/Fetch.zig");
	34	const Package = @import("Maker/Package.zig");
28	35
29	36	pub const std_options: std.Options = .{
30	37	.side_channels_mitigations = .none,
31		.http_disable_tls = true,
32	38	};
33	39
34	40	gpa: Allocator,
...	...	@@ -45,7 +51,7 @@ max_rss_mutex: Io.Mutex,
45	51	skip_oom_steps: bool,
46	52	unit_test_timeout_ns: ?u64,
47	53	watch: bool,
48		web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn,
	54	web_server: ?*AvoidableWebServer,
49	55	/// Allocated into `gpa`.
50	56	memory_blocked_steps: std.ArrayList(Configuration.Step.Index),
51	57	/// Allocated into `gpa`.
...	...	@@ -61,6 +67,8 @@ var stdio_buffer_allocation: [256]u8 = undefined;
61	67	var stdout_writer_allocation: Io.File.Writer = undefined;
62	68	var debug_maker_leaks: bool = false;
63	69
	70	const AvoidableWebServer = if (builtin.single_threaded) void else WebServer;
	71
64	72	const is_debug_mode = builtin.mode == .Debug;
65	73	const use_safe_allocator = switch (builtin.mode) {
66	74	.Debug, .ReleaseSafe => true,
...	...	@@ -99,11 +107,35 @@ const ErrorStyle = enum {
99	107	};
100	108	const MultilineErrors = enum { indent, newline, none };
101	109	const Summary = enum { all, new, failures, line, none };
	110	const PrintConfiguration = enum { none, zon, path };
	111
	112	/// Used to build the -M flags to pass to build-exe.
	113	pub const CliModule = struct {
	114	name: []const u8,
	115	root_path: []const u8,
	116	deps: Deps = .empty,
	117
	118	const Deps = std.array_hash_map.String(*CliModule);
	119
	120	fn lower(cm: const CliModule, arena: Allocator, gpa: Allocator, argv: std.ArrayList([]const u8)) !void {
	121	try argv.ensureUnusedCapacity(gpa, 2 * cm.deps.count() + 1);
	122	for (cm.deps.keys(), cm.deps.values()) \|name, dep\| {
	123	argv.appendAssumeCapacity("--dep");
	124	if (mem.eql(u8, name, dep.name)) {
	125	argv.appendAssumeCapacity(dep.name);
	126	} else {
	127	argv.appendAssumeCapacity(try arena.print("{s}={s}", .{ name, dep.name }));
	128	}
	129	}
	130	argv.appendAssumeCapacity(try arena.print("-M{s}={s}", .{ cm.name, cm.root_path }));
	131	}
	132	};
102	133
103	134	pub fn main(init: process.Init.Minimal) !void {
104	135	// The build runner is long-lived in the following use cases:
105	136	// * `--watch` mode
106	137	// * `--webui` mode
	138	// * `--fuzz` mode
107	139	// * A project that has a large, complex build graph.
108	140	const gpa = if (use_safe_allocator) safe_allocator_instance.allocator() else std.heap.smp_allocator;
109	141	defer if (use_safe_allocator) {
...	...	@@ -117,77 +149,63 @@ pub fn main(init: process.Init.Minimal) !void {
117	149	defer threaded.deinit();
118	150	const io = threaded.io();
119	151
120		// ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
121	152	var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
122	153	defer arena_instance.deinit();
123	154	defer if (debugMakerLeaks()) log.debug("used {Bi} of arena", .{arena_instance.queryCapacity()});
124	155	const arena = arena_instance.allocator();
125	156
126	157	const args = try init.args.toSlice(arena);
127
128		// skip my own exe name
129		var arg_idx: usize = 1;
130
131		const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig");
132		const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir");
133		const build_root = expectArgOrFatal(args, &arg_idx, "--build-root");
134		const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache");
135		const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache");
136		const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration");
	158	var arg_i: usize = 1;
	159	const cmd_name = nextArgOrFatal(args, &arg_i);
	160	const zig_lib_arg = prefixedArgOrFatal(args, &arg_i, "--zig-lib=");
	161	const zig_exe_arg = prefixedArgOrFatal(args, &arg_i, "--zig=");
	162	const global_cache_arg = prefixedArgOrFatal(args, &arg_i, "--global-cache=");
	163	const seed_arg = prefixedArgOrFatal(args, &arg_i, "--seed=");
137	164
138	165	const cwd: Dir = .cwd();
139	166
140	167	const zig_lib_directory: Cache.Directory = .{
141		.path = zig_lib_dir,
142		.handle = try cwd.openDir(io, zig_lib_dir, .{}),
143		};
144
145		const build_root_directory: Cache.Directory = .{
146		.path = build_root,
147		.handle = try cwd.openDir(io, build_root, .{}),
148		};
149
150		const local_cache_directory: Cache.Directory = .{
151		.path = local_cache_root,
152		.handle = try cwd.createDirPathOpen(io, local_cache_root, .{}),
	168	.path = zig_lib_arg,
	169	.handle = try cwd.openDir(io, zig_lib_arg, .{}),
153	170	};
154	171
155	172	const global_cache_directory: Cache.Directory = .{
156		.path = global_cache_root,
157		.handle = try cwd.createDirPathOpen(io, global_cache_root, .{}),
	173	.path = global_cache_arg,
	174	.handle = try cwd.createDirPathOpen(io, global_cache_arg, .{}),
158	175	};
159	176
160	177	var graph: Graph = .{
161	178	.io = io,
162	179	.arena = arena,
163		.cache = .{
164		.io = io,
165		.gpa = gpa,
166		.manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}),
167		.cwd = try process.currentPathAlloc(io, arena),
168		},
169		.zig_exe = zig_exe,
	180	.cache = undefined,
	181	.zig_exe = zig_exe_arg,
170	182	.environ_map = try init.environ.createMap(arena),
171	183	.global_cache_root = global_cache_directory,
172		.local_cache_root = local_cache_directory,
	184	.local_cache_root = undefined,
173	185	.zig_lib_directory = zig_lib_directory,
174		.build_root_directory = build_root_directory,
	186	.build_root_directory = undefined,
	187	.random_seed = parseRandomSeed(seed_arg),
175	188	};
176	189
177		graph.cache.addPrefix(.{ .path = null, .handle = cwd });
178		graph.cache.addPrefix(build_root_directory);
179		graph.cache.addPrefix(local_cache_directory);
180		graph.cache.addPrefix(global_cache_directory);
181		graph.cache.hash.addBytes(builtin.zig_version_string);
	190	const cmd = stringToEnum(enum { libc, init, fetch, build }, cmd_name) orelse
	191	fatal("bad command name: {q}", .{cmd_name});
	192	switch (cmd) {
	193	.libc => return cmdLibC(gpa, &graph, args[arg_i..]),
	194	.init => return cmdInit(gpa, &graph, args[arg_i..]),
	195	.fetch => return cmdFetch(gpa, &graph, args[arg_i..]),
	196	.build => {},
	197	}
182	198
183	199	var step_names: std.ArrayList([]const u8) = .empty;
184	200	var help_menu = false;
185	201	var steps_menu = false;
186		var print_configuration = false;
	202	var print_configuration: PrintConfiguration = .none;
187	203	var override_install_prefix: ?[]const u8 = null;
188	204	var override_lib_dir: ?[]const u8 = null;
189	205	var override_bin_dir: ?[]const u8 = null;
190	206	var override_include_dir: ?[]const u8 = null;
	207	var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(&graph.environ_map);
	208	var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(&graph.environ_map);
191	209	var error_style: ErrorStyle = .verbose;
192	210	var multiline_errors: MultilineErrors = .indent;
193	211	var summary: ?Summary = null;
...	...	@@ -201,39 +219,125 @@ pub fn main(init: process.Init.Minimal) !void {
201	219	var webui_listen: ?Io.net.IpAddress = null;
202	220	var debug_pkg_config = false;
203	221	var run_args: ?[]const []const u8 = null;
204
205		if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) \|str\| {
206		if (std.meta.stringToEnum(ErrorStyle, str)) \|style\| {
	222	var build_file: ?[]const u8 = null;
	223
	224	var configure_argv: std.ArrayList([]const u8) = .empty;
	225	var cached_passthru_configure: std.ArrayList(u32) = .empty;
	226	var forks: std.ArrayList(Fork) = .empty;
	227	var system_pkg_dir_path: ?[]const u8 = null;
	228	var fetch_only = false;
	229	var fetch_mode: Fetch.JobQueue.Mode = .needed;
	230	var debug_target: ?[]const u8 = null;
	231	var cache_poison: std.Build.Graph.CachePoison = .pure;
	232
	233	if (EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) \|str\| {
	234	if (stringToEnum(ErrorStyle, str)) \|style\| {
207	235	error_style = style;
208	236	}
209	237	}
210	238
211		if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) \|str\| {
212		if (std.meta.stringToEnum(MultilineErrors, str)) \|style\| {
	239	if (EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) \|str\| {
	240	if (stringToEnum(MultilineErrors, str)) \|style\| {
213	241	multiline_errors = style;
214	242	}
215	243	}
216	244
217		while (nextArg(args, &arg_idx)) \|arg\| {
	245	try configure_argv.ensureUnusedCapacity(arena, 16);
	246	try cached_passthru_configure.ensureUnusedCapacity(arena, 16);
	247
	248	_ = configure_argv.addOneAssumeCapacity(); // configurer executable
	249	configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", graph.zig_exe };
	250	configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined };
	251	const conf_argv_index_build_root = configure_argv.items.len - 1;
	252
	253	while (nextArg(args, &arg_i)) \|arg\| {
218	254	if (mem.startsWith(u8, arg, "-")) {
219		if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	255	try configure_argv.ensureUnusedCapacity(arena, 2);
	256	if (mem.startsWith(u8, arg, "-D") or
	257	mem.startsWith(u8, arg, "-fsys=") or
	258	mem.startsWith(u8, arg, "-fno-sys=") or
	259	mem.startsWith(u8, arg, "--release=") or
	260	mem.eql(u8, arg, "--release"))
	261	{
	262	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	263	configure_argv.appendAssumeCapacity(arg);
	264	} else if (mem.eql(u8, arg, "--system")) {
	265	system_pkg_dir_path = nextArgOrFatal(args, &arg_i);
	266
	267	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	268	configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path.
	269	} else if (mem.cutPrefix(u8, arg, "--color=")) \|rest\| {
	270	color = stringToEnum(Color, rest) orelse
	271	fatalWithHint("expected --color=[auto\|on\|off]; found {q}", .{arg});
	272
	273	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	274	configure_argv.appendAssumeCapacity(arg);
	275	} else if (mem.eql(u8, arg, "--color")) {
	276	const next_arg = nextArgOrFatal(args, &arg_i);
	277	color = stringToEnum(Color, next_arg) orelse
	278	fatalWithHint("expected [auto\|on\|off] found {q}", .{next_arg});
	279
	280	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	281	configure_argv.appendAssumeCapacity(try arena.print("--color={t}", .{color}));
	282	} else if (mem.eql(u8, arg, "--cache-poison")) {
	283	cache_poison = .poisoned;
	284	configure_argv.appendAssumeCapacity("--cache-poison=poisoned");
	285	} else if (mem.cutPrefix(u8, arg, "--cache-poison=")) \|rest\| {
	286	// Allow the configurer process to report parse failure.
	287	if (stringToEnum(std.Build.Graph.CachePoison, rest)) \|poison\| {
	288	cache_poison = poison;
	289	}
	290	configure_argv.appendAssumeCapacity(arg);
	291	} else if (mem.eql(u8, arg, "--verbose")) {
	292	// Intentionally is added both to make and configure but
	293	// does not go into the cache hash.
	294	configure_argv.appendAssumeCapacity(arg);
	295	graph.verbose = true;
	296	} else if (mem.eql(u8, arg, "--search-prefix")) {
	297	const prefix = nextArgOrFatal(args, &arg_i);
	298
	299	// This argument is cache poisonous: it does not go into
	300	// the cache and configurer must set the poison bit when
	301	// choosing to observe it.
	302	configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, prefix };
	303
	304	try graph.search_prefixes.append(arena, prefix);
	305	} else if (mem.eql(u8, arg, "--cache-dir")) {
	306	override_local_cache_dir = nextArgOrFatal(args, &arg_i);
	307	} else if (mem.eql(u8, arg, "--pkg-dir")) {
	308	override_pkg_dir = nextArgOrFatal(args, &arg_i);
	309	} else if (mem.eql(u8, arg, "--fetch")) {
	310	fetch_only = true;
	311	} else if (mem.cutPrefix(u8, arg, "--fetch=")) \|rest\| {
	312	fetch_only = true;
	313	fetch_mode = stringToEnum(Fetch.JobQueue.Mode, rest) orelse
	314	fatal("expected [needed\|all] after \"--fetch=\", found {q}", .{rest});
	315	} else if (mem.cutPrefix(u8, arg, "--fork=")) \|rest\| {
	316	try forks.append(arena, .init(rest));
	317	} else if (mem.eql(u8, arg, "--fork")) {
	318	try forks.append(arena, .init(nextArgOrFatal(args, &arg_i)));
	319	} else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
220	320	help_menu = true;
221	321	} else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) {
222	322	steps_menu = true;
223	323	} else if (mem.eql(u8, arg, "--print-configuration")) {
224		print_configuration = true;
	324	print_configuration = .zon;
	325	} else if (mem.eql(u8, arg, "--print-configuration-path")) {
	326	print_configuration = .path;
225	327	} else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) {
226		override_install_prefix = nextArgOrFatal(args, &arg_idx);
	328	override_install_prefix = nextArgOrFatal(args, &arg_i);
	329	} else if (mem.eql(u8, arg, "--build-file")) {
	330	build_file = nextArgOrFatal(args, &arg_i);
227	331	} else if (mem.eql(u8, arg, "--prefix-lib-dir")) {
228		override_lib_dir = nextArgOrFatal(args, &arg_idx);
	332	override_lib_dir = nextArgOrFatal(args, &arg_i);
229	333	} else if (mem.eql(u8, arg, "--prefix-exe-dir")) {
230		override_bin_dir = nextArgOrFatal(args, &arg_idx);
	334	override_bin_dir = nextArgOrFatal(args, &arg_i);
231	335	} else if (mem.eql(u8, arg, "--prefix-include-dir")) {
232		override_include_dir = nextArgOrFatal(args, &arg_idx);
	336	override_include_dir = nextArgOrFatal(args, &arg_i);
233	337	} else if (mem.eql(u8, arg, "--sysroot")) {
234		graph.sysroot = nextArgOrFatal(args, &arg_idx);
	338	graph.sysroot = nextArgOrFatal(args, &arg_i);
235	339	} else if (mem.eql(u8, arg, "--maxrss")) {
236		const max_rss_text = nextArgOrFatal(args, &arg_idx);
	340	const max_rss_text = nextArgOrFatal(args, &arg_i);
237	341	max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch \|err\|
238	342	fatal("invalid byte size {q}: {t}", .{ max_rss_text, err });
239	343	} else if (mem.eql(u8, arg, "--skip-oom-steps")) {
...	...	@@ -253,7 +357,7 @@ pub fn main(init: process.Init.Minimal) !void {
253	357	.{ "h", std.time.ns_per_hour },
254	358	.{ "hour", std.time.ns_per_hour },
255	359	};
256		const timeout_str = nextArgOrFatal(args, &arg_idx);
	360	const timeout_str = nextArgOrFatal(args, &arg_i);
257	361	const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal(
258	362	"invalid timeout {q}: expected unit (ns, us, ms, s, m, h)",
259	363	.{timeout_str},
...	...	@@ -273,51 +377,37 @@ pub fn main(init: process.Init.Minimal) !void {
273	377	.{ timeout_str, num_str, err },
274	378	);
275	379	test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed);
276		} else if (mem.eql(u8, arg, "--search-prefix")) {
277		try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx));
278	380	} else if (mem.eql(u8, arg, "--libc")) {
279		graph.libc_file = nextArgOrFatal(args, &arg_idx);
280		} else if (mem.eql(u8, arg, "--color")) {
281		const next_arg = nextArg(args, &arg_idx) orelse
282		fatalWithHint("expected [auto\|on\|off] after {q}", .{arg});
283		color = std.meta.stringToEnum(Color, next_arg) orelse {
284		fatalWithHint("expected [auto\|on\|off] after {q}, found {q}", .{
285		arg, next_arg,
286		});
287		};
	381	graph.libc_file = nextArgOrFatal(args, &arg_i);
288	382	} else if (mem.eql(u8, arg, "--error-style")) {
289		const next_arg = nextArg(args, &arg_idx) orelse
	383	const next_arg = nextArg(args, &arg_i) orelse
290	384	fatalWithHint("expected style after {q}", .{arg});
291		error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse {
	385	error_style = stringToEnum(ErrorStyle, next_arg) orelse {
292	386	fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg });
293	387	};
294	388	} else if (mem.eql(u8, arg, "--multiline-errors")) {
295		const next_arg = nextArg(args, &arg_idx) orelse
	389	const next_arg = nextArg(args, &arg_i) orelse
296	390	fatalWithHint("expected style after {q}", .{arg});
297		multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse {
	391	multiline_errors = stringToEnum(MultilineErrors, next_arg) orelse {
298	392	fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg });
299	393	};
300	394	} else if (mem.eql(u8, arg, "--summary")) {
301		const next_arg = nextArg(args, &arg_idx) orelse
	395	const next_arg = nextArg(args, &arg_i) orelse
302	396	fatalWithHint("expected [all\|new\|failures\|line\|none] after {q}", .{arg});
303		summary = std.meta.stringToEnum(Summary, next_arg) orelse {
	397	summary = stringToEnum(Summary, next_arg) orelse {
304	398	fatalWithHint("expected [all\|new\|failures\|line\|none] after {q}, found {q}", .{
305	399	arg, next_arg,
306	400	});
307	401	};
308		} else if (mem.eql(u8, arg, "--seed")) {
309		const next_arg = nextArg(args, &arg_idx) orelse
310		fatalWithHint("expected u32 after {q}", .{arg});
311		graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch \|err\| {
312		fatal("unable to parse seed {q} as unsigned 32-bit integer: {t}", .{ next_arg, err });
313		};
	402	} else if (mem.cutPrefix(u8, arg, "--seed=")) \|rest\| {
	403	graph.random_seed = parseRandomSeed(rest);
314	404	} else if (mem.eql(u8, arg, "--build-id")) {
315	405	graph.build_id = .fast;
316	406	} else if (mem.cutPrefix(u8, arg, "--build-id=")) \|style\| {
317	407	graph.build_id = std.zig.BuildId.parse(style) catch \|err\|
318	408	fatal("unable to parse --build-id style {q}: {t}", .{ style, err });
319	409	} else if (mem.eql(u8, arg, "--debounce")) {
320		const next_arg = nextArg(args, &arg_idx) orelse
	410	const next_arg = nextArg(args, &arg_i) orelse
321	411	fatalWithHint("expected u16 after {q}", .{arg});
322	412	debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch \|err\| {
323	413	fatal("unable to parse debounce interval {q} as unsigned 16-bit integer: {t}", .{
...	...	@@ -332,9 +422,10 @@ pub fn main(init: process.Init.Minimal) !void {
332	422	webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch \|err\| {
333	423	fatal("invalid web UI address {q}: {t}", .{ addr_str, err });
334	424	};
	425	} else if (mem.eql(u8, arg, "--debug-target")) {
	426	debug_target = nextArgOrFatal(args, &arg_i);
335	427	} else if (mem.eql(u8, arg, "--debug-log")) {
336		const next_arg = nextArgOrFatal(args, &arg_idx);
337		try graph.debug_log_scopes.append(arena, next_arg);
	428	try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i));
338	429	} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
339	430	graph.debug_compile_errors = true;
340	431	} else if (mem.eql(u8, arg, "--debug-incremental")) {
...	...	@@ -344,19 +435,19 @@ pub fn main(init: process.Init.Minimal) !void {
344	435	} else if (mem.eql(u8, arg, "--debug-rt")) {
345	436	graph.debug_compiler_runtime_libs = .Debug;
346	437	} else if (mem.cutPrefix(u8, arg, "--debug-rt=")) \|rest\| {
347		graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse
	438	graph.debug_compiler_runtime_libs = stringToEnum(std.lang.OptimizeMode, rest) orelse
348	439	fatal("unrecognized optimization mode: {s}", .{rest});
349	440	} else if (is_debug_mode and mem.eql(u8, arg, "--debug-maker-leaks")) {
350	441	debug_maker_leaks = true;
351	442	} else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) {
352	443	// --glibc-runtimes was the old name of the flag; kept for compatibility for now.
353		graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
354		} else if (mem.eql(u8, arg, "--verbose")) {
355		graph.verbose = true;
	444	graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_i);
356	445	} else if (mem.eql(u8, arg, "--verbose-air")) {
357	446	graph.verbose_air = true;
358	447	} else if (mem.eql(u8, arg, "--verbose-cc")) {
359	448	graph.verbose_cc = true;
	449	} else if (mem.eql(u8, arg, "--verbose-link")) {
	450	graph.verbose_link = true;
360	451	} else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
361	452	graph.verbose_llvm_ir = true;
362	453	} else if (mem.eql(u8, arg, "--watch")) {
...	...	@@ -439,7 +530,7 @@ pub fn main(init: process.Init.Minimal) !void {
439	530	} else if (mem.eql(u8, arg, "-fno-reference-trace")) {
440	531	graph.reference_trace = null;
441	532	} else if (mem.eql(u8, arg, "--error-limit")) {
442		const next_arg = nextArgOrFatal(args, &arg_idx);
	533	const next_arg = nextArgOrFatal(args, &arg_i);
443	534	graph.error_limit = std.fmt.parseUnsigned(u32, next_arg, 0) catch \|err\|
444	535	fatal("unable to parse error limit {q}: {t}", .{ next_arg, err });
445	536	} else if (mem.cutPrefix(u8, arg, "-j")) \|text\| {
...	...	@@ -449,7 +540,7 @@ pub fn main(init: process.Init.Minimal) !void {
449	540	threaded.setAsyncLimit(.limited(n));
450	541	graph.max_jobs = n;
451	542	} else if (mem.eql(u8, arg, "--")) {
452		run_args = argsRest(args, arg_idx);
	543	run_args = argsRest(args, arg_i);
453	544	break;
454	545	} else {
455	546	fatalWithHint("unrecognized argument: {s}", .{arg});
...	...	@@ -459,8 +550,51 @@ pub fn main(init: process.Init.Minimal) !void {
459	550	}
460	551	}
461	552
462		const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map);
463		const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
	553	const early_exit_mode = fetch_only or help_menu or steps_menu or print_configuration != .none;
	554	const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null);
	555
	556	process.raiseFileDescriptorLimit();
	557
	558	const cwd_path = std.zig.getResolvedCwd(io, arena) catch \|err\|
	559	fatal("resolving current directory path failed: {t}", .{err});
	560
	561	const build_root = try findBuildRoot(arena, io, .{
	562	.cwd_path = cwd_path,
	563	.build_file = build_file,
	564	});
	565
	566	graph.build_root_directory = build_root.directory;
	567	graph.local_cache_root = if (override_local_cache_dir) \|unresolved_path\| std.zig.Directories.openUnresolved(
	568	arena,
	569	io,
	570	cwd_path,
	571	unresolved_path,
	572	.@"local cache",
	573	) else .{
	574	.path = try Dir.path.join(arena, &.{ build_root.directory.path orelse ".", default_local_zig_cache_basename }),
	575	.handle = try build_root.directory.handle.createDirPathOpen(io, default_local_zig_cache_basename, .{}),
	576	};
	577	graph.cache = .{
	578	.io = io,
	579	.gpa = gpa,
	580	.manifest_dir = try graph.local_cache_root.handle.createDirPathOpen(io, "h", .{}),
	581	.cwd = cwd_path,
	582	};
	583
	584	graph.cache.addPrefix(.{ .path = null, .handle = cwd });
	585	graph.cache.addPrefix(zig_lib_directory);
	586	graph.cache.addPrefix(graph.local_cache_root);
	587	graph.cache.addPrefix(global_cache_directory);
	588	graph.cache.addPrefix(graph.build_root_directory);
	589	comptime assert(0 == @intFromEnum(std.zig.Server.Message.PathPrefix.cwd));
	590	comptime assert(1 == @intFromEnum(std.zig.Server.Message.PathPrefix.zig_lib));
	591	comptime assert(2 == @intFromEnum(std.zig.Server.Message.PathPrefix.local_cache));
	592	comptime assert(3 == @intFromEnum(std.zig.Server.Message.PathPrefix.global_cache));
	593
	594	graph.cache.hash.addBytes(builtin.zig_version_string);
	595
	596	const NO_COLOR = EnvVar.NO_COLOR.isSet(&graph.environ_map);
	597	const CLICOLOR_FORCE = EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
464	598
465	599	graph.stderr_mode = switch (color) {
466	600	.auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE),
...	...	@@ -468,69 +602,15 @@ pub fn main(init: process.Init.Minimal) !void {
468	602	.off => .no_color,
469	603	};
470	604
471		const scanned_config: ScannedConfig = sc: {
472		const configuration = c: {
473		var file = cwd.openFile(io, configure_path, .{}) catch \|err\|
474		fatal("failed to open configuration file {s}: {t}", .{ configure_path, err });
475		defer file.close(io);
476		break :c Configuration.loadFile(arena, io, file) catch \|err\|
477		fatal("failed to load configuration file {s}: {t}", .{ configure_path, err });
	605	const pkg_root: Path = if (override_pkg_dir) \|p\|
	606	.initCwd(p)
	607	else if (system_pkg_dir_path) \|p\|
	608	.initCwd(p)
	609	else
	610	.{
	611	.root_dir = build_root.directory,
	612	.sub_path = "zig-pkg",
478	613	};
479		// Technically if the configuration is marked as poisoned, we could
480		// already delete the file now, but we leave it around in case the
481		// maker process fails or crashes and it's helpful to be able to repeat
482		// execution of the command line or otherwise inspect the configuration file.
483		const c = &configuration;
484		var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty;
485		for (configuration.steps, 0..) \|*conf_step, step_index_usize\| {
486		if (conf_step.owner != .root) continue;
487		const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize);
488		const flags = conf_step.flags(c);
489		switch (flags.tag) {
490		.top_level => {
491		const name = step_index.ptr(c).name.slice(c);
492		try top_level_steps.put(arena, name, step_index);
493		},
494		else => {},
495		}
496		}
497		for (c.search_prefixes) \|search_prefix\| {
498		try graph.search_prefixes.append(arena, search_prefix.slice(c));
499		}
500		break :sc .{
501		.configuration = configuration,
502		.top_level_steps = top_level_steps,
503		.path = configure_path,
504		};
505		};
506
507		if (help_menu) {
508		var w = initStdoutWriter(io);
509		scanned_config.printUsage(&graph, w) catch \|err\| switch (err) {
510		error.WriteFailed => return stdout_writer_allocation.err.?,
511		else => \|e\| return e,
512		};
513		w.flush() catch return stdout_writer_allocation.err.?;
514		return cleanExit(io, &scanned_config);
515		} else if (steps_menu) {
516		var w = initStdoutWriter(io);
517		scanned_config.printSteps(&graph, w) catch \|err\| switch (err) {
518		error.WriteFailed => return stdout_writer_allocation.err.?,
519		else => \|e\| return e,
520		};
521		w.flush() catch return stdout_writer_allocation.err.?;
522		return cleanExit(io, &scanned_config);
523		} else if (print_configuration) {
524		var w = initStdoutWriter(io);
525		scanned_config.print(w) catch return stdout_writer_allocation.err.?;
526		w.flush() catch return stdout_writer_allocation.err.?;
527		return cleanExit(io, &scanned_config);
528		}
529
530		if (webui_listen != null) {
531		if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{});
532		if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{});
533		}
534	614
535	615	const main_progress_node = std.Progress.start(io, .{
536	616	.disable_printing = (graph.stderr_mode.? == .no_color),
...	...	@@ -544,7 +624,7 @@ pub fn main(init: process.Init.Minimal) !void {
544	624	.root_dir = .cwd(),
545	625	.sub_path = cwd_relative,
546	626	} else .{
547		.root_dir = build_root_directory,
	627	.root_dir = graph.build_root_directory,
548	628	.sub_path = "zig-out",
549	629	};
550	630
...	...	@@ -563,146 +643,1336 @@ pub fn main(init: process.Init.Minimal) !void {
563	643	.sub_path = cwd_relative,
564	644	} else try install_prefix_path.join(arena, "include");
565	645
566		var maker: Maker = .{
567		.gpa = gpa,
568		.graph = &graph,
569		.scanned_config = &scanned_config,
570		.install_paths = .{
571		.prefix = install_prefix_path,
572		.lib = install_lib_path,
573		.bin = install_bin_path,
574		.include = install_include_path,
	646	const now = Io.Clock.Timestamp.now(io, .awake);
	647
	648	var web_server_allocation: AvoidableWebServer = undefined;
	649	const web_server: ?*AvoidableWebServer = if (webui_listen) \|listen_address\| ws: {
	650	if (builtin.single_threaded) fatal("--webui is not yet supported on single-threaded hosts", .{});
	651	web_server_allocation = .init(.{
	652	.graph = &graph,
	653	.root_prog_node = main_progress_node,
	654	.listen_address = listen_address,
	655	.base_timestamp = now,
	656	});
	657	web_server_allocation.start() catch \|err\| fatal("failed to start web server: {t}", .{err});
	658	break :ws &web_server_allocation;
	659	} else null;
	660
	661	while (true) {
	662	// If this fails, we can still start the server and wait for user
	663	// to request a rebuild. If it returns error.FailedButCacheIntact
	664	// we can even still do file system watching and automatically
	665	// rebuild on source changes.
	666	if (configure(&graph, .{
	667	.configure_argv = configure_argv.items,
	668	.conf_argv_index_build_root = conf_argv_index_build_root,
	669	.cached_passthru_configure = cached_passthru_configure.items,
	670
	671	.cache_poison = cache_poison,
	672	.pkg_root = pkg_root,
	673	.build_root = build_root,
	674	.cwd_path = cwd_path,
	675	.color = color,
	676	.debug_target = debug_target,
	677	.parent_progress_node = main_progress_node,
	678	.fetch_mode = fetch_mode,
	679	.system_pkg_dir_path = system_pkg_dir_path,
	680	.fetch_only = fetch_only,
	681	.print_configuration = print_configuration,
	682	.forks = forks.items,
	683	})) \|scanned_config\| {
	684	if (help_menu) {
	685	scanned_config.printUsage(&graph, initStdoutWriter(io)) catch \|err\| switch (err) {
	686	error.WriteFailed => return stdout_writer_allocation.err.?,
	687	else => \|e\| return e,
	688	};
	689	try stdout_writer_allocation.flush();
	690	return cleanExit(io, &scanned_config);
	691	} else if (steps_menu) {
	692	scanned_config.printSteps(&graph, initStdoutWriter(io)) catch \|err\| switch (err) {
	693	error.WriteFailed => return stdout_writer_allocation.err.?,
	694	else => \|e\| return e,
	695	};
	696	try stdout_writer_allocation.flush();
	697	return cleanExit(io, &scanned_config);
	698	} else switch (print_configuration) {
	699	.none => {},
	700	.zon => {
	701	scanned_config.print(initStdoutWriter(io)) catch return stdout_writer_allocation.err.?;
	702	try stdout_writer_allocation.flush();
	703	return cleanExit(io, &scanned_config);
	704	},
	705	.path => unreachable,
	706	}
	707
	708	var maker: Maker = .{
	709	.gpa = gpa,
	710	.graph = &graph,
	711	.scanned_config = &scanned_config,
	712	.install_paths = .{
	713	.prefix = install_prefix_path,
	714	.lib = install_lib_path,
	715	.bin = install_bin_path,
	716	.include = install_include_path,
	717	},
	718
	719	.steps = try arena.alloc(Step, scanned_config.configuration.steps.len),
	720	.generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len),
	721	.run_args = run_args,
	722
	723	.available_rss = max_rss,
	724	.max_rss_is_default = false,
	725	.max_rss_mutex = .init,
	726	.skip_oom_steps = skip_oom_steps,
	727	.unit_test_timeout_ns = test_timeout_ns,
	728
	729	.watch = watch,
	730	.web_server = web_server,
	731	.memory_blocked_steps = .empty,
	732	.step_stack = .empty,
	733	.pkg_config = .{ .debug = debug_pkg_config },
	734
	735	.error_style = error_style,
	736	.multiline_errors = multiline_errors,
	737	.summary = summary orelse if (watch or webui_listen != null) .new else .failures,
	738	};
	739	defer {
	740	maker.memory_blocked_steps.deinit(gpa);
	741	maker.step_stack.deinit(gpa);
	742	}
	743
	744	if (maker.available_rss == 0) {
	745	maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64);
	746	maker.max_rss_is_default = true;
	747	}
	748
	749	maker.prepare(step_names.items) catch \|err\| switch (err) {
	750	error.DependencyLoopDetected, error.InsufficientMemory => {
	751	// TODO handle DependencyLoopDetected as error.FailedButCacheIntact
	752	// and handle InsufficientMemory as error.AlreadyReported
	753	_ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
	754	process.exit(1);
	755	},
	756	else => \|e\| return e,
	757	};
	758
	759	var w: Watch = w: {
	760	if (!watch) break :w undefined;
	761	if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{native_os});
	762	break :w try .init(&maker);
	763	};
	764
	765	if (web_server) \|ws\| try ws.updateConfiguration(&maker);
	766
	767	rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) {
	768	const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
	769	defer io.unlockStderr();
	770	stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch \|err\| switch (err) {
	771	error.WriteFailed => return stderr.file_writer.err.?,
	772	};
	773	}) {
	774	if (web_server) \|ws\| ws.startBuild();
	775
	776	try maker.makeStepNames(step_names.items, main_progress_node, fuzz);
	777
	778	if (web_server) \|ws\| {
	779	if (fuzz) \|mode\| if (mode != .forever) fatal(
	780	"error: limited fuzzing is not implemented yet for --webui",
	781	.{},
	782	);
	783
	784	ws.finishBuild(.{ .fuzz = fuzz != null });
	785	}
	786
	787	if (web_server) \|ws\| {
	788	const c = &scanned_config.configuration;
	789	assert(!watch); // fatal error after CLI parsing
	790	while (true) switch (try ws.wait()) {
	791	.rebuild => {
	792	for (maker.step_stack.keys()) \|step_index\| {
	793	const step = maker.stepByIndex(step_index);
	794	step.state = .precheck_done;
	795	const deps = step_index.ptr(c).deps.slice(c);
	796	step.pending_deps = @intCast(deps.len);
	797	step.reset(&maker);
	798	}
	799	continue :rebuild;
	800	},
	801	};
	802	}
	803
	804	if (!maker.watch) return;
	805
	806	// Comptime-known guard to prevent including the logic below when `!Watch.have_impl`.
	807	if (!Watch.have_impl) unreachable;
	808
	809	try w.update(maker.step_stack.keys());
	810
	811	// Wait until a file system notification arrives. Read all such events
	812	// until the buffer is empty. Then wait for a debounce interval, resetting
	813	// if any more events come in. After the debounce interval has passed,
	814	// trigger a rebuild on all steps with modified inputs, as well as their
	815	// recursive dependants.
	816	var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
	817	const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
	818	w.dir_count, countSubProcesses(&maker),
	819	}) catch &caption_buf;
	820	var debouncing_node = main_progress_node.start(caption, 0);
	821	var in_debounce = false;
	822	while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) {
	823	.timeout => {
	824	assert(in_debounce);
	825	debouncing_node.end();
	826	markFailedStepsDirty(&maker);
	827	continue :rebuild;
	828	},
	829	.dirty => if (!in_debounce) {
	830	in_debounce = true;
	831	debouncing_node.end();
	832	debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0);
	833	},
	834	.clean => {},
	835	};
	836	}
	837	} else \|err\| {
	838	const can_fs_watch = switch (err) {
	839	error.AlreadyReported => false,
	840	error.FailedButCacheIntact => true,
	841	else => \|e\| w: {
	842	log.err("configuration failed: {t}", .{e});
	843	break :w false;
	844	},
	845	};
	846	if (!server_mode) {
	847	_ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
	848	process.exit(1);
	849	}
	850	if (watch and can_fs_watch) {
	851	fatal("(zig build system) TODO set up fs watching even when build.zig compilation fails", .{});
	852	} else {
	853	fatal("(zig build system) TODO stay running and wait for user to request rebuild even when build.zig compilation fails", .{});
	854	}
	855	}
	856	}
	857	}
	858
	859	const ConfigureOptions = struct {
	860	configure_argv: [][]const u8,
	861	conf_argv_index_build_root: usize,
	862	cached_passthru_configure: []const u32,
	863
	864	cache_poison: std.Build.Graph.CachePoison,
	865	pkg_root: Path,
	866	build_root: BuildRoot,
	867	cwd_path: []const u8,
	868	color: Color,
	869	debug_target: ?[]const u8,
	870	parent_progress_node: std.Progress.Node,
	871	fetch_mode: Fetch.JobQueue.Mode,
	872	system_pkg_dir_path: ?[]const u8,
	873	fetch_only: bool,
	874	print_configuration: PrintConfiguration,
	875	forks: []Fork,
	876	};
	877
	878	fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
	879	const configure_argv = options.configure_argv;
	880	const gpa = graph.cache.gpa;
	881	const io = graph.io;
	882	const arena = graph.arena;
	883
	884	// Cache lookup for configure options. If we get a match, we can skip
	885	// execution of the configure script. If not, we get the file path to pass
	886	// to the configure process.
	887	//
	888	// In the hot path, we only check this cache, which means that also
	889	// configure source files need to go in here.
	890	var config_man = graph.cache.obtain();
	891	defer config_man.deinit();
	892
	893	for (options.cached_passthru_configure) \|i\|
	894	config_man.hash.addBytes(configure_argv[i]);
	895
	896	// Prevents a `zig build` from getting a false positive cache hit following
	897	// a `zig build --cache-poison=ignored`.
	898	config_man.hash.add(options.cache_poison == .ignored);
	899
	900	configure_argv[options.conf_argv_index_build_root] = options.build_root.directory.path orelse options.cwd_path;
	901
	902	var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
	903	defer http_client.deinit();
	904
	905	var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
	906	var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
	907
	908	{
	909	// Populate fork_set.
	910	var group: Io.Group = .init;
	911	defer group.cancel(io);
	912
	913	for (options.forks) \|*fork\|
	914	group.async(io, Fork.load, .{ io, gpa, fork, options.color });
	915
	916	try group.await(io);
	917
	918	for (options.forks) \|*fork\| {
	919	if (fork.failed) return error.AlreadyReported;
	920	try fork_set.put(arena, .{
	921	.path = fork.path,
	922	.manifest_ast = fork.manifest_ast,
	923	.manifest = fork.manifest,
	924	.uses = 0,
	925	}, {});
	926	}
	927	}
	928	defer Fork.deinitList(options.forks);
	929
	930	var build_configurer_argv: std.ArrayList([]const u8) = .empty;
	931	defer build_configurer_argv.deinit(gpa);
	932
	933	var dependencies_source: std.ArrayList(u8) = .empty;
	934	defer dependencies_source.deinit(gpa);
	935
	936	const configurer_root_src_path: Cache.Path = .{
	937	.root_dir = graph.zig_lib_directory,
	938	.sub_path = "compiler/configurer.zig",
	939	};
	940
	941	const root_build_src_path: Cache.Path = .{
	942	.root_dir = options.build_root.directory,
	943	.sub_path = options.build_root.build_zig_basename,
	944	};
	945
	946	const configurer_exe_name = "configurer";
	947
	948	try build_configurer_argv.appendSlice(gpa, &.{
	949	graph.zig_exe, "build-exe", //
	950	"--cache-dir", graph.local_cache_root.path orelse ".", //
	951	"--global-cache-dir", graph.global_cache_root.path orelse ".", //
	952	"--zig-lib-dir", graph.zig_lib_directory.path orelse ".", //
	953	"--name", configurer_exe_name, //
	954	"-fsingle-threaded", //
	955	});
	956
	957	// Normally the build runner is compiled for the host target but here is
	958	// some code to help when debugging edits to the build runner so that you
	959	// can make sure it compiles successfully on other targets.
	960	const target_arch_os_abi: ?[]const u8 = if (options.debug_target) \|triple\| t: {
	961	config_man.hash.addBytes(triple);
	962	try build_configurer_argv.appendSlice(gpa, &.{ "-target", triple });
	963	break :t triple;
	964	} else null;
	965
	966	if (graph.libc_file) \|libc_file\| {
	967	try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file });
	968	}
	969	if (graph.reference_trace) \|n\| {
	970	try build_configurer_argv.append(gpa, try arena.print("-freference-trace={d}", .{n}));
	971	}
	972	if (graph.debug_compile_errors) {
	973	try build_configurer_argv.append(gpa, "--debug-compile-errors");
	974	}
	975	try build_configurer_argv.appendSlice(gpa, &.{
	976	"--dep", "@build", //
	977	"--dep", "@dependencies", //
	978	try arena.print("-Mroot={f}", .{configurer_root_src_path}), //
	979	});
	980
	981	// In the loop below, after doing the fetch operation, the argv will be
	982	// truncated at this point, dependencies added, and then the
	983	// "--listen=-" arg appended at the end.
	984	const argv_deps_index = build_configurer_argv.items.len;
	985
	986	const build_mod = try arena.create(CliModule);
	987	build_mod.* = .{
	988	.name = "@build",
	989	.root_path = try root_build_src_path.toString(arena),
	990	};
	991
	992	const deps_mod = try arena.create(CliModule);
	993	deps_mod.* = .{
	994	.name = "@dependencies",
	995	.root_path = undefined,
	996	};
	997
	998	// This loop is re-evaluated when the build script exits with an indication that it
	999	// could not continue due to missing lazy dependencies.
	1000	const configuration_path: Path, const poisoned: bool = cp: while (true) {
	1001	build_mod.deps.clearRetainingCapacity();
	1002	deps_mod.deps.clearRetainingCapacity();
	1003
	1004	// We want to release all the locks before executing the child process, so we make a nice
	1005	// big block here to ensure the cleanup gets run when we extract out our argv.
	1006	{
	1007	{
	1008	const fetch_prog_node = options.parent_progress_node.start("Fetch Packages", 0);
	1009	defer fetch_prog_node.end();
	1010
	1011	// Reset fork match counts.
	1012	for (fork_set.keys()) \|*fork\| fork.uses = 0;
	1013
	1014	var job_queue: Package.Fetch.JobQueue = .{
	1015	.io = io,
	1016	.http_client = &http_client,
	1017	.global_cache = graph.global_cache_root,
	1018	.local_storage = &.{
	1019	.cache_root = .{ .root_dir = graph.local_cache_root },
	1020	.pkg_root = options.pkg_root,
	1021	},
	1022	.recursive = true,
	1023	.debug_hash = false,
	1024	.unlazy_set = unlazy_set,
	1025	.fork_set = fork_set,
	1026	.mode = options.fetch_mode,
	1027	.prog_node = fetch_prog_node,
	1028	.read_only = options.system_pkg_dir_path != null,
	1029	};
	1030	defer job_queue.deinit();
	1031
	1032	if (options.system_pkg_dir_path == null) {
	1033	try http_client.initDefaultProxies(arena, &graph.environ_map);
	1034	}
	1035
	1036	try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
	1037	try job_queue.table.ensureUnusedCapacity(gpa, 1);
	1038
	1039	const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };
	1040
	1041	var fetch: Package.Fetch = .{
	1042	.arena = std.heap.ArenaAllocator.init(gpa),
	1043	.location = .{ .relative_path = phantom_package_root },
	1044	.location_tok = 0,
	1045	.hash_tok = .none,
	1046	.name_tok = 0,
	1047	.lazy_status = .eager,
	1048	.remote_package_root = phantom_package_root,
	1049	.parent_package_root = phantom_package_root,
	1050	.parent_manifest_ast = null,
	1051	.prog_node = fetch_prog_node,
	1052	.job_queue = &job_queue,
	1053	.omit_missing_hash_error = true,
	1054	.allow_missing_paths_field = false,
	1055	.use_latest_commit = false,
	1056
	1057	.package_root = undefined,
	1058	.error_bundle = undefined,
	1059	.manifest = undefined,
	1060	.manifest_ast = undefined,
	1061	.have_manifest = false,
	1062	.computed_hash = undefined,
	1063	.has_build_zig = true,
	1064	.oom_flag = false,
	1065	.latest_commit = null,
	1066
	1067	.cli_module = build_mod,
	1068	};
	1069
	1070	job_queue.all_fetches.appendAssumeCapacity(&fetch);
	1071
	1072	job_queue.table.putAssumeCapacityNoClobber(
	1073	Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
	1074	&fetch,
	1075	);
	1076
	1077	job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
	1078	try job_queue.group.await(io);
	1079
	1080	{
	1081	// Ensure that forks were actually used. This is done
	1082	// before printing manifest errors because using a fork can
	1083	// prevent them.
	1084	var any_unused = false;
	1085	for (fork_set.keys()) \|*fork\| {
	1086	if (fork.uses == 0) {
	1087	log.err("fork {f} matched no {s} packages", .{
	1088	fork.path, fork.manifest.name,
	1089	});
	1090	any_unused = true;
	1091	} else {
	1092	log.info("fork {f} matched {d} {s} packages", .{
	1093	fork.path, fork.uses, fork.manifest.name,
	1094	});
	1095	}
	1096	}
	1097	if (any_unused) return error.FailedButCacheIntact;
	1098	}
	1099
	1100	try job_queue.consolidateErrors();
	1101
	1102	if (fetch.error_bundle.root_list.items.len > 0) {
	1103	var errors = try fetch.error_bundle.toOwnedBundle("");
	1104	errors.renderToStderr(io, .{}, options.color) catch process.exit(1);
	1105	return error.FailedButCacheIntact;
	1106	}
	1107
	1108	if (options.fetch_only) {
	1109	_ = io.lockStderr(&.{}, .no_color) catch {};
	1110	process.exit(0);
	1111	}
	1112
	1113	// Create the dependencies.zig file for configurer to
	1114	// obtain via `@import("@dependencies")`.
	1115	{
	1116	{
	1117	dependencies_source.clearRetainingCapacity();
	1118	var source_writer: Io.Writer.Allocating = .fromArrayList(gpa, &dependencies_source);
	1119	defer dependencies_source = source_writer.toArrayList();
	1120	job_queue.createDependenciesSource(&source_writer.writer) catch \|err\| switch (err) {
	1121	error.WriteFailed => return error.OutOfMemory,
	1122	};
	1123	}
	1124	// Atomically create the file in a directory named after the hash of its contents.
	1125	var hh: Cache.HashHelper = .{};
	1126	hh.addBytes(builtin.zig_version_string);
	1127	hh.addBytes(dependencies_source.items);
	1128	const hex_digest = hh.final();
	1129	const dependencies_zig_path: Path = .{
	1130	.root_dir = graph.local_cache_root,
	1131	.sub_path = try arena.print("o/{s}/dependencies.zig", .{&hex_digest}),
	1132	};
	1133	var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic(
	1134	io,
	1135	dependencies_zig_path.sub_path,
	1136	.{ .make_path = true, .replace = true },
	1137	);
	1138	defer atomic_file.deinit(io);
	1139	atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch \|err\|
	1140	fatal("writing dependencies.zig contents: {t}", .{err});
	1141	atomic_file.replace(io) catch \|err\|
	1142	fatal("replacing {f}: {t}", .{ dependencies_zig_path, err });
	1143
	1144	deps_mod.root_path = try dependencies_zig_path.toString(arena);
	1145	}
	1146
	1147	{
	1148	// Add a CliModule for each package's build.zig.
	1149	const hashes = job_queue.table.keys();
	1150	const fetches = job_queue.table.values();
	1151	try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
	1152	for (hashes, fetches) \|*hash, f\| {
	1153	if (f == &fetch) {
	1154	// The first one is a dummy package for the current project.
	1155	continue;
	1156	}
	1157	if (!f.has_build_zig)
	1158	continue;
	1159	const hash_slice = try arena.dupe(u8, hash.toSlice());
	1160
	1161	const m = try arena.create(CliModule);
	1162	m.* = .{
	1163	.root_path = try f.package_root.toString(arena),
	1164	.name = hash_slice,
	1165	};
	1166	deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m);
	1167	f.cli_module = m;
	1168	}
	1169
	1170	// Each build.zig module needs access to each of its
	1171	// dependencies' build.zig modules by name.
	1172	for (fetches) \|f\| {
	1173	const mod = f.cli_module orelse continue;
	1174	if (!f.have_manifest) continue;
	1175	const man = &f.manifest;
	1176	const dep_names = man.dependencies.keys();
	1177	try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
	1178	for (dep_names, man.dependencies.values()) \|name, dep\| {
	1179	const dep_digest = Package.Fetch.depDigest(
	1180	f.package_root,
	1181	graph.global_cache_root,
	1182	dep,
	1183	) orelse continue;
	1184	const dep_mod = job_queue.table.get(dep_digest).?.cli_module orelse continue;
	1185	const name_cloned = try arena.dupe(u8, name);
	1186	mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
	1187	}
	1188	}
	1189	}
	1190
	1191	// Lower module dependencies to CLI argv.
	1192	build_configurer_argv.shrinkRetainingCapacity(argv_deps_index);
	1193	for (deps_mod.deps.values()) \|dep\| {
	1194	try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1);
	1195	for (dep.deps.keys(), dep.deps.values()) \|name, sub\| {
	1196	build_configurer_argv.appendAssumeCapacity("--dep");
	1197	if (mem.eql(u8, name, sub.name)) {
	1198	build_configurer_argv.appendAssumeCapacity(sub.name);
	1199	} else {
	1200	build_configurer_argv.appendAssumeCapacity(try arena.print("{s}={s}", .{
	1201	name, sub.name,
	1202	}));
	1203	}
	1204	}
	1205	build_configurer_argv.appendAssumeCapacity(try arena.print("-M{s}={s}/{s}", .{
	1206	dep.name, dep.root_path, std.zig.build_zig_basename,
	1207	}));
	1208	}
	1209	try deps_mod.lower(arena, gpa, &build_configurer_argv);
	1210	try build_mod.lower(arena, gpa, &build_configurer_argv);
	1211
	1212	try build_configurer_argv.append(gpa, "--listen=-");
	1213	}
	1214
	1215	const compile_prog_node = options.parent_progress_node.start("Compile Configure Script", 0);
	1216	defer compile_prog_node.end();
	1217
	1218	switch (options.cache_poison) {
	1219	.pure, .disallowed, .ignored => if (try config_man.hit()) {
	1220	const digest = config_man.final();
	1221	break :cp .{
	1222	.{
	1223	.root_dir = graph.local_cache_root,
	1224	.sub_path = try arena.print("c/{s}", .{&digest}),
	1225	},
	1226	false,
	1227	};
	1228	},
	1229	.poisoned => {}, // Don't bother checking for cache hit.
	1230	}
	1231
	1232	const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{
	1233	.argv = build_configurer_argv.items,
	1234	.cache_root = graph.local_cache_root,
	1235	.root_name = configurer_exe_name,
	1236	.environ_map = &graph.environ_map,
	1237	.cache_manifest = &config_man,
	1238	.arch_os_abi = target_arch_os_abi,
	1239	.progress_node = compile_prog_node,
	1240	.skip_log_cmdline_on_compile_errors = !graph.verbose,
	1241	})) \|r\| r.path else \|err\| return err;
	1242	defer gpa.free(configure_exe_path.sub_path);
	1243
	1244	configure_argv[0] = try configure_exe_path.toString(arena);
	1245	}
	1246
	1247	if (!process.can_spawn) {
	1248	fatal("cannot spawn command on {t}: {f}", .{ native_os, @as(std.zig.SubprocessCommand, .{
	1249	.argv = configure_argv,
	1250	}) });
	1251	}
	1252
	1253	const rand_int = randInt(io, u64);
	1254	const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
	1255	const config_tmp_path: Path = .{
	1256	.root_dir = graph.local_cache_root,
	1257	.sub_path = tmp_dir_sub_path,
	1258	};
	1259	const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile(
	1260	io,
	1261	config_tmp_path.sub_path,
	1262	.{ .read = true, .exclusive = true },
	1263	);
	1264	defer config_tmp_file.close(io);
	1265
	1266	const term = term: {
	1267	const child_node = options.parent_progress_node.start("Run Configure Script", 0);
	1268	defer child_node.end();
	1269	var child = process.spawn(io, .{
	1270	.argv = configure_argv,
	1271	.stdout = .{ .file = config_tmp_file },
	1272	.progress_node = child_node,
	1273	}) catch \|err\| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv[0], err });
	1274	defer child.kill(io);
	1275	break :term child.wait(io) catch \|err\|
	1276	fatal("failed to wait configure script {q}: {t}", .{ configure_argv[0], err });
	1277	};
	1278	if (!term.success()) {
	1279	// Failure to produce the configuration file.
	1280	fatal("configure command {f}: {f}", .{ term, @as(std.zig.SubprocessCommand, .{
	1281	.argv = configure_argv,
	1282	}) });
	1283	}
	1284	// Even though the file is designed to be sent directly to make
	1285	// runner, we must load it now because:
	1286	// * If it contains additional file dependencies, we need to
	1287	// add them to `config_man` before obtaining the final digest.
	1288	// * If it contains a set of lazy packages that need to be
	1289	// fetched, we need to fetch those now and re-run configure.
	1290	var configuration = Configuration.loadFile(arena, io, config_tmp_file) catch \|err\|
	1291	fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err });
	1292
	1293	if (configuration.unlazy_deps.len != 0) {
	1294	var any_errors = false;
	1295	for (configuration.unlazy_deps) \|hash_string\| {
	1296	const hash = hash_string.slice(&configuration);
	1297	assert(hash.len != 0);
	1298	if (hash.len > Package.Hash.max_len) {
	1299	log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
	1300	any_errors = true;
	1301	continue;
	1302	}
	1303	try unlazy_set.put(arena, .fromSlice(hash), {});
	1304	}
	1305	if (any_errors) return error.FailedButCacheIntact;
	1306	if (options.system_pkg_dir_path) \|p\| {
	1307	// In this mode, the system needs to provide these packages; they
	1308	// cannot be fetched by Zig.
	1309	const s = Dir.path.sep_str;
	1310	for (unlazy_set.keys()) \|*hash\| {
	1311	log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
	1312	}
	1313	log.info("remote package fetching disabled due to --system mode", .{});
	1314	log.info("dependencies might be avoidable depending on build configuration", .{});
	1315	return error.FailedButCacheIntact;
	1316	}
	1317	continue :cp;
	1318	}
	1319
	1320	for (configuration.path_deps) \|path_dep\| {
	1321	switch (path_dep.flags.mode) {
	1322	.directory => {}, // TODO
	1323	.contents => try config_man.addPathPost(confPathDepToCachePath(graph, &configuration, path_dep)),
	1324	.metadata => {}, // TODO
	1325	}
	1326	}
	1327
	1328	// If it is poisoned, there is no point in moving it to cached
	1329	// location. Just leave it in the tmp directory.
	1330	if (configuration.poisoned) {
	1331	break :cp .{ config_tmp_path, true };
	1332	} else {
	1333	const digest = config_man.final();
	1334	const final_path: Path = .{
	1335	.root_dir = graph.local_cache_root,
	1336	.sub_path = try arena.print("c/{s}", .{&digest}),
	1337	};
	1338	Io.Dir.rename(
	1339	config_tmp_path.root_dir.handle,
	1340	config_tmp_path.sub_path,
	1341	final_path.root_dir.handle,
	1342	final_path.sub_path,
	1343	io,
	1344	) catch \|err\| retry: {
	1345	const e = switch (err) {
	1346	error.FileNotFound => e: {
	1347	const dir_path = final_path.dirname().?;
	1348	dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch \|e\|
	1349	fatal("failed to create directory {f}: {t}", .{ dir_path, e });
	1350	if (Io.Dir.rename(
	1351	config_tmp_path.root_dir.handle,
	1352	config_tmp_path.sub_path,
	1353	final_path.root_dir.handle,
	1354	final_path.sub_path,
	1355	io,
	1356	)) \|_\| break :retry else \|e\| break :e e;
	1357	},
	1358	else => \|e\| e,
	1359	};
	1360	fatal("failed to rename configuration file from {f} into {f}: {t}", .{
	1361	config_tmp_path, final_path, e,
	1362	});
	1363	};
	1364	config_man.writeManifest() catch \|err\| log.warn("failed to write cache manifest: {t}", .{err});
	1365	break :cp .{ final_path, false };
	1366	}
	1367	};
	1368
	1369	// Hang on to the configuration file lock until we finish loading the configuration file.
	1370	var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null;
	1371	defer if (configuration_lock) \|*l\| l.release(io);
	1372
	1373	switch (options.print_configuration) {
	1374	.path => {
	1375	initStdoutWriter(io).print("{f}\n", .{configuration_path}) catch
	1376	fatal("failed printing cache file path: {t}", .{stdout_writer_allocation.err.?});
	1377	stdout_writer_allocation.flush() catch \|err\|
	1378	fatal("failed printing cache file path: {t}", .{err});
	1379	_ = io.lockStderr(&.{}, .no_color) catch {};
	1380	process.exit(0);
575	1381	},
	1382	.none, .zon => {},
	1383	}
576	1384
577		.steps = try arena.alloc(Step, scanned_config.configuration.steps.len),
578		.generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len),
579		.run_args = run_args,
580
581		.available_rss = max_rss,
582		.max_rss_is_default = false,
583		.max_rss_mutex = .init,
584		.skip_oom_steps = skip_oom_steps,
585		.unit_test_timeout_ns = test_timeout_ns,
586
587		.watch = watch,
588		.web_server = undefined, // set after `prepare`
589		.memory_blocked_steps = .empty,
590		.step_stack = .empty,
591		.pkg_config = .{ .debug = debug_pkg_config },
592
593		.error_style = error_style,
594		.multiline_errors = multiline_errors,
595		.summary = summary orelse if (watch or webui_listen != null) .line else .failures,
	1385	const configuration = c: {
	1386	var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch \|err\|
	1387	fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err });
	1388	defer file.close(io);
	1389	break :c Configuration.loadFile(arena, io, file) catch \|err\|
	1390	fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err });
596	1391	};
597		defer {
598		maker.memory_blocked_steps.deinit(gpa);
599		maker.step_stack.deinit(gpa);
	1392	// Technically if the configuration is marked as poisoned, we could
	1393	// already delete the file now, but we leave it around in case the
	1394	// maker process fails or crashes and it's helpful to be able to repeat
	1395	// execution of the command line or otherwise inspect the configuration file.
	1396	const c = &configuration;
	1397	var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty;
	1398	for (configuration.steps, 0..) \|*conf_step, step_index_usize\| {
	1399	if (conf_step.owner != .root) continue;
	1400	const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize);
	1401	const flags = conf_step.flags(c);
	1402	switch (flags.tag) {
	1403	.top_level => {
	1404	const name = step_index.ptr(c).name.slice(c);
	1405	try top_level_steps.put(arena, name, step_index);
	1406	},
	1407	else => {},
	1408	}
	1409	}
	1410	for (c.search_prefixes) \|search_prefix\| {
	1411	try graph.search_prefixes.append(arena, search_prefix.slice(c));
600	1412	}
	1413	return .{
	1414	.configuration = configuration,
	1415	.top_level_steps = top_level_steps,
	1416	.path = configuration_path,
	1417	};
	1418	}
	1419
	1420	fn cmdFetch(gpa: Allocator, graph: *Graph, args: []const []const u8) !void {
	1421	const environ_map = &graph.environ_map;
	1422	const io = graph.io;
	1423	const arena = graph.arena;
601	1424
602		if (maker.available_rss == 0) {
603		maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64);
604		maker.max_rss_is_default = true;
	1425	const color: Color = Color.settingFromEnvironment(environ_map);
	1426	var opt_path_or_url: ?[]const u8 = null;
	1427	var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
	1428	var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
	1429	var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
	1430	var debug_hash: bool = false;
	1431	var save: union(enum) {
	1432	no,
	1433	yes: ?[]const u8,
	1434	exact: ?[]const u8,
	1435	} = .no;
	1436
	1437	var arg_i: usize = 0;
	1438	while (nextArg(args, &arg_i)) \|arg\| {
	1439	if (mem.startsWith(u8, arg, "-")) {
	1440	if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	1441	try Io.File.stdout().writeStreamingAll(io, usage_fetch);
	1442	return process.cleanExit(io);
	1443	} else if (mem.eql(u8, arg, "--global-cache-dir")) {
	1444	override_global_cache_dir = nextArgOrFatal(args, &arg_i);
	1445	} else if (mem.eql(u8, arg, "--cache-dir")) {
	1446	override_local_cache_dir = nextArgOrFatal(args, &arg_i);
	1447	} else if (mem.eql(u8, arg, "--pkg-dir")) {
	1448	override_pkg_dir = nextArgOrFatal(args, &arg_i);
	1449	} else if (mem.eql(u8, arg, "--debug-hash")) {
	1450	debug_hash = true;
	1451	} else if (mem.eql(u8, arg, "--debug-log")) {
	1452	try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i));
	1453	} else if (mem.eql(u8, arg, "--save")) {
	1454	save = .{ .yes = null };
	1455	} else if (mem.cutPrefix(u8, arg, "--save=")) \|rest\| {
	1456	save = .{ .yes = rest };
	1457	} else if (mem.eql(u8, arg, "--save-exact")) {
	1458	save = .{ .exact = null };
	1459	} else if (mem.cutPrefix(u8, arg, "--save-exact=")) \|rest\| {
	1460	save = .{ .exact = rest };
	1461	} else {
	1462	fatal("unrecognized parameter: {q}", .{arg});
	1463	}
	1464	} else if (opt_path_or_url != null) {
	1465	fatal("unexpected extra parameter: {q}", .{arg});
	1466	} else {
	1467	opt_path_or_url = arg;
	1468	}
605	1469	}
606	1470
607		maker.prepare(step_names.items) catch \|err\| switch (err) {
608		error.DependencyLoopDetected, error.InsufficientMemory => {
609		_ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
610		process.exit(1);
	1471	const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
	1472
	1473	var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
	1474	defer http_client.deinit();
	1475
	1476	try http_client.initDefaultProxies(arena, environ_map);
	1477
	1478	var root_prog_node = std.Progress.start(io, .{
	1479	.root_name = "Fetch",
	1480	});
	1481	defer root_prog_node.end();
	1482
	1483	var local_storage: Fetch.LocalStorage = undefined;
	1484	var build_root: BuildRoot = undefined;
	1485	var build_root_initialized = false;
	1486	defer if (build_root_initialized) build_root.deinit(io);
	1487
	1488	const cwd_path = try std.zig.getResolvedCwd(io, arena);
	1489
	1490	const local_storage_ptr = switch (save) {
	1491	.no => null,
	1492	.yes, .exact => ls: {
	1493	build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
	1494	build_root_initialized = true;
	1495
	1496	local_storage = .{
	1497	.cache_root = if (override_local_cache_dir) \|p\| .initCwd(p) else .{
	1498	.root_dir = build_root.directory,
	1499	.sub_path = ".zig-cache",
	1500	},
	1501	.pkg_root = if (override_pkg_dir) \|p\| .initCwd(p) else .{
	1502	.root_dir = build_root.directory,
	1503	.sub_path = "zig-pkg",
	1504	},
	1505	};
	1506
	1507	break :ls &local_storage;
611	1508	},
612		else => \|e\| return e,
613	1509	};
614	1510
615		var w: Watch = w: {
616		if (!watch) break :w undefined;
617		if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag});
618		break :w try .init(&maker);
	1511	var job_queue: Fetch.JobQueue = .{
	1512	.io = io,
	1513	.http_client = &http_client,
	1514	.global_cache = graph.global_cache_root,
	1515	.local_storage = local_storage_ptr,
	1516	.recursive = false,
	1517	.read_only = false,
	1518	.debug_hash = debug_hash,
	1519	.mode = .all,
	1520	.prog_node = root_prog_node,
	1521	};
	1522	defer job_queue.deinit();
	1523
	1524	var fetch: Fetch = .{
	1525	.arena = std.heap.ArenaAllocator.init(gpa),
	1526	.location = .{ .path_or_url = path_or_url },
	1527	.location_tok = 0,
	1528	.hash_tok = .none,
	1529	.name_tok = 0,
	1530	.lazy_status = .eager,
	1531	.remote_package_root = undefined,
	1532	.parent_package_root = undefined,
	1533	.parent_manifest_ast = null,
	1534	.prog_node = root_prog_node,
	1535	.job_queue = &job_queue,
	1536	.omit_missing_hash_error = true,
	1537	.allow_missing_paths_field = false,
	1538	.use_latest_commit = true,
	1539
	1540	.package_root = undefined,
	1541	.error_bundle = undefined,
	1542	.manifest = undefined,
	1543	.manifest_ast = undefined,
	1544	.have_manifest = false,
	1545	.computed_hash = undefined,
	1546	.has_build_zig = false,
	1547	.oom_flag = false,
	1548	.latest_commit = null,
	1549
	1550	.cli_module = null,
619	1551	};
	1552	defer fetch.deinit();
620	1553
621		const now = Io.Clock.Timestamp.now(io, .awake);
	1554	fetch.run() catch \|err\| switch (err) {
	1555	error.OutOfMemory, error.Canceled => \|e\| return e,
	1556	error.FetchFailed => {}, // error bundle checked below
	1557	};
622	1558
623		maker.web_server = if (webui_listen) \|listen_address\| ws: {
624		if (builtin.single_threaded) unreachable; // `fatal` above
625		break :ws .init(.{
626		.maker = &maker,
627		.root_prog_node = main_progress_node,
628		.listen_address = listen_address,
629		.base_timestamp = now,
630		});
631		} else null;
	1559	try job_queue.group.await(io);
632	1560
633		if (maker.web_server) \|*ws\| {
634		ws.start() catch \|err\| fatal("failed to start web server: {t}", .{err});
	1561	if (fetch.error_bundle.root_list.items.len > 0) {
	1562	var errors = try fetch.error_bundle.toOwnedBundle("");
	1563	errors.renderToStderr(io, .{}, color) catch {};
	1564	process.exit(1);
635	1565	}
636	1566
637		rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) {
638		const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
639		defer io.unlockStderr();
640		stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch \|err\| switch (err) {
641		error.WriteFailed => return stderr.file_writer.err.?,
642		};
643		}) {
644		if (maker.web_server) \|*ws\| ws.startBuild();
	1567	const package_hash = fetch.computedPackageHash();
	1568	const package_hash_slice = package_hash.toSlice();
	1569
	1570	root_prog_node.end();
	1571	root_prog_node = .{ .index = .none };
	1572
	1573	const name = switch (save) {
	1574	.no => {
	1575	var data: [2][]const u8 = .{ package_hash_slice, "\n" };
	1576	const w = initStdoutWriter(io);
	1577	w.writeVecAll(&data) catch return stdout_writer_allocation.err.?;
	1578	try stdout_writer_allocation.flush();
	1579	return process.cleanExit(io);
	1580	},
	1581	.yes, .exact => \|name\| name: {
	1582	if (name) \|n\| break :name n;
	1583	if (!fetch.have_manifest)
	1584	fatal("unable to determine name; fetched package has no build.zig.zon file", .{});
	1585	break :name fetch.manifest.name;
	1586	},
	1587	};
	1588
	1589	// The name to use in case the manifest file needs to be created now.
	1590	const init_root_name = Dir.path.basename(build_root.directory.path orelse cwd_path);
	1591	var manifest, var ast = try loadManifest(gpa, arena, io, .{
	1592	.root_name = try sanitizeExampleName(arena, init_root_name),
	1593	.dir = build_root.directory.handle,
	1594	.color = color,
	1595	});
	1596	defer {
	1597	manifest.deinit(gpa);
	1598	ast.deinit(gpa);
	1599	}
	1600
	1601	var fixups: std.zig.Ast.Render.Fixups = .{};
	1602	defer fixups.deinit(gpa);
	1603
	1604	var saved_path_or_url = path_or_url;
645	1605
646		try maker.makeStepNames(step_names.items, main_progress_node, fuzz);
	1606	if (fetch.latest_commit) \|latest_commit\| resolved: {
	1607	const latest_commit_hex = try arena.print("{f}", .{latest_commit});
647	1608
648		if (maker.web_server) \|*web_server\| {
649		if (fuzz) \|mode\| if (mode != .forever) fatal(
650		"error: limited fuzzing is not implemented yet for --webui",
651		.{},
652		);
	1609	var uri = try std.Uri.parse(path_or_url);
653	1610
654		web_server.finishBuild(.{ .fuzz = fuzz != null });
	1611	if (uri.fragment) \|fragment\| {
	1612	const target_ref = try fragment.toRawMaybeAlloc(arena);
	1613
	1614	// the refspec may already be fully resolved
	1615	if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved;
	1616
	1617	log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex });
	1618
	1619	// include the original refspec in a query parameter, could be used to check for updates
	1620	uri.query = .{ .percent_encoded = try arena.print("ref={f}", .{
	1621	std.fmt.alt(fragment, .formatEscaped),
	1622	}) };
	1623	} else {
	1624	log.info("resolved to commit {s}", .{latest_commit_hex});
655	1625	}
656	1626
657		if (maker.web_server) \|*web_server\| {
658		const c = &scanned_config.configuration;
659		assert(!watch); // fatal error after CLI parsing
660		while (true) switch (try web_server.wait()) {
661		.rebuild => {
662		for (maker.step_stack.keys()) \|step_index\| {
663		const step = maker.stepByIndex(step_index);
664		step.state = .precheck_done;
665		const deps = step_index.ptr(c).deps.slice(c);
666		step.pending_deps = @intCast(deps.len);
667		step.reset(&maker);
	1627	// replace the refspec with the resolved commit SHA
	1628	uri.fragment = .{ .raw = latest_commit_hex };
	1629
	1630	switch (save) {
	1631	.yes => saved_path_or_url = try arena.print("{f}", .{uri}),
	1632	.no, .exact => {}, // keep the original URL
	1633	}
	1634	}
	1635
	1636	const new_node_init = try arena.print(
	1637	\\.{{
	1638	\\ .url = "{f}",
	1639	\\ .hash = "{f}",
	1640	\\ }}
	1641	, .{
	1642	std.zig.fmtString(saved_path_or_url),
	1643	std.zig.fmtString(package_hash_slice),
	1644	});
	1645
	1646	const new_node_text = try arena.print(".{f} = {s},\n", .{
	1647	std.zig.fmtIdPU(name), new_node_init,
	1648	});
	1649
	1650	const dependencies_init = try arena.print(".{{\n {s} }}", .{
	1651	new_node_text,
	1652	});
	1653
	1654	const dependencies_text = try arena.print(".dependencies = {s},\n", .{
	1655	dependencies_init,
	1656	});
	1657
	1658	if (manifest.dependencies.get(name)) \|dep\| {
	1659	if (dep.hash) \|h\| {
	1660	switch (dep.location) {
	1661	.url => \|u\| {
	1662	if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
	1663	log.info("existing dependency named {q} is up-to-date", .{name});
	1664	process.exit(0);
668	1665	}
669		continue :rebuild;
	1666	},
	1667	.path => {},
	1668	}
	1669	}
	1670
	1671	const location_replace = try arena.print("{q}", .{saved_path_or_url});
	1672	const hash_replace = try arena.print("{q}", .{package_hash_slice});
	1673
	1674	log.warn("overwriting existing dependency named {q}", .{name});
	1675	try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace);
	1676	if (dep.hash_node.unwrap()) \|hash_node\| {
	1677	try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace);
	1678	} else {
	1679	// https://github.com/ziglang/zig/issues/21690
	1680	}
	1681	} else if (manifest.dependencies.count() > 0) {
	1682	// Add fixup for adding another dependency.
	1683	const deps = manifest.dependencies.values();
	1684	const last_dep_node = deps[deps.len - 1].node;
	1685	try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text);
	1686	} else if (manifest.dependencies_node.unwrap()) \|dependencies_node\| {
	1687	// Add fixup for replacing the entire dependencies struct.
	1688	try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init);
	1689	} else {
	1690	// Add fixup for adding dependencies struct.
	1691	try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text);
	1692	}
	1693
	1694	var aw: Io.Writer.Allocating = .init(gpa);
	1695	defer aw.deinit();
	1696	try ast.render(gpa, &aw.writer, fixups);
	1697	const rendered = aw.written();
	1698
	1699	build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch \|err\| {
	1700	fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err });
	1701	};
	1702
	1703	return process.cleanExit(io);
	1704	}
	1705
	1706	const usage_fetch =
	1707	\\Usage: zig fetch [options] <url>
	1708	\\Usage: zig fetch [options] <path>
	1709	\\
	1710	\\ Copy a package into the global cache and print its hash.
	1711	\\ <url> must point to one of the following:
	1712	\\ - A git+http / git+https server for the package
	1713	\\ - A tarball file (with or without compression) containing
	1714	\\ package source
	1715	\\ - A git bundle file containing package source
	1716	\\
	1717	\\Examples:
	1718	\\
	1719	\\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git
	1720	\\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz
	1721	\\
	1722	\\Options:
	1723	\\ -h, --help Print this help and exit
	1724	\\ --global-cache-dir [path] Override path to global Zig cache directory
	1725	\\ --cache-dir [path] Override path to local cache directory
	1726	\\ --pkg-dir [path] Override path to local package directory
	1727	\\ --debug-hash Print verbose hash information to stdout
	1728	\\ --debug-log [scope] Enable printing debug/info log messages for scope
	1729	\\ --save Add the fetched package to build.zig.zon
	1730	\\ --save=[name] Add the fetched package to build.zig.zon as name
	1731	\\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim
	1732	\\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim
	1733	\\
	1734	;
	1735
	1736	const usage_init =
	1737	\\Usage: zig init
	1738	\\
	1739	\\ Initializes a `zig build` project in the current working
	1740	\\ directory.
	1741	\\
	1742	\\Options:
	1743	\\ -m, --minimal Use minimal init template
	1744	\\ -h, --help Print this help and exit
	1745	\\
	1746	\\
	1747	;
	1748
	1749	const usage_libc =
	1750	\\Usage: zig libc
	1751	\\
	1752	\\ Detect the native libc installation and print the resulting
	1753	\\ paths to stdout. You can save this into a file and then edit
	1754	\\ the paths to create a cross compilation libc kit. Then you
	1755	\\ can pass `--libc [file]` for Zig to use it.
	1756	\\
	1757	\\Usage: zig libc [paths_file]
	1758	\\
	1759	\\ Parse a libc installation text file and validate it.
	1760	\\
	1761	\\Options:
	1762	\\ -h, --help Print this help and exit
	1763	\\ -target [name] <arch><sub>-<os>-<abi> see the targets command
	1764	\\ -includes Print the libc include directories for the target
	1765	\\
	1766	;
	1767
	1768	fn cmdInit(gpa: Allocator, graph: *Graph, args: []const []const u8) !void {
	1769	const arena = graph.arena;
	1770	const io = graph.io;
	1771	const default_build_zig_basename = std.zig.build_zig_basename;
	1772
	1773	var template: enum { example, minimal } = .example;
	1774	{
	1775	var i: usize = 0;
	1776	while (i < args.len) : (i += 1) {
	1777	const arg = args[i];
	1778	if (mem.startsWith(u8, arg, "-")) {
	1779	if (mem.eql(u8, arg, "-m") or mem.eql(u8, arg, "--minimal")) {
	1780	template = .minimal;
	1781	} else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	1782	try Io.File.stdout().writeStreamingAll(io, usage_init);
	1783	return process.cleanExit(io);
	1784	} else {
	1785	fatal("unrecognized parameter: {q}", .{arg});
	1786	}
	1787	} else {
	1788	fatal("unexpected extra parameter: {q}", .{arg});
	1789	}
	1790	}
	1791	}
	1792
	1793	const cwd_path = try std.zig.getResolvedCwd(io, arena);
	1794	const cwd_basename = Dir.path.basename(cwd_path);
	1795	const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename);
	1796
	1797	const rng: std.Random.IoSource = .{ .io = io };
	1798	const fingerprint: Package.Fingerprint = .generate(rng.interface(), sanitized_root_name);
	1799
	1800	switch (template) {
	1801	.example => {
	1802	var templates = Templates.find(gpa, io, graph.zig_lib_directory);
	1803	defer templates.deinit(io);
	1804
	1805	const s = Dir.path.sep_str;
	1806	const template_paths = [_][]const u8{
	1807	default_build_zig_basename,
	1808	Package.Manifest.basename,
	1809	"src" ++ s ++ "main.zig",
	1810	"src" ++ s ++ "root.zig",
	1811	};
	1812	var ok_count: usize = 0;
	1813
	1814	for (template_paths) \|template_path\| {
	1815	if (templates.write(arena, io, Io.Dir.cwd(), sanitized_root_name, template_path, fingerprint)) \|_\| {
	1816	log.info("created {s}", .{template_path});
	1817	ok_count += 1;
	1818	} else \|err\| switch (err) {
	1819	error.PathAlreadyExists => log.info("preserving already existing file: {s}", .{
	1820	template_path,
	1821	}),
	1822	else => log.err("unable to write {s}: {t}", .{ template_path, err }),
	1823	}
	1824	}
	1825
	1826	if (ok_count == template_paths.len) {
	1827	log.info("see `zig build --help` for a menu of options", .{});
	1828	}
	1829	return process.cleanExit(io);
	1830	},
	1831	.minimal => {
	1832	Templates.writeSimpleFile(io, Package.Manifest.basename,
	1833	\\.{{
	1834	\\ .name = .{s},
	1835	\\ .version = "0.0.1",
	1836	\\ .minimum_zig_version = "{s}",
	1837	\\ .paths = .{{""}},
	1838	\\ .fingerprint = 0x{x},
	1839	\\}}
	1840	\\
	1841	, .{
	1842	sanitized_root_name,
	1843	builtin.zig_version_string,
	1844	fingerprint.int(),
	1845	}) catch \|err\| switch (err) {
	1846	else => fatal("failed to create {q}: {t}", .{ Package.Manifest.basename, err }),
	1847	error.PathAlreadyExists => fatal("refusing to overwrite {q}", .{Package.Manifest.basename}),
	1848	};
	1849	Templates.writeSimpleFile(io, default_build_zig_basename,
	1850	\\const std = @import("std");
	1851	\\
	1852	\\pub fn build(b: *std.Build) void {{
	1853	\\ _ = b; // stub
	1854	\\}}
	1855	\\
	1856	, .{}) catch \|err\| switch (err) {
	1857	else => fatal("failed to create {q}: {t}", .{ default_build_zig_basename, err }),
	1858	// `build.zig` already existing is okay: the user has just used `zig init` to set up
	1859	// their `build.zig.zon` after writing their `build.zig`. So this one isn't fatal.
	1860	error.PathAlreadyExists => {
	1861	log.info("successfully populated {q}, preserving existing {q}", .{
	1862	Package.Manifest.basename, default_build_zig_basename,
	1863	});
	1864	return process.cleanExit(io);
670	1865	},
671	1866	};
	1867	log.info("successfully populated {q} and {q}", .{ Package.Manifest.basename, default_build_zig_basename });
	1868	return process.cleanExit(io);
	1869	},
	1870	}
	1871	}
	1872
	1873	fn cmdLibC(gpa: Allocator, graph: *Graph, args: []const []const u8) !void {
	1874	const environ_map = &graph.environ_map;
	1875	const io = graph.io;
	1876	const arena = graph.arena;
	1877	const LibCInstallation = std.zig.LibCInstallation;
	1878
	1879	var input_file: ?[]const u8 = null;
	1880	var target_arch_os_abi: []const u8 = "native";
	1881	var print_includes: bool = false;
	1882	const stdout = initStdoutWriter(io);
	1883	{
	1884	var i: usize = 0;
	1885	while (i < args.len) : (i += 1) {
	1886	const arg = args[i];
	1887	if (mem.startsWith(u8, arg, "-")) {
	1888	if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	1889	try stdout.writeAll(usage_libc);
	1890	try stdout.flush();
	1891	return std.process.cleanExit(io);
	1892	} else if (mem.eql(u8, arg, "-target")) {
	1893	if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
	1894	i += 1;
	1895	target_arch_os_abi = args[i];
	1896	} else if (mem.eql(u8, arg, "-includes")) {
	1897	print_includes = true;
	1898	} else {
	1899	fatal("unrecognized parameter: '{s}'", .{arg});
	1900	}
	1901	} else if (input_file != null) {
	1902	fatal("unexpected extra parameter: '{s}'", .{arg});
	1903	} else {
	1904	input_file = arg;
	1905	}
672	1906	}
	1907	}
673	1908
674		if (!maker.watch) return;
675
676		// Comptime-known guard to prevent including the logic below when `!Watch.have_impl`.
677		if (!Watch.have_impl) unreachable;
678
679		try w.update(maker.step_stack.keys());
680
681		// Wait until a file system notification arrives. Read all such events
682		// until the buffer is empty. Then wait for a debounce interval, resetting
683		// if any more events come in. After the debounce interval has passed,
684		// trigger a rebuild on all steps with modified inputs, as well as their
685		// recursive dependants.
686		var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
687		const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
688		w.dir_count, countSubProcesses(&maker),
689		}) catch &caption_buf;
690		var debouncing_node = main_progress_node.start(caption, 0);
691		var in_debounce = false;
692		while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) {
693		.timeout => {
694		assert(in_debounce);
695		debouncing_node.end();
696		markFailedStepsDirty(&maker);
697		continue :rebuild;
698		},
699		.dirty => if (!in_debounce) {
700		in_debounce = true;
701		debouncing_node.end();
702		debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0);
703		},
704		.clean => {},
	1909	const target_query = std.zig.parseTargetQueryOrReportFatalError(gpa, .{
	1910	.arch_os_abi = target_arch_os_abi,
	1911	});
	1912	const target = std.zig.resolveTargetQueryOrFatal(io, target_query);
	1913
	1914	if (print_includes) {
	1915	const libc_installation: ?*LibCInstallation = libc: {
	1916	if (input_file) \|libc_file\| {
	1917	const libc = try arena.create(LibCInstallation);
	1918	libc.* = LibCInstallation.parse(arena, io, libc_file, &target) catch \|err\| {
	1919	fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err });
	1920	};
	1921	break :libc libc;
	1922	} else {
	1923	break :libc null;
	1924	}
705	1925	};
	1926
	1927	const is_native_abi = target_query.isNativeAbi();
	1928
	1929	const libc_dirs = std.zig.LibCDirs.detect(
	1930	arena,
	1931	io,
	1932	.{ .root_dir = graph.zig_lib_directory },
	1933	&target,
	1934	is_native_abi,
	1935	true,
	1936	libc_installation,
	1937	environ_map,
	1938	) catch \|err\| {
	1939	const zig_target = try target.zigTriple(arena);
	1940	fatal("unable to detect libc for target {s}: {t}", .{ zig_target, err });
	1941	};
	1942
	1943	if (libc_dirs.libc_include_dir_list.len == 0) {
	1944	const zig_target = try target.zigTriple(arena);
	1945	fatal("no include dirs detected for target {s}", .{zig_target});
	1946	}
	1947
	1948	for (libc_dirs.libc_include_dir_list) \|include_dir\| {
	1949	try stdout.writeAll(include_dir);
	1950	try stdout.writeByte('\n');
	1951	}
	1952	try stdout.flush();
	1953	return std.process.cleanExit(io);
	1954	}
	1955
	1956	if (input_file) \|libc_file\| {
	1957	var libc = LibCInstallation.parse(gpa, io, libc_file, &target) catch \|err\| {
	1958	fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err });
	1959	};
	1960	defer libc.deinit(gpa);
	1961	} else {
	1962	if (!target_query.canDetectLibC()) {
	1963	fatal("unable to detect libc for non-native target", .{});
	1964	}
	1965	var libc = LibCInstallation.findNative(gpa, io, .{
	1966	.verbose = true,
	1967	.target = &target,
	1968	.environ_map = environ_map,
	1969	}) catch \|err\| {
	1970	fatal("unable to detect native libc: {t}", .{err});
	1971	};
	1972	defer libc.deinit(gpa);
	1973
	1974	try libc.render(stdout);
	1975	try stdout.flush();
706	1976	}
707	1977	}
708	1978
...	...	@@ -807,9 +2077,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void {
807	2077	}
808	2078	if (any_problems) {
809	2079	if (maker.max_rss_is_default) {
810		std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{
811		max_needed,
812		});
	2080	log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed});
813	2081	}
814	2082	return error.InsufficientMemory;
815	2083	}
...	...	@@ -819,7 +2087,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void {
819	2087	fn makeStepNames(
820	2088	maker: *Maker,
821	2089	step_names: []const []const u8,
822		parent_prog_node: std.Progress.Node,
	2090	parent_progress_node: std.Progress.Node,
823	2091	fuzz: ?Fuzz.Mode,
824	2092	) !void {
825	2093	const graph = maker.graph;
...	...	@@ -843,7 +2111,7 @@ fn makeStepNames(
843	2111	}
844	2112	}
845	2113
846		const step_prog = parent_prog_node.start("steps", step_stack.count());
	2114	const step_prog = parent_progress_node.start("steps", step_stack.count());
847	2115	defer step_prog.end();
848	2116
849	2117	var group: Io.Group = .init;
...	...	@@ -901,12 +2169,12 @@ fn makeStepNames(
901	2169	}
902	2170
903	2171	if (fuzz) \|mode\| blk: {
904		switch (builtin.os.tag) {
	2172	switch (native_os) {
905	2173	// Current implementation depends on two things that need to be ported to Windows:
906	2174	// * Memory-mapping to share data between the fuzzer and build runner.
907	2175	// * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
908	2176	// many addresses to source locations).
909		.windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}),
	2177	.windows => fatal("--fuzz not yet implemented for {t}", .{native_os}),
910	2178	else => {},
911	2179	}
912	2180	if (@bitSizeOf(usize) != 64) {
...	...	@@ -923,7 +2191,7 @@ fn makeStepNames(
923	2191	}
924	2192
925	2193	assert(mode == .limit);
926		var f = Fuzz.init(maker, step_stack.keys(), parent_prog_node, mode) catch \|err\|
	2194	var f = Fuzz.init(maker, step_stack.keys(), parent_progress_node, mode) catch \|err\|
927	2195	fatal("failed to start fuzzer: {t}", .{err});
928	2196	defer f.deinit();
929	2197
...	...	@@ -997,7 +2265,7 @@ fn makeStepNames(
997	2265	t.setColor(.reset) catch {};
998	2266	}
999	2267
1000		w.writeAll("\n") catch {};
	2268	w.writeByte('\n') catch {};
1001	2269
1002	2270	if (maker.summary == .line) break :summary;
1003	2271
...	...	@@ -1114,7 +2382,7 @@ fn makeStep(
1114	2382	const step_prog_node = root_prog_node.start(step_name, 0);
1115	2383	defer step_prog_node.end();
1116	2384
1117		if (maker.web_server) \|*ws\| ws.updateStepStatus(step_index, .wip);
	2385	if (maker.web_server) \|ws\| ws.updateStepStatus(step_index, .wip);
1118	2386
1119	2387	const new_state: Step.State = for (deps) \|dep_index\| {
1120	2388	const dep_make_step = maker.stepByIndex(dep_index);
...	...	@@ -1149,14 +2417,14 @@ fn makeStep(
1149	2417	.dependency_failure,
1150	2418	.skipped_oom,
1151	2419	=> {
1152		if (maker.web_server) \|*ws\| ws.updateStepStatus(step_index, .failure);
	2420	if (maker.web_server) \|ws\| ws.updateStepStatus(step_index, .failure);
1153	2421	std.Progress.setStatus(.failure_working);
1154	2422	},
1155	2423
1156	2424	.success,
1157	2425	.skipped,
1158	2426	=> {
1159		if (maker.web_server) \|*ws\| ws.updateStepStatus(step_index, .success);
	2427	if (maker.web_server) \|ws\| ws.updateStepStatus(step_index, .success);
1160	2428	},
1161	2429	}
1162	2430	}
...	...	@@ -1670,26 +2938,23 @@ pub fn printErrorMessages(
1670	2938	try writer.writeByte('\n');
1671	2939	}
1672	2940
1673		fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 {
1674		if (idx.* >= args.len) return null;
1675		defer idx.* += 1;
1676		return args[idx.*];
	2941	fn nextArg(args: []const []const u8, i: *usize) ?[]const u8 {
	2942	if (i.* >= args.len) return null;
	2943	defer i.* += 1;
	2944	return args[i.*];
1677	2945	}
1678	2946
1679		fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 {
1680		return nextArg(args, idx) orelse {
1681		fatalWithHint("expected argument after {q}", .{args[idx.* - 1]});
1682		};
	2947	fn nextArgOrFatal(args: []const []const u8, i: *usize) []const u8 {
	2948	return nextArg(args, i) orelse fatalWithHint("expected another argument after {q}", .{args[i.* - 1]});
1683	2949	}
1684	2950
1685		fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 {
1686		const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first});
1687		if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg });
1688		const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first});
1689		return arg;
	2951	fn prefixedArgOrFatal(args: []const []const u8, i: *usize, prefix: []const u8) []const u8 {
	2952	const arg = nextArgOrFatal(args, i);
	2953	if (mem.cutPrefix(u8, arg, prefix)) \|rest\| return rest;
	2954	fatal("expected {q} to instead begin with {q}", .{ arg, prefix });
1690	2955	}
1691	2956
1692		fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {
	2957	fn argsRest(args: []const []const u8, idx: usize) ?[]const []const u8 {
1693	2958	if (idx >= args.len) return null;
1694	2959	return args[idx..];
1695	2960	}
...	...	@@ -2006,11 +3271,11 @@ pub fn installSymLinks(
2006	3271	const name = conf_comp.root_name.slice(c);
2007	3272
2008	3273	const filename_major_only, const filename_name_only = if (os_tag.isDarwin()) .{
2009		try std.fmt.allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }),
2010		try std.fmt.allocPrint(arena, "lib{s}.dylib", .{name}),
	3274	try arena.print("lib{s}.{d}.dylib", .{ name, version.major }),
	3275	try arena.print("lib{s}.dylib", .{name}),
2011	3276	} else .{
2012		try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }),
2013		try std.fmt.allocPrint(arena, "lib{s}.so", .{name}),
	3277	try arena.print("lib{s}.so.{d}", .{ name, version.major }),
	3278	try arena.print("lib{s}.so", .{name}),
2014	3279	};
2015	3280
2016	3281	return installSymLinksInner(maker, arena, output_path, asking_step_index, filename_major_only, filename_name_only);
...	...	@@ -2050,8 +3315,8 @@ fn removePoisonedConfiguration(io: Io, scanned_config: *const ScannedConfig) voi
2050	3315	if (scanned_config.configuration.poisoned) {
2051	3316	// This configuration file was good for only 1 invocation of the maker
2052	3317	// process. Delete it to save space on disk.
2053		Io.Dir.cwd().deleteFile(io, scanned_config.path) catch \|err\|
2054		log.warn("failed deleting poisoned configuration file {s}: {t}", .{ scanned_config.path, err });
	3318	scanned_config.path.root_dir.handle.deleteFile(io, scanned_config.path.sub_path) catch \|err\|
	3319	log.warn("failed deleting poisoned configuration file {f}: {t}", .{ scanned_config.path, err });
2055	3320	}
2056	3321	}
2057	3322
...	...	@@ -2059,3 +3324,385 @@ inline fn debugMakerLeaks() bool {
2059	3324	if (!is_debug_mode) return false;
2060	3325	return debug_maker_leaks;
2061	3326	}
	3327
	3328	const BuildRoot = struct {
	3329	directory: Cache.Directory,
	3330	build_zig_basename: []const u8,
	3331	cleanup_build_dir: ?Io.Dir,
	3332
	3333	fn deinit(br: *BuildRoot, io: Io) void {
	3334	if (br.cleanup_build_dir) \|*dir\| dir.close(io);
	3335	br.* = undefined;
	3336	}
	3337	};
	3338
	3339	const FindBuildRootOptions = struct {
	3340	build_file: ?[]const u8 = null,
	3341	cwd_path: ?[]const u8 = null,
	3342	};
	3343
	3344	fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot {
	3345	const cwd_path = options.cwd_path orelse try std.zig.getResolvedCwd(io, arena);
	3346	const build_zig_basename = if (options.build_file) \|bf\|
	3347	Dir.path.basename(bf)
	3348	else
	3349	std.zig.build_zig_basename;
	3350
	3351	if (options.build_file) \|bf\| {
	3352	if (Dir.path.dirname(bf)) \|dirname\| {
	3353	const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
	3354	fatal("failed opening directory containing {q}: {t}", .{ bf, err });
	3355	};
	3356	return .{
	3357	.build_zig_basename = build_zig_basename,
	3358	.directory = .{ .path = dirname, .handle = dir },
	3359	.cleanup_build_dir = dir,
	3360	};
	3361	}
	3362
	3363	return .{
	3364	.build_zig_basename = build_zig_basename,
	3365	.directory = .{ .path = null, .handle = Io.Dir.cwd() },
	3366	.cleanup_build_dir = null,
	3367	};
	3368	}
	3369	// Search up parent directories until we find build.zig.
	3370	var dirname: []const u8 = cwd_path;
	3371	while (true) {
	3372	const joined_path = try Dir.path.join(arena, &[_][]const u8{ dirname, build_zig_basename });
	3373	if (Io.Dir.cwd().access(io, joined_path, .{})) \|_\| {
	3374	const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
	3375	fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err });
	3376	};
	3377	return .{
	3378	.build_zig_basename = build_zig_basename,
	3379	.directory = .{
	3380	.path = dirname,
	3381	.handle = dir,
	3382	},
	3383	.cleanup_build_dir = dir,
	3384	};
	3385	} else \|err\| switch (err) {
	3386	error.FileNotFound => {
	3387	dirname = Dir.path.dirname(dirname) orelse {
	3388	log.info("initialize {s} template file with \"zig init\"", .{std.zig.build_zig_basename});
	3389	log.info("see \"zig --help\" for more options", .{});
	3390	fatal("no build.zig file found, in the current directory or any parent directories", .{});
	3391	};
	3392	continue;
	3393	},
	3394	else => \|e\| return e,
	3395	}
	3396	}
	3397	}
	3398
	3399	const Fork = struct {
	3400	path: Path,
	3401	manifest_ast: std.zig.Ast,
	3402	manifest: Package.Manifest,
	3403	error_bundle: std.zig.ErrorBundle.Wip,
	3404	failed: bool,
	3405	arena_allocator: std.heap.ArenaAllocator,
	3406
	3407	fn init(cwd_relative_path: []const u8) Fork {
	3408	return .{
	3409	.manifest_ast = undefined,
	3410	.manifest = undefined,
	3411	.error_bundle = undefined,
	3412	.arena_allocator = undefined,
	3413	.path = .{
	3414	.root_dir = .cwd(),
	3415	.sub_path = cwd_relative_path,
	3416	},
	3417	.failed = false,
	3418	};
	3419	}
	3420
	3421	fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void {
	3422	loadFallible(io, gpa, fork, color) catch \|err\| switch (err) {
	3423	error.Canceled => \|e\| return e,
	3424	error.AlreadyReported => fork.failed = true,
	3425	else => \|e\| {
	3426	log.err("failed to load fork at {f}: {t}", .{ fork.path, e });
	3427	fork.failed = true;
	3428	},
	3429	};
	3430	}
	3431
	3432	fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void {
	3433	fork.arena_allocator = .init(gpa);
	3434	const arena = fork.arena_allocator.allocator();
	3435
	3436	var error_bundle: std.zig.ErrorBundle.Wip = undefined;
	3437	try error_bundle.init(gpa);
	3438	defer error_bundle.deinit();
	3439
	3440	const manifest_path = try fork.path.join(arena, Package.Manifest.basename);
	3441
	3442	Package.Manifest.load(
	3443	io,
	3444	arena,
	3445	manifest_path,
	3446	&fork.manifest_ast,
	3447	&error_bundle,
	3448	&fork.manifest,
	3449	true,
	3450	) catch \|err\| switch (err) {
	3451	error.Canceled => \|e\| return e,
	3452	error.ErrorsBundled => {
	3453	assert(error_bundle.root_list.items.len > 0);
	3454	var errors = try error_bundle.toOwnedBundle("");
	3455	errors.renderToStderr(io, .{}, color) catch {};
	3456	return error.AlreadyReported;
	3457	},
	3458	else => \|e\| {
	3459	log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e });
	3460	return error.AlreadyReported;
	3461	},
	3462	};
	3463	}
	3464
	3465	fn deinitList(forks: []Fork) void {
	3466	for (forks) \|*fork\| fork.arena_allocator.deinit();
	3467	}
	3468	};
	3469
	3470	fn parseRandomSeed(arg: []const u8) u32 {
	3471	return std.fmt.parseUnsigned(u32, arg, 0) catch \|err\|
	3472	fatal("failed parsing random seed {q} as unsigned 32-bit integer: {t}", .{ arg, err });
	3473	}
	3474
	3475	fn randInt(io: Io, comptime T: type) T {
	3476	var x: T = undefined;
	3477	io.random(@ptrCast(&x));
	3478	return x;
	3479	}
	3480
	3481	const LoadManifestOptions = struct {
	3482	root_name: []const u8,
	3483	dir: Io.Dir,
	3484	color: Color,
	3485	};
	3486
	3487	fn loadManifest(
	3488	gpa: Allocator,
	3489	arena: Allocator,
	3490	io: Io,
	3491	options: LoadManifestOptions,
	3492	) !struct { Package.Manifest, std.zig.Ast } {
	3493	const rng: std.Random.IoSource = .{ .io = io };
	3494
	3495	const manifest_bytes = while (true) {
	3496	break options.dir.readFileAllocOptions(
	3497	io,
	3498	Package.Manifest.basename,
	3499	arena,
	3500	.limited(Package.Manifest.max_bytes),
	3501	.@"1",
	3502	0,
	3503	) catch \|err\| switch (err) {
	3504	error.FileNotFound => {
	3505	Templates.writeSimpleFile(io, Package.Manifest.basename,
	3506	\\.{{
	3507	\\ .name = .{s},
	3508	\\ .version = "{s}",
	3509	\\ .paths = .{{""}},
	3510	\\ .fingerprint = 0x{x},
	3511	\\}}
	3512	\\
	3513	, .{
	3514	options.root_name,
	3515	builtin.zig_version_string,
	3516	Package.Fingerprint.generate(rng.interface(), options.root_name).int(),
	3517	}) catch \|e\| {
	3518	fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e });
	3519	};
	3520	continue;
	3521	},
	3522	else => \|e\| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }),
	3523	};
	3524	};
	3525	var ast = try std.zig.Ast.parse(gpa, manifest_bytes, .zon);
	3526	errdefer ast.deinit(gpa);
	3527
	3528	if (ast.errors.len > 0) {
	3529	try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color);
	3530	process.exit(2);
	3531	}
	3532
	3533	var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{});
	3534	errdefer manifest.deinit(gpa);
	3535
	3536	if (manifest.errors.len > 0) {
	3537	var wip_errors: std.zig.ErrorBundle.Wip = undefined;
	3538	try wip_errors.init(gpa);
	3539	defer wip_errors.deinit();
	3540
	3541	const src_path = try wip_errors.addString(Package.Manifest.basename);
	3542	try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors);
	3543
	3544	var error_bundle = try wip_errors.toOwnedBundle("");
	3545	defer error_bundle.deinit(gpa);
	3546	error_bundle.renderToStderr(io, .{}, options.color) catch {};
	3547
	3548	process.exit(2);
	3549	}
	3550	return .{ manifest, ast };
	3551	}
	3552
	3553	fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 {
	3554	var result: std.ArrayList(u8) = .empty;
	3555	for (bytes, 0..) \|byte, i\| switch (byte) {
	3556	'0'...'9' => {
	3557	if (i == 0) try result.append(arena, '_');
	3558	try result.append(arena, byte);
	3559	},
	3560	'_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte),
	3561	'-', '.', ' ' => try result.append(arena, '_'),
	3562	else => continue,
	3563	};
	3564	if (!std.zig.isValidId(result.items)) return "foo";
	3565	if (result.items.len > Package.Manifest.max_name_len)
	3566	result.shrinkRetainingCapacity(Package.Manifest.max_name_len);
	3567
	3568	return result.toOwnedSlice(arena);
	3569	}
	3570
	3571	test sanitizeExampleName {
	3572	var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator);
	3573	defer arena_instance.deinit();
	3574	const arena = arena_instance.allocator();
	3575
	3576	try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+"));
	3577	try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, ""));
	3578	try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!"));
	3579	try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a"));
	3580	try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!"));
	3581	try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234"));
	3582	try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error"));
	3583	try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test"));
	3584	try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests"));
	3585	try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project"));
	3586	}
	3587
	3588	const Templates = struct {
	3589	zig_lib_directory: Cache.Directory,
	3590	dir: Io.Dir,
	3591	buffer: std.array_list.Managed(u8),
	3592
	3593	fn deinit(templates: *Templates, io: Io) void {
	3594	templates.zig_lib_directory.handle.close(io);
	3595	templates.dir.close(io);
	3596	templates.buffer.deinit();
	3597	templates.* = undefined;
	3598	}
	3599
	3600	fn write(
	3601	templates: *Templates,
	3602	arena: Allocator,
	3603	io: Io,
	3604	out_dir: Io.Dir,
	3605	root_name: []const u8,
	3606	template_path: []const u8,
	3607	fingerprint: Package.Fingerprint,
	3608	) !void {
	3609	if (Dir.path.dirname(template_path)) \|dirname\| {
	3610	out_dir.createDirPath(io, dirname) catch \|err\| {
	3611	fatal("unable to make path {q}: {t}", .{ dirname, err });
	3612	};
	3613	}
	3614
	3615	const max_bytes = 10 * 1024 * 1024;
	3616	const contents = templates.dir.readFileAlloc(io, template_path, arena, .limited(max_bytes)) catch \|err\| {
	3617	fatal("unable to read template file {q}: {t}", .{ template_path, err });
	3618	};
	3619	templates.buffer.clearRetainingCapacity();
	3620	try templates.buffer.ensureUnusedCapacity(contents.len);
	3621	var i: usize = 0;
	3622	while (i < contents.len) {
	3623	if (contents[i] == '_' or contents[i] == '.') {
	3624	// Both '_' and '.' are allowed because depending on the context
	3625	// one prefix will be valid, while the other might not.
	3626	if (std.mem.startsWith(u8, contents[i + 1 ..], "NAME")) {
	3627	try templates.buffer.appendSlice(root_name);
	3628	i += "_NAME".len;
	3629	continue;
	3630	} else if (std.mem.startsWith(u8, contents[i + 1 ..], "FINGERPRINT")) {
	3631	try templates.buffer.print("0x{x}", .{fingerprint.int()});
	3632	i += "_FINGERPRINT".len;
	3633	continue;
	3634	} else if (std.mem.startsWith(u8, contents[i + 1 ..], "ZIGVER")) {
	3635	try templates.buffer.appendSlice(builtin.zig_version_string);
	3636	i += "_ZIGVER".len;
	3637	continue;
	3638	}
	3639	}
	3640
	3641	try templates.buffer.append(contents[i]);
	3642	i += 1;
	3643	}
	3644
	3645	return out_dir.writeFile(io, .{
	3646	.sub_path = template_path,
	3647	.data = templates.buffer.items,
	3648	.flags = .{ .exclusive = true },
	3649	});
	3650	}
	3651
	3652	fn find(gpa: Allocator, io: Io, zig_lib_directory: Cache.Directory) Templates {
	3653	const template_path: Path = .{
	3654	.root_dir = zig_lib_directory,
	3655	.sub_path = "init",
	3656	};
	3657	const template_dir = template_path.root_dir.handle.openDir(io, template_path.sub_path, .{}) catch \|err\|
	3658	fatal("unable to open zig project template directory {f}: {t}", .{ template_path, err });
	3659	return .{
	3660	.zig_lib_directory = zig_lib_directory,
	3661	.dir = template_dir,
	3662	.buffer = std.array_list.Managed(u8).init(gpa),
	3663	};
	3664	}
	3665
	3666	fn writeSimpleFile(io: Io, file_name: []const u8, comptime format: []const u8, args: anytype) !void {
	3667	const f = try Io.Dir.cwd().createFile(io, file_name, .{ .exclusive = true });
	3668	defer f.close(io);
	3669	var buf: [4096]u8 = undefined;
	3670	var fw = f.writer(io, &buf);
	3671	try fw.interface.print(format, args);
	3672	try fw.interface.flush();
	3673	}
	3674	};
	3675
	3676	fn confPathDepToCachePath(graph: const Graph, c: const Configuration, path_dep: Configuration.PathDep) Path {
	3677	const sub_path = path_dep.sub.slice(c);
	3678	return switch (path_dep.flags.base) {
	3679	.cwd => .{
	3680	.root_dir = .cwd(),
	3681	.sub_path = sub_path,
	3682	},
	3683	.local_cache => .{
	3684	.root_dir = graph.local_cache_root,
	3685	.sub_path = sub_path,
	3686	},
	3687	.global_cache => .{
	3688	.root_dir = graph.global_cache_root,
	3689	.sub_path = sub_path,
	3690	},
	3691	.build_root => .{
	3692	.root_dir = switch (path_dep.pkg.unwrap().?) {
	3693	.root => graph.build_root_directory,
	3694	_ => @panic("TODO"),
	3695	},
	3696	.sub_path = sub_path,
	3697	},
	3698	.zig_lib => .{
	3699	.root_dir = graph.zig_lib_directory,
	3700	.sub_path = sub_path,
	3701	},
	3702	.zig_exe => @panic("TODO"),
	3703	.install_prefix => @panic("TODO"),
	3704	.install_lib => @panic("TODO"),
	3705	.install_bin => @panic("TODO"),
	3706	.install_include => @panic("TODO"),
	3707	};
	3708	}

lib/compiler/Maker/Fetch.zig created+2284

...	...	@@ -0,0 +1,2284 @@
	1	//! Represents one independent job whose responsibility is to:
	2	//!
	3	//! 1. Check the local zig package directory to see if the hash already exists.
	4	//! If so, load, parse, and validate the build.zig.zon file therein, and
	5	//! goto step 9. Likewise if the location is a relative path, treat this
	6	//! the same as a cache hit. Otherwise, proceed.
	7	//! 2. Check the global package cache for a compressed tarball matching the
	8	//! hash. If it is found, unpack the contents into a temporary directory inside
	9	//! project local zig cache. Rename this directory into the local zig package
	10	//! directory and goto step 9, skipping step 10.
	11	//! 3. Fetch and unpack a URL into a temporary directory.
	12	//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
	13	//! for the file to be missing, in which case this fetched package is considered
	14	//! to be a "naked" package.
	15	//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
	16	//! deleting excluded files. If any files had errors for files that were
	17	//! ultimately excluded, those errors should be ignored, such as failure to
	18	//! create symlinks that weren't supposed to be included anyway.
	19	//! 6. Compute the package hash based on the remaining files in the temporary
	20	//! directory.
	21	//! 7. Rename the temporary directory into the local zig package directory. If
	22	//! the hash already exists, delete the temporary directory and leave the zig
	23	//! package directory untouched as it may be in use. This is done even if
	24	//! the hash is invalid, in case the package with the different hash is used
	25	//! in the future.
	26	//! 8. Validate the computed hash against the expected hash. If invalid,
	27	//! this job is done.
	28	//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
	29	//! a mutex and a hash map so that redundant jobs do not get queued up.
	30	//! 10.Compress the package directory and store it into the global package
	31	//! cache.
	32	//!
	33	//! All of this must be done with only referring to the state inside this struct
	34	//! because this work will be done in a dedicated thread.
	35	const Fetch = @This();
	36
	37	const builtin = @import("builtin");
	38	const native_os = builtin.os.tag;
	39
	40	const std = @import("std");
	41	const Io = std.Io;
	42	const fs = std.fs;
	43	const log = std.log.scoped(.fetch);
	44	const assert = std.debug.assert;
	45	const ascii = std.ascii;
	46	const Allocator = std.mem.Allocator;
	47	const Path = std.Build.Cache.Path;
	48	const Directory = std.Build.Cache.Directory;
	49	const git = @import("Fetch/git.zig");
	50	const Package = @import("Package.zig");
	51	const Manifest = Package.Manifest;
	52	const ErrorBundle = std.zig.ErrorBundle;
	53
	54	arena: std.heap.ArenaAllocator,
	55	location: Location,
	56	location_tok: std.zig.Ast.TokenIndex,
	57	hash_tok: std.zig.Ast.OptionalTokenIndex,
	58	name_tok: std.zig.Ast.TokenIndex,
	59	lazy_status: LazyStatus,
	60	/// Same as `parent_packge_root` except it is unchanged when recursing into
	61	/// relative file paths (as opposed to URL).
	62	remote_package_root: Path,
	63	parent_package_root: Path,
	64	parent_manifest_ast: ?*const std.zig.Ast,
	65	prog_node: std.Progress.Node,
	66	job_queue: *JobQueue,
	67	/// If true, don't add an error for a missing hash. This flag is not passed
	68	/// down to recursive dependencies. It's intended to be used only be the CLI.
	69	omit_missing_hash_error: bool,
	70	/// If true, don't fail when a manifest file is missing the `paths` field,
	71	/// which specifies inclusion rules. This is intended to be true for the first
	72	/// fetch task and false for the recursive dependencies.
	73	allow_missing_paths_field: bool,
	74	/// If true and URL points to a Git repository, will use the latest commit.
	75	use_latest_commit: bool,
	76
	77	// Above this are fields provided as inputs to `run`.
	78	// Below this are fields populated by `run`.
	79
	80	/// Relative to the build root of the root package.
	81	package_root: Path,
	82	error_bundle: ErrorBundle.Wip,
	83	manifest: Manifest,
	84	manifest_ast: std.zig.Ast,
	85	have_manifest: bool,
	86	computed_hash: ComputedHash,
	87	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
	88	has_build_zig: bool,
	89	/// Indicates whether the task aborted due to an out-of-memory condition.
	90	oom_flag: bool,
	91	/// If `use_latest_commit` was true, this will be set to the commit that was used.
	92	/// If the resource pointed to by the location is not a Git-repository, this
	93	/// will be left unchanged.
	94	latest_commit: ?git.Oid,
	95
	96	// This field is used by the CLI only, untouched by this file.
	97
	98	/// The module for this `Fetch` tasks's package, which exposes `build.zig` as
	99	/// the root source file.
	100	///
	101	/// This could be an opaque "userdata" field because this code does not observe
	102	/// this data in any way but let's have some type safety because we can.
	103	cli_module: ?*@import("../Maker.zig").CliModule,
	104
	105	pub const LazyStatus = enum {
	106	/// Not lazy.
	107	eager,
	108	/// Lazy, found.
	109	available,
	110	/// Lazy, not found.
	111	unavailable,
	112	};
	113
	114	pub const LocalStorage = struct {
	115	cache_root: Path,
	116	/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
	117	pkg_root: Path,
	118	};
	119
	120	/// Contains shared state among all `Fetch` tasks.
	121	pub const JobQueue = struct {
	122	io: Io,
	123	mutex: Io.Mutex = .init,
	124	/// It's an array hash map so that it can be sorted before rendering the
	125	/// dependencies.zig source file.
	126	/// Protected by `mutex`.
	127	table: Table = .{},
	128	/// `table` may be missing some tasks such as ones that failed, so this
	129	/// field contains references to all of them.
	130	/// Protected by `mutex`.
	131	all_fetches: std.ArrayList(*Fetch) = .empty,
	132	prog_node: std.Progress.Node,
	133
	134	http_client: *std.http.Client,
	135	/// This tracks `Fetch` tasks as well as recompression tasks.
	136	group: Io.Group = .init,
	137	global_cache: Directory,
	138	/// If `null`, indicates fetch globally only.
	139	local_storage: ?*const LocalStorage,
	140	/// If true then, no fetching occurs, and:
	141	/// * The `global_cache` directory is assumed to be the direct parent
	142	/// directory of on-disk packages rather than having the "p/" directory
	143	/// prefix inside of it.
	144	/// * An error occurs if any non-lazy packages are not already present in
	145	/// the package cache directory.
	146	/// * Missing hash field causes an error, and no fetching occurs so it does
	147	/// not print the correct hash like usual.
	148	read_only: bool,
	149	recursive: bool,
	150	/// Dumps hash information to stdout which can be used to troubleshoot why
	151	/// two hashes of the same package do not match.
	152	/// If this is true, `recursive` must be false.
	153	debug_hash: bool,
	154	mode: Mode,
	155	/// Set of hashes that will be additionally fetched even if they are marked
	156	/// as lazy.
	157	unlazy_set: UnlazySet = .{},
	158	/// Identifies paths that override all packages in the tree with matching
	159	/// project ids.
	160	fork_set: ForkSet = .{},
	161
	162	pub const Mode = enum {
	163	/// Non-lazy dependencies are always fetched.
	164	/// Lazy dependencies are fetched only when needed.
	165	needed,
	166	/// Both non-lazy and lazy dependencies are always fetched.
	167	all,
	168	};
	169	pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch);
	170	pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void);
	171	pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false);
	172
	173	pub const Fork = struct {
	174	path: Path,
	175	manifest_ast: std.zig.Ast,
	176	manifest: Package.Manifest,
	177	uses: usize,
	178
	179	pub const Context = struct {
	180	pub fn hash(_: @This(), a: Fork) u32 {
	181	const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
	182	return @truncate(project_id.hash());
	183	}
	184
	185	pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool {
	186	const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
	187	const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
	188	return a_project_id.eql(&b_project_id);
	189	}
	190	};
	191
	192	pub const Adapter = struct {
	193	pub fn hash(_: @This(), a: Package.ProjectId) u32 {
	194	return @truncate(a.hash());
	195	}
	196
	197	pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool {
	198	const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
	199	return a_project_id.eql(&b_project_id);
	200	}
	201	};
	202	};
	203
	204	pub fn deinit(jq: *JobQueue) void {
	205	const io = jq.io;
	206	jq.group.cancel(io);
	207	if (jq.all_fetches.items.len == 0) return;
	208	const gpa = jq.all_fetches.items[0].arena.child_allocator;
	209	jq.table.deinit(gpa);
	210	// These must be deinitialized in reverse order because subsequent
	211	// `Fetch` instances are allocated in prior ones' arenas.
	212	// Sorry, I know it's a bit weird, but it slightly simplifies the
	213	// critical section.
	214	while (jq.all_fetches.pop()) \|f\| f.deinit();
	215	jq.all_fetches.deinit(gpa);
	216	jq.* = undefined;
	217	}
	218
	219	/// Dumps all subsequent error bundles into the first one.
	220	pub fn consolidateErrors(jq: *JobQueue) !void {
	221	const root = &jq.all_fetches.items[0].error_bundle;
	222	const gpa = root.gpa;
	223	for (jq.all_fetches.items[1..]) \|fetch\| {
	224	if (fetch.error_bundle.root_list.items.len > 0) {
	225	var bundle = try fetch.error_bundle.toOwnedBundle("");
	226	defer bundle.deinit(gpa);
	227	try root.addBundleAsRoots(bundle);
	228	}
	229	}
	230	}
	231
	232	/// Creates the dependencies.zig source code for the build runner to obtain
	233	/// via `@import("@dependencies")`.
	234	pub fn createDependenciesSource(jq: JobQueue, w: Io.Writer) Io.Writer.Error!void {
	235	const keys = jq.table.keys();
	236
	237	assert(keys.len != 0); // caller should have added the first one
	238	if (keys.len == 1) {
	239	// This is the first one. It must have no dependencies.
	240	return createEmptyDependenciesSource(w);
	241	}
	242
	243	try w.writeAll("pub const packages = struct {\n");
	244
	245	// Ensure the generated .zig file is deterministic.
	246	jq.table.sortUnstable(@as(struct {
	247	keys: []const Package.Hash,
	248	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
	249	return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
	250	}
	251	}, .{ .keys = keys }));
	252
	253	for (keys, jq.table.values()) \|*hash, fetch\| {
	254	if (fetch == jq.all_fetches.items[0]) {
	255	// The first one is a dummy package for the current project.
	256	continue;
	257	}
	258
	259	const hash_slice = hash.toSlice();
	260
	261	try w.print(
	262	\\ pub const {f} = struct {{
	263	\\
	264	, .{std.zig.fmtId(hash_slice)});
	265
	266	lazy: {
	267	switch (fetch.lazy_status) {
	268	.eager => break :lazy,
	269	.available => {
	270	try w.writeAll(
	271	\\ pub const available = true;
	272	\\
	273	);
	274	break :lazy;
	275	},
	276	.unavailable => {
	277	try w.writeAll(
	278	\\ pub const available = false;
	279	\\ };
	280	\\
	281	);
	282	continue;
	283	},
	284	}
	285	}
	286
	287	try w.print(
	288	\\ pub const build_root = "{f}";
	289	\\
	290	, .{std.fmt.alt(fetch.package_root, .formatEscapeString)});
	291
	292	if (fetch.has_build_zig) {
	293	try w.print(
	294	\\ pub const build_zig = @import("{f}");
	295	\\
	296	, .{std.zig.fmtString(hash_slice)});
	297	}
	298
	299	if (fetch.have_manifest) {
	300	const manifest = &fetch.manifest;
	301	try w.writeAll(
	302	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
	303	\\
	304	);
	305	for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
	306	const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
	307	try w.print(
	308	" .{{ \"{f}\", \"{f}\" }},\n",
	309	.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
	310	);
	311	}
	312
	313	try w.writeAll(
	314	\\ };
	315	\\ };
	316	\\
	317	);
	318	} else {
	319	try w.writeAll(
	320	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{};
	321	\\ };
	322	\\
	323	);
	324	}
	325	}
	326
	327	try w.writeAll(
	328	\\};
	329	\\
	330	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
	331	\\
	332	);
	333
	334	const root_fetch = jq.all_fetches.items[0];
	335	assert(root_fetch.have_manifest);
	336	const root_manifest = &root_fetch.manifest;
	337
	338	for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) \|name, dep\| {
	339	const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
	340	try w.print(
	341	" .{{ \"{f}\", \"{f}\" }},\n",
	342	.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
	343	);
	344	}
	345	try w.writeAll("};\n");
	346	}
	347
	348	pub fn createEmptyDependenciesSource(w: *Io.Writer) Io.Writer.Error!void {
	349	try w.writeAll(
	350	\\pub const packages = struct {};
	351	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
	352	\\
	353	);
	354	}
	355
	356	fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Path) Io.Cancelable!void {
	357	const pkg_hash_slice = package_hash.toSlice();
	358
	359	const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
	360	defer prog_node.end();
	361
	362	var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
	363	const dest_path: Path = .{
	364	.root_dir = jq.global_cache,
	365	.sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
	366	};
	367
	368	const gpa = jq.http_client.allocator;
	369
	370	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	371	defer arena_instance.deinit();
	372	const arena = arena_instance.allocator();
	373
	374	recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch \|err\| switch (err) {
	375	error.Canceled => \|e\| return e,
	376	error.ReadFailed => comptime unreachable,
	377	error.WriteFailed => comptime unreachable,
	378	else => \|e\| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
	379	};
	380	}
	381
	382	fn recompressFallible(
	383	jq: *JobQueue,
	384	arena: Allocator,
	385	dest_path: Path,
	386	pkg_hash_slice: []const u8,
	387	package_root: Path,
	388	prog_node: std.Progress.Node,
	389	) !void {
	390	const gpa = jq.http_client.allocator;
	391	const io = jq.io;
	392
	393	// We have to walk the file system up front in order to sort the file
	394	// list for determinism purposes. The hash of the recompressed file is
	395	// not critical because the true hash is based on the content alone.
	396	// However, if we want Zig users to be able to share cached package
	397	// data with each other via peer-to-peer protocols, we benefit greatly
	398	// from the data being identical on everyone's computers.
	399	var scanned_files: std.ArrayList(ScannedFile) = .empty;
	400	defer scanned_files.deinit(gpa);
	401
	402	var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
	403	defer pkg_dir.close(io);
	404
	405	{
	406	var walker = try pkg_dir.walk(gpa);
	407	defer walker.deinit();
	408
	409	while (try walker.next(io)) \|entry\| {
	410	const symlink = switch (entry.kind) {
	411	.directory => continue,
	412	.file => false,
	413	.sym_link => true,
	414	else => return error.IllegalFileType,
	415	};
	416	const entry_path = try arena.dupe(u8, entry.path);
	417	// If necessary, normalize path separators to POSIX-style since the tar format requires that.
	418	if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) {
	419	std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix);
	420	}
	421	try scanned_files.append(gpa, .{
	422	.ptr = entry_path.ptr,
	423	.len = @intCast(entry_path.len),
	424	.symlink = symlink,
	425	});
	426	}
	427
	428	std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp);
	429	}
	430
	431	prog_node.setEstimatedTotalItems(scanned_files.items.len);
	432
	433	var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
	434	.make_path = true,
	435	.replace = true,
	436	});
	437	defer atomic_file.deinit(io);
	438
	439	var file_write_buffer: [4096]u8 = undefined;
	440	var file_writer = atomic_file.file.writer(io, &file_write_buffer);
	441
	442	var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	443	var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch \|err\| switch (err) {
	444	error.WriteFailed => return file_writer.err.?,
	445	};
	446
	447	var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
	448	archiver.prefix = pkg_hash_slice;
	449
	450	var file_read_buffer: [4096]u8 = undefined;
	451	var link_buf: [fs.max_path_bytes]u8 = undefined;
	452
	453	for (scanned_files.items) \|scanned_file\| {
	454	const entry_path = scanned_file.ptr[0..scanned_file.len];
	455	if (scanned_file.symlink) {
	456	const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)];
	457	archiver.writeLink(entry_path, link_name, .{}) catch \|err\| switch (err) {
	458	error.WriteFailed => return file_writer.err.?,
	459	else => \|e\| return e,
	460	};
	461	} else {
	462	var file = try pkg_dir.openFile(io, entry_path, .{});
	463	defer file.close(io);
	464	var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
	465	archiver.writeFile(entry_path, &file_reader, 0) catch \|err\| switch (err) {
	466	error.ReadFailed => return file_reader.err.?,
	467	error.WriteFailed => return file_writer.err.?,
	468	else => \|e\| return e,
	469	};
	470	}
	471	prog_node.completeOne();
	472	}
	473
	474	// intentionally omitting the pointless trailer
	475	//try archiver.finish();
	476	compress.finish() catch \|err\| switch (err) {
	477	error.WriteFailed => return file_writer.err.?,
	478	};
	479	try file_writer.flush();
	480	try atomic_file.replace(io);
	481	}
	482	};
	483
	484	const ScannedFile = struct {
	485	ptr: [*]const u8,
	486	len: u32,
	487	symlink: bool,
	488	};
	489
	490	fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool {
	491	return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]);
	492	}
	493
	494	pub const Location = union(enum) {
	495	remote: Remote,
	496	/// A directory found inside the parent package.
	497	relative_path: Path,
	498	/// Recursive Fetch tasks will never use this Location, but it may be
	499	/// passed in by the CLI. Indicates the file contents here should be copied
	500	/// into the global package cache. It may be a file relative to the cwd or
	501	/// absolute, in which case it should be treated exactly like a `file://`
	502	/// URL, or a directory, in which case it should be treated as an
	503	/// already-unpacked directory (but still needs to be copied into the
	504	/// global package cache and have inclusion rules applied).
	505	path_or_url: []const u8,
	506
	507	pub const Remote = struct {
	508	url: []const u8,
	509	/// If this is null it means the user omitted the hash field from a dependency.
	510	/// It will be an error but the logic should still fetch and print the discovered hash.
	511	hash: ?Package.Hash,
	512	};
	513	};
	514
	515	pub const RunError = error{
	516	OutOfMemory,
	517	Canceled,
	518	/// This error code is intended to be handled by inspecting the
	519	/// `error_bundle` field.
	520	FetchFailed,
	521	};
	522
	523	pub fn run(f: *Fetch) RunError!void {
	524	const job_queue = f.job_queue;
	525	const io = job_queue.io;
	526	const eb = &f.error_bundle;
	527	const arena = f.arena.allocator();
	528	const gpa = f.arena.child_allocator;
	529
	530	try eb.init(gpa);
	531
	532	// Check the global zig package cache to see if the hash already exists. If
	533	// so, load, parse, and validate the build.zig.zon file therein, and skip
	534	// ahead to queuing up jobs for dependencies. Likewise if the location is a
	535	// relative path, treat this the same as a cache hit. Otherwise, proceed.
	536
	537	const remote = switch (f.location) {
	538	.relative_path => \|pkg_root\| {
	539	if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
	540	f.location_tok,
	541	try eb.addString("expected path relative to build root; found absolute path"),
	542	);
	543	if (f.hash_tok.unwrap()) \|hash_tok\| return f.fail(
	544	hash_tok,
	545	try eb.addString("path-based dependencies are not hashed"),
	546	);
	547	// Packages fetched by URL may not use relative paths to escape outside the
	548	// fetched package directory from within the package cache.
	549
	550	// This code path is only reachable recursively and the sub_path
	551	// will already have been resolved to no longer have extra ".." or
	552	// "." components.
	553	assert(job_queue.local_storage != null);
	554	assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir));
	555	if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail(
	556	f.location_tok,
	557	try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
	558	);
	559	f.package_root = pkg_root;
	560	try loadManifest(f, pkg_root);
	561	if (!f.has_build_zig) try checkBuildFileExistence(f);
	562	if (!job_queue.recursive) return;
	563	return queueJobsForDeps(f);
	564	},
	565	.remote => \|remote\| remote,
	566	.path_or_url => \|path_or_url\| {
	567	if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) \|dir\| {
	568	var resource: Resource = .{ .dir = dir };
	569	return f.runResource(path_or_url, &resource, null, false);
	570	} else \|dir_err\| {
	571	var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
	572
	573	const file_err = if (dir_err == error.NotDir) e: {
	574	if (Io.Dir.cwd().openFile(io, path_or_url, .{})) \|file\| {
	575	var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };
	576	return f.runResource(path_or_url, &resource, null, false);
	577	} else \|err\| break :e err;
	578	} else dir_err;
	579
	580	const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
	581	return f.fail(0, try eb.printString(
	582	"'{s}' could not be recognized as a file path ({t}) or an URL ({t})",
	583	.{ path_or_url, file_err, uri_err },
	584	));
	585	};
	586	var resource: Resource = undefined;
	587	try f.initResource(uri, &resource, &server_header_buffer);
	588	return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false);
	589	}
	590	},
	591	};
	592
	593	var resource_buffer: [init_resource_buffer_size]u8 = undefined;
	594
	595	if (remote.hash) \|expected_hash\| {
	596	const expected_project_id: Package.ProjectId = expected_hash.projectId();
	597	if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) \|fork\| {
	598	log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
	599	fork.uses += 1;
	600	f.package_root = fork.path;
	601	f.remote_package_root = f.package_root;
	602	f.manifest_ast = fork.manifest_ast;
	603	f.manifest = fork.manifest;
	604	f.have_manifest = true;
	605	try checkBuildFileExistence(f);
	606	if (!job_queue.recursive) return;
	607	return queueJobsForDeps(f);
	608	}
	609
	610	if (job_queue.local_storage) \|ls\| {
	611	const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
	612	if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) \|_\| {
	613	assert(f.lazy_status != .unavailable);
	614	f.package_root = package_root;
	615	f.remote_package_root = f.package_root;
	616	try loadManifest(f, f.package_root);
	617	try checkBuildFileExistence(f);
	618	if (!job_queue.recursive) return;
	619	return queueJobsForDeps(f);
	620	} else \|err\| switch (err) {
	621	error.FileNotFound => {
	622	log.debug("FileNotFound: {f}", .{package_root});
	623	if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
	624	f.name_tok,
	625	try eb.printString("package not found at '{f}'", .{package_root}),
	626	);
	627	},
	628	error.Canceled => \|e\| return e,
	629	else => \|e\| {
	630	try eb.addRootErrorMessage(.{
	631	.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
	632	package_root, e,
	633	}),
	634	});
	635	return error.FetchFailed;
	636	},
	637	}
	638	}
	639
	640	// Check global cache before remote fetch.
	641	const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()});
	642	const cached_tarball_path: Path = .{
	643	.root_dir = job_queue.global_cache,
	644	.sub_path = cached_tarball_sub_path,
	645	};
	646	if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) \|file\| {
	647	log.debug("found global cached tarball {f}", .{cached_tarball_path});
	648	var resource: Resource = .{ .file = file.reader(io, &resource_buffer) };
	649	return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true);
	650	} else \|err\| switch (err) {
	651	error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}),
	652	error.Canceled => \|e\| return e,
	653	else => \|e\| {
	654	try eb.addRootErrorMessage(.{
	655	.msg = try eb.printString("unable to open globally cached package {f}: {t}", .{
	656	cached_tarball_path, e,
	657	}),
	658	});
	659	return error.FetchFailed;
	660	},
	661	}
	662
	663	switch (f.lazy_status) {
	664	.eager => {},
	665	.available => if (!job_queue.unlazy_set.contains(expected_hash)) {
	666	f.lazy_status = .unavailable;
	667	return;
	668	},
	669	.unavailable => unreachable,
	670	}
	671	} else if (job_queue.read_only) {
	672	try eb.addRootErrorMessage(.{
	673	.msg = try eb.addString("dependency is missing hash field"),
	674	.src_loc = try f.srcLoc(f.location_tok),
	675	});
	676	return error.FetchFailed;
	677	}
	678
	679	// Fetch and unpack the remote into a temporary directory.
	680	const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
	681	f.location_tok,
	682	try eb.printString("invalid URI: {t}", .{err}),
	683	);
	684	var resource: Resource = undefined;
	685	try f.initResource(uri, &resource, &resource_buffer);
	686	return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false);
	687	}
	688
	689	pub fn deinit(f: *Fetch) void {
	690	f.error_bundle.deinit();
	691	f.arena.deinit();
	692	}
	693
	694	/// Consumes `resource`, even if an error is returned.
	695	fn runResource(
	696	f: *Fetch,
	697	uri_path: []const u8,
	698	resource: *Resource,
	699	remote_hash: ?Package.Hash,
	700	disable_recompress: bool,
	701	) RunError!void {
	702	const job_queue = f.job_queue;
	703	assert(!job_queue.read_only);
	704
	705	const io = job_queue.io;
	706	defer resource.deinit(io);
	707
	708	const arena = f.arena.allocator();
	709	const eb = &f.error_bundle;
	710	const rand_int = r: {
	711	var x: u64 = undefined;
	712	io.random(@ptrCast(&x));
	713	break :r x;
	714	};
	715	const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
	716	const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
	717	const tmp_directory_path: Path = if (job_queue.local_storage) \|ls\|
	718	try ls.pkg_root.join(arena, tmp_dir_sub_path)
	719	else
	720	.{
	721	.root_dir = job_queue.global_cache,
	722	.sub_path = tmp_tmp_dir_sub_path,
	723	};
	724
	725	const package_sub_path = blk: {
	726	var tmp_directory: Directory = .{
	727	.path = tmp_directory_path.sub_path,
	728	.handle = handle: {
	729	const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
	730	.open_options = .{ .iterate = true },
	731	}) catch \|err\| {
	732	try eb.addRootErrorMessage(.{
	733	.msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
	734	tmp_directory_path, err,
	735	}),
	736	});
	737	return error.FetchFailed;
	738	};
	739	break :handle dir;
	740	},
	741	};
	742	defer tmp_directory.handle.close(io);
	743
	744	// Fetch and unpack a resource into a temporary directory.
	745	var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
	746
	747	const pkg_path: Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
	748
	749	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
	750	// for the file to be missing, in which case this fetched package is
	751	// considered to be a "naked" package.
	752	try loadManifest(f, pkg_path);
	753
	754	const filter: Filter = .{
	755	.include_paths = if (f.have_manifest) f.manifest.paths else .{},
	756	};
	757
	758	// Ignore errors that were excluded by manifest, such as failure to
	759	// create symlinks that weren't supposed to be included anyway.
	760	try unpack_result.validate(f, filter);
	761
	762	// Apply the manifest's inclusion rules to the temporary directory by
	763	// deleting excluded files.
	764	// Empty directories have already been omitted by `unpackResource`.
	765	// Compute the package hash based on the remaining files in the temporary
	766	// directory.
	767	f.computed_hash = try computeHash(f, pkg_path, filter);
	768
	769	if (unpack_result.root_dir.len > 0)
	770	break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
	771
	772	break :blk tmp_directory_path;
	773	};
	774
	775	const computed_package_hash = computedPackageHash(f);
	776
	777	// Rename the temporary directory into the local zig package directory. If
	778	// the hash already exists, delete the temporary directory and leave the
	779	// zig package directory untouched as it may be in use. This is done even
	780	// if the hash is invalid, in case the package with the different hash is
	781	// used in the future.
	782	if (job_queue.local_storage) \|ls\| {
	783	f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
	784	renameTmpIntoCache(io, package_sub_path, f.package_root) catch \|err\| {
	785	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	786	"failed to rename temporary directory {f} into package cache directory {f}: {t}",
	787	.{ package_sub_path, f.package_root, err },
	788	) });
	789	return error.FetchFailed;
	790	};
	791	} else {
	792	f.package_root = tmp_directory_path;
	793	}
	794	f.remote_package_root = f.package_root;
	795
	796	if (!disable_recompress) {
	797	// Spin off a task to recompress the tarball, with filtered files deleted, into
	798	// the global cache.
	799	job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
	800	}
	801
	802	// Remove temporary directory root if not already renamed to global cache.
	803	if (!package_sub_path.eql(tmp_directory_path)) {
	804	tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch \|err\| switch (err) {
	805	error.Canceled => \|e\| return e,
	806	else => \|e\| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }),
	807	};
	808	}
	809
	810	// Validate the computed hash against the expected hash. If invalid, this
	811	// job is done.
	812
	813	if (remote_hash) \|declared_hash\| {
	814	const hash_tok = f.hash_tok.unwrap().?;
	815	if (!computed_package_hash.eql(&declared_hash)) {
	816	return f.fail(hash_tok, try eb.printString(
	817	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	818	.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
	819	));
	820	}
	821	} else if (!f.omit_missing_hash_error) {
	822	const notes_len = 1;
	823	try eb.addRootErrorMessage(.{
	824	.msg = try eb.addString("dependency is missing hash field"),
	825	.src_loc = try f.srcLoc(f.location_tok),
	826	.notes_len = notes_len,
	827	});
	828	const notes_start = try eb.reserveNotes(notes_len);
	829	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
	830	.msg = try eb.printString("expected .hash = {q},", .{computed_package_hash.toSlice()}),
	831	}));
	832	return error.FetchFailed;
	833	}
	834
	835	// Spawn a new fetch job for each dependency in the manifest file. Use
	836	// a mutex and a hash map so that redundant jobs do not get queued up.
	837	if (!job_queue.recursive) return;
	838	return queueJobsForDeps(f);
	839	}
	840
	841	pub fn computedPackageHash(f: *const Fetch) Package.Hash {
	842	const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
	843	if (f.have_manifest) {
	844	const man = &f.manifest;
	845	var version_buffer: [32]u8 = undefined;
	846	const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer;
	847	return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
	848	}
	849	// In the future build.zig.zon fields will be added to allow overriding these values
	850	// for naked tarballs.
	851	return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
	852	}
	853
	854	/// `computeHash` gets a free check for the existence of `build.zig`, but when
	855	/// not computing a hash, we need to do a syscall to check for it.
	856	fn checkBuildFileExistence(f: *Fetch) RunError!void {
	857	const io = f.job_queue.io;
	858	const eb = &f.error_bundle;
	859	if (f.package_root.access(io, std.zig.build_zig_basename, .{})) \|_\| {
	860	f.has_build_zig = true;
	861	} else \|err\| switch (err) {
	862	error.FileNotFound => {},
	863	else => \|e\| {
	864	try eb.addRootErrorMessage(.{
	865	.msg = try eb.printString("unable to access {f}/{s}: {t}", .{
	866	f.package_root, std.zig.build_zig_basename, e,
	867	}),
	868	});
	869	return error.FetchFailed;
	870	},
	871	}
	872	}
	873
	874	/// This function populates `f.manifest` or leaves it `null`.
	875	fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
	876	const io = f.job_queue.io;
	877	const eb = &f.error_bundle;
	878	const arena = f.arena.allocator();
	879	const manifest_path = try pkg_root.join(arena, Manifest.basename);
	880
	881	Manifest.load(
	882	io,
	883	arena,
	884	manifest_path,
	885	&f.manifest_ast,
	886	eb,
	887	&f.manifest,
	888	f.allow_missing_paths_field,
	889	) catch \|err\| switch (err) {
	890	error.FileNotFound => return,
	891	error.Canceled => \|e\| return e,
	892	error.ErrorsBundled => return error.FetchFailed,
	893	else => \|e\| {
	894	try eb.addRootErrorMessage(.{
	895	.msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
	896	});
	897	return error.FetchFailed;
	898	},
	899	};
	900	f.have_manifest = true;
	901	}
	902
	903	fn queueJobsForDeps(f: *Fetch) RunError!void {
	904	const io = f.job_queue.io;
	905
	906	assert(f.job_queue.recursive);
	907
	908	// If the package does not have a build.zig.zon file then there are no dependencies.
	909	if (!f.have_manifest) return;
	910	const manifest = &f.manifest;
	911
	912	const new_fetches, const prog_names = nf: {
	913	const parent_arena = f.arena.allocator();
	914	const gpa = f.arena.child_allocator;
	915	const cache_root = f.job_queue.global_cache;
	916	const dep_names = manifest.dependencies.keys();
	917	const deps = manifest.dependencies.values();
	918	// Grab the new tasks into a temporary buffer so we can unlock that mutex
	919	// as fast as possible.
	920	// This overallocates any fetches that get skipped by the `continue` in the
	921	// loop below.
	922	const new_fetches = try parent_arena.alloc(Fetch, deps.len);
	923	const prog_names = try parent_arena.alloc([]const u8, deps.len);
	924	var new_fetch_index: usize = 0;
	925
	926	try f.job_queue.mutex.lock(io);
	927	defer f.job_queue.mutex.unlock(io);
	928
	929	try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
	930	try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
	931
	932	// There are four cases here:
	933	// * Correct hash is provided by manifest.
	934	// - Hash map already has the entry, no need to add it again.
	935	// * Incorrect hash is provided by manifest.
	936	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
	937	// * Hash is not provided by manifest.
	938	// - Hash missing error emitted; `queueJobsForDeps` is not called.
	939	// * path-based location is used without a hash.
	940	// - Hash is added to the table based on the path alone before
	941	// calling run(); no need to add it again.
	942	//
	943	// If we add a dep as lazy and then later try to add the same dep as eager,
	944	// eagerness takes precedence and the existing entry is updated and re-scheduled
	945	// for fetching.
	946
	947	for (dep_names, deps) \|dep_name, dep\| {
	948	var promoted_existing_to_eager = false;
	949	const new_fetch = &new_fetches[new_fetch_index];
	950	const location: Location = switch (dep.location) {
	951	.url => \|url\| .{
	952	.remote = .{
	953	.url = url,
	954	.hash = h: {
	955	const h = dep.hash orelse break :h null;
	956	const pkg_hash: Package.Hash = .fromSlice(h);
	957	if (h.len == 0) break :h pkg_hash;
	958	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
	959	if (gop.found_existing) {
	960	if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
	961	gop.value_ptr.*.lazy_status = .eager;
	962	promoted_existing_to_eager = true;
	963	} else {
	964	continue;
	965	}
	966	}
	967	gop.value_ptr.* = new_fetch;
	968	break :h pkg_hash;
	969	},
	970	},
	971	},
	972	.path => \|rel_path\| l: {
	973	// This might produce an invalid path, which is checked for
	974	// at the beginning of run().
	975	const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
	976	const pkg_hash = relativePathDigest(new_root, cache_root);
	977	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
	978	if (gop.found_existing) {
	979	if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
	980	gop.value_ptr.*.lazy_status = .eager;
	981	promoted_existing_to_eager = true;
	982	} else {
	983	continue;
	984	}
	985	}
	986	gop.value_ptr.* = new_fetch;
	987	break :l .{ .relative_path = new_root };
	988	},
	989	};
	990	prog_names[new_fetch_index] = dep_name;
	991	new_fetch_index += 1;
	992	if (!promoted_existing_to_eager) {
	993	f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
	994	}
	995	new_fetch.* = .{
	996	.arena = std.heap.ArenaAllocator.init(gpa),
	997	.location = location,
	998	.location_tok = dep.location_tok,
	999	.hash_tok = dep.hash_tok,
	1000	.name_tok = dep.name_tok,
	1001	.lazy_status = switch (f.job_queue.mode) {
	1002	.needed => if (dep.lazy) .available else .eager,
	1003	.all => .eager,
	1004	},
	1005	.parent_package_root = f.package_root,
	1006	.remote_package_root = f.remote_package_root,
	1007	.parent_manifest_ast = &f.manifest_ast,
	1008	.prog_node = f.prog_node,
	1009	.job_queue = f.job_queue,
	1010	.omit_missing_hash_error = false,
	1011	.allow_missing_paths_field = true,
	1012	.use_latest_commit = false,
	1013
	1014	.package_root = undefined,
	1015	.error_bundle = undefined,
	1016	.manifest = undefined,
	1017	.manifest_ast = undefined,
	1018	.have_manifest = false,
	1019	.computed_hash = undefined,
	1020	.has_build_zig = false,
	1021	.oom_flag = false,
	1022	.latest_commit = null,
	1023
	1024	.cli_module = null,
	1025	};
	1026	}
	1027
	1028	f.prog_node.increaseEstimatedTotalItems(new_fetch_index);
	1029
	1030	break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
	1031	};
	1032
	1033	// Now it's time to dispatch tasks.
	1034	for (new_fetches, prog_names) \|*new_fetch, prog_name\| {
	1035	f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name });
	1036	}
	1037	}
	1038
	1039	pub fn relativePathDigest(pkg_root: Path, cache_root: Directory) Package.Hash {
	1040	return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
	1041	}
	1042
	1043	pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void {
	1044	const prog_node = f.prog_node.start(prog_name, 0);
	1045	defer prog_node.end();
	1046
	1047	run(f) catch \|err\| switch (err) {
	1048	error.OutOfMemory => f.oom_flag = true,
	1049	error.Canceled => \|e\| return e,
	1050	error.FetchFailed => {
	1051	// Nothing to do because the errors are already reported in `error_bundle`,
	1052	// and a reference is kept to the `Fetch` task inside `all_fetches`.
	1053	},
	1054	};
	1055	}
	1056
	1057	fn srcLoc(
	1058	f: *Fetch,
	1059	tok: std.zig.Ast.TokenIndex,
	1060	) Allocator.Error!ErrorBundle.SourceLocationIndex {
	1061	const ast = f.parent_manifest_ast orelse return .none;
	1062	const eb = &f.error_bundle;
	1063	const start_loc = ast.tokenLocation(0, tok);
	1064	const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root});
	1065	const msg_off = 0;
	1066	return eb.addSourceLocation(.{
	1067	.src_path = src_path,
	1068	.span_start = ast.tokenStart(tok),
	1069	.span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len),
	1070	.span_main = ast.tokenStart(tok) + msg_off,
	1071	.line = @intCast(start_loc.line),
	1072	.column = @intCast(start_loc.column),
	1073	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	1074	});
	1075	}
	1076
	1077	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
	1078	const eb = &f.error_bundle;
	1079	try eb.addRootErrorMessage(.{
	1080	.msg = msg_str,
	1081	.src_loc = try f.srcLoc(msg_tok),
	1082	});
	1083	return error.FetchFailed;
	1084	}
	1085
	1086	const Resource = union(enum) {
	1087	file: Io.File.Reader,
	1088	http_request: HttpRequest,
	1089	git: Git,
	1090	dir: Io.Dir,
	1091
	1092	const Git = struct {
	1093	session: git.Session,
	1094	fetch_stream: git.Session.FetchStream,
	1095	want_oid: git.Oid,
	1096	};
	1097
	1098	const HttpRequest = struct {
	1099	request: std.http.Client.Request,
	1100	response: std.http.Client.Response,
	1101	transfer_buffer: []u8,
	1102	decompress: std.http.Decompress,
	1103	decompress_buffer: []u8,
	1104	};
	1105
	1106	fn deinit(resource: *Resource, io: Io) void {
	1107	switch (resource.*) {
	1108	.file => \|*file_reader\| file_reader.file.close(io),
	1109	.http_request => \|*http_request\| http_request.request.deinit(),
	1110	.git => \|*git_resource\| {
	1111	git_resource.fetch_stream.deinit();
	1112	},
	1113	.dir => \|*dir\| dir.close(io),
	1114	}
	1115	resource.* = undefined;
	1116	}
	1117
	1118	fn reader(resource: Resource) Io.Reader {
	1119	return switch (resource.*) {
	1120	.file => \|*file_reader\| return &file_reader.interface,
	1121	.http_request => \|*http_request\| return http_request.response.readerDecompressing(
	1122	http_request.transfer_buffer,
	1123	&http_request.decompress,
	1124	http_request.decompress_buffer,
	1125	),
	1126	.git => \|*g\| return &g.fetch_stream.reader,
	1127	.dir => unreachable,
	1128	};
	1129	}
	1130	};
	1131
	1132	const FileType = enum {
	1133	tar,
	1134	@"tar.gz",
	1135	@"tar.xz",
	1136	@"tar.zst",
	1137	git_pack,
	1138	zip,
	1139
	1140	fn fromPath(file_path: []const u8) ?FileType {
	1141	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
	1142	if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz";
	1143	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
	1144	if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz";
	1145	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
	1146	if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";
	1147	if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";
	1148	if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip;
	1149	if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip;
	1150	return null;
	1151	}
	1152
	1153	/// Parameter is a content-disposition header value.
	1154	fn fromContentDisposition(cd_header: []const u8) ?FileType {
	1155	const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse
	1156	return null;
	1157
	1158	var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse
	1159	return null;
	1160	value_start += "filename".len;
	1161	if (cd_header[value_start] == '*') {
	1162	value_start += 1;
	1163	}
	1164	if (cd_header[value_start] != '=') return null;
	1165	value_start += 1;
	1166
	1167	var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len;
	1168	if (cd_header[value_end - 1] == '\"') {
	1169	value_end -= 1;
	1170	}
	1171	return fromPath(cd_header[value_start..value_end]);
	1172	}
	1173
	1174	test fromContentDisposition {
	1175	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
	1176	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\""));
	1177	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
	1178	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
	1179	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
	1180	try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
	1181
	1182	try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
	1183	try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
	1184	try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
	1185	try std.testing.expect(fromContentDisposition("inline; size=42") == null);
	1186	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null);
	1187	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null);
	1188	}
	1189	};
	1190
	1191	const init_resource_buffer_size = git.Packet.max_data_length;
	1192
	1193	fn initResource(f: Fetch, uri: std.Uri, resource: Resource, reader_buffer: []u8) RunError!void {
	1194	const io = f.job_queue.io;
	1195	const arena = f.arena.allocator();
	1196	const eb = &f.error_bundle;
	1197
	1198	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
	1199	const path = try uri.path.toRawMaybeAlloc(arena);
	1200	const file = f.parent_package_root.openFile(io, path, .{}) catch \|err\| {
	1201	return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{
	1202	f.parent_package_root, path, err,
	1203	}));
	1204	};
	1205	resource.* = .{ .file = file.reader(io, reader_buffer) };
	1206	return;
	1207	}
	1208
	1209	const http_client = f.job_queue.http_client;
	1210
	1211	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
	1212	ascii.eqlIgnoreCase(uri.scheme, "https"))
	1213	{
	1214	resource.* = .{ .http_request = .{
	1215	.request = http_client.request(.GET, uri, .{}) catch \|err\|
	1216	return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})),
	1217	.response = undefined,
	1218	.transfer_buffer = reader_buffer,
	1219	.decompress_buffer = &.{},
	1220	.decompress = undefined,
	1221	} };
	1222	const request = &resource.http_request.request;
	1223	errdefer request.deinit();
	1224
	1225	request.sendBodiless() catch \|err\|
	1226	return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
	1227
	1228	var redirect_buffer: [8000]u8 = undefined;
	1229	const response = &resource.http_request.response;
	1230	response.* = request.receiveHead(&redirect_buffer) catch \|err\| switch (err) {
	1231	error.ReadFailed => {
	1232	return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
	1233	request.connection.?.getReadError().?,
	1234	}));
	1235	},
	1236	else => \|e\| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
	1237	};
	1238
	1239	if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
	1240	"bad HTTP response code: '{d} {s}'",
	1241	.{ response.head.status, response.head.status.phrase() orelse "" },
	1242	));
	1243
	1244	resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	1245	return;
	1246	}
	1247
	1248	if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
	1249	ascii.eqlIgnoreCase(uri.scheme, "git+https"))
	1250	{
	1251	var transport_uri = uri;
	1252	transport_uri.scheme = uri.scheme["git+".len..];
	1253	var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch \|err\| {
	1254	return f.fail(
	1255	f.location_tok,
	1256	try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
	1257	);
	1258	};
	1259
	1260	const want_oid = want_oid: {
	1261	const want_ref =
	1262	if (uri.fragment) \|fragment\| try fragment.toRawMaybeAlloc(arena) else "HEAD";
	1263	if (git.Oid.parseAny(want_ref)) \|oid\| break :want_oid oid else \|_\| {}
	1264
	1265	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
	1266	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
	1267
	1268	var ref_iterator: git.Session.RefIterator = undefined;
	1269	session.listRefs(&ref_iterator, .{
	1270	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
	1271	.include_peeled = true,
	1272	.buffer = reader_buffer,
	1273	}) catch \|err\| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err}));
	1274	defer ref_iterator.deinit();
	1275	while (ref_iterator.next() catch \|err\| {
	1276	return f.fail(f.location_tok, try eb.printString(
	1277	"unable to iterate refs: {s}",
	1278	.{@errorName(err)},
	1279	));
	1280	}) \|ref\| {
	1281	if (std.mem.eql(u8, ref.name, want_ref) or
	1282	std.mem.eql(u8, ref.name, want_ref_head) or
	1283	std.mem.eql(u8, ref.name, want_ref_tag))
	1284	{
	1285	break :want_oid ref.peeled orelse ref.oid;
	1286	}
	1287	}
	1288	return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
	1289	};
	1290	if (f.use_latest_commit) {
	1291	f.latest_commit = want_oid;
	1292	} else if (uri.fragment == null) {
	1293	const notes_len = 1;
	1294	try eb.addRootErrorMessage(.{
	1295	.msg = try eb.addString("url field is missing an explicit ref"),
	1296	.src_loc = try f.srcLoc(f.location_tok),
	1297	.notes_len = notes_len,
	1298	});
	1299	const notes_start = try eb.reserveNotes(notes_len);
	1300	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
	1301	.msg = try eb.printString("try .url = \"{f}#{f}\",", .{
	1302	uri.fmt(.{ .scheme = true, .authority = true, .path = true }),
	1303	want_oid,
	1304	}),
	1305	}));
	1306	return error.FetchFailed;
	1307	}
	1308
	1309	var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;
	1310	_ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable;
	1311	resource.* = .{ .git = .{
	1312	.session = session,
	1313	.fetch_stream = undefined,
	1314	.want_oid = want_oid,
	1315	} };
	1316	const fetch_stream = &resource.git.fetch_stream;
	1317	session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch \|err\| {
	1318	return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
	1319	};
	1320	errdefer fetch_stream.deinit(fetch_stream);
	1321
	1322	return;
	1323	}
	1324
	1325	return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme}));
	1326	}
	1327
	1328	fn unpackResource(
	1329	f: *Fetch,
	1330	resource: *Resource,
	1331	uri_path: []const u8,
	1332	tmp_directory: Directory,
	1333	) RunError!UnpackResult {
	1334	const eb = &f.error_bundle;
	1335	const file_type = switch (resource.*) {
	1336	.file => FileType.fromPath(uri_path) orelse
	1337	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
	1338
	1339	.http_request => \|*http_request\| ft: {
	1340	const head = &http_request.response.head;
	1341
	1342	// Content-Type takes first precedence.
	1343	const content_type = head.content_type orelse
	1344	return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
	1345
	1346	// Extract the MIME type, ignoring charset and boundary directives
	1347	const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len;
	1348	const mime_type = content_type[0..mime_type_end];
	1349
	1350	if (ascii.eqlIgnoreCase(mime_type, "application/x-tar"))
	1351	break :ft .tar;
	1352
	1353	if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or
	1354	ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or
	1355	ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or
	1356	ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or
	1357	ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed"))
	1358	{
	1359	break :ft .@"tar.gz";
	1360	}
	1361
	1362	if (ascii.eqlIgnoreCase(mime_type, "application/x-xz"))
	1363	break :ft .@"tar.xz";
	1364
	1365	if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))
	1366	break :ft .@"tar.zst";
	1367
	1368	if (ascii.eqlIgnoreCase(mime_type, "application/zip") or
	1369	ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or
	1370	ascii.eqlIgnoreCase(mime_type, "application/java-archive"))
	1371	{
	1372	break :ft .zip;
	1373	}
	1374
	1375	if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and
	1376	!ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))
	1377	{
	1378	return f.fail(f.location_tok, try eb.printString(
	1379	"unrecognized 'Content-Type' header: '{s}'",
	1380	.{content_type},
	1381	));
	1382	}
	1383
	1384	// Next, the filename from 'content-disposition: attachment' takes precedence.
	1385	if (head.content_disposition) \|cd_header\| {
	1386	break :ft FileType.fromContentDisposition(cd_header) orelse {
	1387	return f.fail(f.location_tok, try eb.printString(
	1388	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
	1389	.{cd_header},
	1390	));
	1391	};
	1392	}
	1393
	1394	// Finally, the path from the URI is used.
	1395	break :ft FileType.fromPath(uri_path) orelse {
	1396	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path}));
	1397	};
	1398	},
	1399
	1400	.git => .git_pack,
	1401
	1402	.dir => \|dir\| {
	1403	f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
	1404	return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{
	1405	uri_path, err,
	1406	}));
	1407	};
	1408	return .{};
	1409	},
	1410	};
	1411
	1412	switch (file_type) {
	1413	.tar => {
	1414	return unpackTarball(f, tmp_directory.handle, resource.reader());
	1415	},
	1416	.@"tar.gz" => {
	1417	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	1418	var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer);
	1419	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
	1420	},
	1421	.@"tar.xz" => {
	1422	const gpa = f.arena.child_allocator;
	1423	var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch \|err\|
	1424	return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
	1425	defer decompress.deinit();
	1426	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
	1427	},
	1428	.@"tar.zst" => {
	1429	const window_len = std.compress.zstd.default_window_len;
	1430	const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max);
	1431	var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
	1432	.verify_checksum = false,
	1433	.window_len = window_len,
	1434	});
	1435	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
	1436	},
	1437	.git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch \|err\| switch (err) {
	1438	error.FetchFailed, error.OutOfMemory => \|e\| return e,
	1439	else => \|e\| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})),
	1440	},
	1441	.zip => return unzip(f, tmp_directory.handle, resource.reader()) catch \|err\| switch (err) {
	1442	error.ReadFailed => return f.fail(f.location_tok, try eb.printString(
	1443	"failed reading resource: {t}",
	1444	.{err},
	1445	)),
	1446	else => \|e\| return e,
	1447	},
	1448	}
	1449	}
	1450
	1451	fn unpackTarball(f: Fetch, out_dir: Io.Dir, reader: Io.Reader) RunError!UnpackResult {
	1452	const eb = &f.error_bundle;
	1453	const arena = f.arena.allocator();
	1454	const io = f.job_queue.io;
	1455
	1456	var diagnostics: std.tar.Diagnostics = .{ .allocator = arena };
	1457
	1458	std.tar.pipeToFileSystem(io, out_dir, reader, .{
	1459	.diagnostics = &diagnostics,
	1460	.strip_components = 0,
	1461	.mode_mode = .ignore,
	1462	.exclude_empty_directories = true,
	1463	}) catch \|err\| return f.fail(
	1464	f.location_tok,
	1465	try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
	1466	);
	1467
	1468	var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
	1469	if (diagnostics.errors.items.len > 0) {
	1470	try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball");
	1471	for (diagnostics.errors.items) \|item\| {
	1472	switch (item) {
	1473	.unable_to_create_file => \|i\| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code),
	1474	.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code),
	1475	.unsupported_file_type => \|i\| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)),
	1476	.components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0
	1477	}
	1478	}
	1479	}
	1480	return res;
	1481	}
	1482
	1483	fn unzip(
	1484	f: *Fetch,
	1485	out_dir: Io.Dir,
	1486	reader: *Io.Reader,
	1487	) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult {
	1488	// We write the entire contents to a file first because zip files
	1489	// must be processed back to front and they could be too large to
	1490	// load into memory.
	1491
	1492	const io = f.job_queue.io;
	1493	const cache_root = f.job_queue.global_cache;
	1494	const prefix = "tmp/";
	1495	const suffix = ".zip";
	1496	const eb = &f.error_bundle;
	1497	const random_len = @sizeOf(u64) * 2;
	1498
	1499	var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined;
	1500	zip_path[0..prefix.len].* = prefix.*;
	1501	zip_path[prefix.len + random_len ..].* = suffix.*;
	1502
	1503	var zip_file = while (true) {
	1504	const random_integer = r: {
	1505	var x: u64 = undefined;
	1506	io.random(@ptrCast(&x));
	1507	break :r x;
	1508	};
	1509	zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer);
	1510
	1511	break cache_root.handle.createFile(io, &zip_path, .{
	1512	.exclusive = true,
	1513	.read = true,
	1514	}) catch \|err\| switch (err) {
	1515	error.PathAlreadyExists => continue,
	1516	error.FileNotFound => {
	1517	cache_root.handle.createDir(io, prefix, .default_dir) catch \|dir_err\| switch (dir_err) {
	1518	error.Canceled => \|e\| return e,
	1519	// error.PathAlreadyExists is considered a failure here because
	1520	// it implies that the prefix is not a directory.
	1521	else => \|e\| return f.fail(
	1522	f.location_tok,
	1523	try eb.printString("failed to create temporary directory: {t}", .{e}),
	1524	),
	1525	};
	1526	continue;
	1527	},
	1528	error.Canceled => \|e\| return e,
	1529	else => \|e\| return f.fail(
	1530	f.location_tok,
	1531	try eb.printString("failed to create temporary zip file: {t}", .{e}),
	1532	),
	1533	};
	1534	};
	1535	defer zip_file.close(io);
	1536	var zip_file_buffer: [4096]u8 = undefined;
	1537	var zip_file_reader = b: {
	1538	var zip_file_writer = zip_file.writer(io, &zip_file_buffer);
	1539
	1540	_ = reader.streamRemaining(&zip_file_writer.interface) catch \|err\| switch (err) {
	1541	error.ReadFailed => \|e\| return e,
	1542	error.WriteFailed => return f.fail(
	1543	f.location_tok,
	1544	try eb.printString("failed writing temporary zip file: {t}", .{err}),
	1545	),
	1546	};
	1547	zip_file_writer.interface.flush() catch \|err\| return f.fail(
	1548	f.location_tok,
	1549	try eb.printString("failed writing temporary zip file: {t}", .{err}),
	1550	);
	1551	break :b zip_file_writer.moveToReader();
	1552	};
	1553
	1554	var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
	1555	// no need to deinit since we are using an arena allocator
	1556
	1557	zip_file_reader.seekTo(0) catch \|err\|
	1558	return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err}));
	1559	std.zip.extract(out_dir, &zip_file_reader, .{
	1560	.allow_backslashes = true,
	1561	.diagnostics = &diagnostics,
	1562	}) catch \|err\| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err}));
	1563
	1564	cache_root.handle.deleteFile(io, &zip_path) catch \|err\|
	1565	return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err}));
	1566
	1567	return .{ .root_dir = diagnostics.root_dir };
	1568	}
	1569
	1570	fn unpackGitPack(f: Fetch, out_dir: Io.Dir, resource: Resource.Git) anyerror!UnpackResult {
	1571	const io = f.job_queue.io;
	1572	const arena = f.arena.allocator();
	1573	// TODO don't try to get a gpa from an arena. expose this dependency higher up
	1574	// because the backing of arena could be page allocator
	1575	const gpa = f.arena.child_allocator;
	1576	const object_format: git.Oid.Format = resource.want_oid;
	1577
	1578	var res: UnpackResult = .{};
	1579	// The .git directory is used to store the packfile and associated index, but
	1580	// we do not attempt to replicate the exact structure of a real .git
	1581	// directory, since that isn't relevant for fetching a package.
	1582	{
	1583	var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{});
	1584	defer pack_dir.close(io);
	1585	var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true });
	1586	defer pack_file.close(io);
	1587	var pack_file_buffer: [4096]u8 = undefined;
	1588	var pack_file_reader = b: {
	1589	var pack_file_writer = pack_file.writer(io, &pack_file_buffer);
	1590	const fetch_reader = &resource.fetch_stream.reader;
	1591	_ = try fetch_reader.streamRemaining(&pack_file_writer.interface);
	1592	try pack_file_writer.interface.flush();
	1593	break :b pack_file_writer.moveToReader();
	1594	};
	1595
	1596	var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true });
	1597	defer index_file.close(io);
	1598	var index_file_buffer: [2000]u8 = undefined;
	1599	var index_file_writer = index_file.writer(io, &index_file_buffer);
	1600	{
	1601	const index_prog_node = f.prog_node.start("Index pack", 0);
	1602	defer index_prog_node.end();
	1603	try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
	1604	}
	1605
	1606	{
	1607	var index_file_reader = index_file.reader(io, &index_file_buffer);
	1608	const checkout_prog_node = f.prog_node.start("Checkout", 0);
	1609	defer checkout_prog_node.end();
	1610	var repository: git.Repository = undefined;
	1611	try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
	1612	defer repository.deinit();
	1613	var diagnostics: git.Diagnostics = .{ .allocator = arena };
	1614	try repository.checkout(io, out_dir, resource.want_oid, &diagnostics);
	1615
	1616	if (diagnostics.errors.items.len > 0) {
	1617	try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");
	1618	for (diagnostics.errors.items) \|item\| {
	1619	switch (item) {
	1620	.unable_to_create_file => \|i\| res.unableToCreateFile(i.file_name, i.code),
	1621	.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(i.file_name, i.link_name, i.code),
	1622	}
	1623	}
	1624	}
	1625	}
	1626	}
	1627
	1628	try out_dir.deleteTree(io, ".git");
	1629	return res;
	1630	}
	1631
	1632	fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void {
	1633	const gpa = f.arena.child_allocator;
	1634	const io = f.job_queue.io;
	1635	// Recursive directory copy.
	1636	var it = try dir.walk(gpa);
	1637	defer it.deinit();
	1638	while (try it.next(io)) \|entry\| {
	1639	switch (entry.kind) {
	1640	.directory => {}, // omit empty directories
	1641	.file => {
	1642	dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch \|err\| switch (err) {
	1643	error.FileNotFound => {
	1644	if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
	1645	try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{});
	1646	},
	1647	else => \|e\| return e,
	1648	};
	1649	},
	1650	.sym_link => {
	1651	var buf: [fs.max_path_bytes]u8 = undefined;
	1652	const link_name = buf[0..try dir.readLink(io, entry.path, &buf)];
	1653	// TODO: if this would create a symlink to outside
	1654	// the destination directory, fail with an error instead.
	1655	tmp_dir.symLink(io, link_name, entry.path, .{}) catch \|err\| switch (err) {
	1656	error.FileNotFound => {
	1657	if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
	1658	try tmp_dir.symLink(io, link_name, entry.path, .{});
	1659	},
	1660	else => \|e\| return e,
	1661	};
	1662	},
	1663	else => return error.IllegalFileTypeInPackage,
	1664	}
	1665	}
	1666	}
	1667
	1668	pub fn renameTmpIntoCache(io: Io, tmp_path: Path, dest_path: Path) !void {
	1669	var handled_missing_dir = false;
	1670	while (true) {
	1671	Io.Dir.rename(
	1672	tmp_path.root_dir.handle,
	1673	tmp_path.sub_path,
	1674	dest_path.root_dir.handle,
	1675	dest_path.sub_path,
	1676	io,
	1677	) catch \|err\| switch (err) {
	1678	error.FileNotFound => {
	1679	if (handled_missing_dir) return err;
	1680	const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
	1681	dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch \|er\| switch (er) {
	1682	error.PathAlreadyExists => handled_missing_dir = true,
	1683	else => \|e\| return e,
	1684	};
	1685	continue;
	1686	},
	1687	error.DirNotEmpty, error.AccessDenied => {
	1688	// Package has been already downloaded and may already be in use on the system.
	1689	tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch \|er\| switch (er) {
	1690	error.Canceled => \|e\| return e,
	1691	// Garbage files leftover in zig-cache/tmp/ is, as they say
	1692	// on Star Trek, "operating within normal parameters".
	1693	else => \|e\| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
	1694	};
	1695	},
	1696	else => \|e\| return e,
	1697	};
	1698	break;
	1699	}
	1700	}
	1701
	1702	const ComputedHash = struct {
	1703	digest: Package.Hash.Digest,
	1704	total_size: u64,
	1705	};
	1706
	1707	/// Assumes that files not included in the package have already been filtered
	1708	/// prior to calling this function. This ensures that files not protected by
	1709	/// the hash are not present on the file system. Empty directories are *not
	1710	/// hashed* and must not be present on the file system when calling this
	1711	/// function.
	1712	fn computeHash(f: *Fetch, pkg_path: Path, filter: Filter) RunError!ComputedHash {
	1713	const io = f.job_queue.io;
	1714	// All the path name strings need to be in memory for sorting.
	1715	const arena = f.arena.allocator();
	1716	const gpa = f.arena.child_allocator;
	1717	const eb = &f.error_bundle;
	1718	const root_dir = pkg_path.root_dir.handle;
	1719
	1720	// Collect all files, recursively, then sort.
	1721	var all_files = std.array_list.Managed(*HashedFile).init(gpa);
	1722	defer all_files.deinit();
	1723
	1724	var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa);
	1725	defer deleted_files.deinit();
	1726
	1727	// Track directories which had any files deleted from them so that empty directories
	1728	// can be deleted.
	1729	var sus_dirs: std.array_hash_map.String(void) = .empty;
	1730	defer sus_dirs.deinit(gpa);
	1731
	1732	var walker = try root_dir.walk(gpa);
	1733	defer walker.deinit();
	1734
	1735	// Total number of bytes of file contents included in the package.
	1736	var total_size: u64 = 0;
	1737
	1738	{
	1739	// The final hash will be a hash of each file hashed independently. This
	1740	// allows hashing in parallel.
	1741	var group: Io.Group = .init;
	1742	defer group.cancel(io);
	1743
	1744	while (walker.next(io) catch \|err\| {
	1745	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1746	"unable to walk temporary directory '{f}': {t}",
	1747	.{ pkg_path, err },
	1748	) });
	1749	return error.FetchFailed;
	1750	}) \|entry\| {
	1751	if (entry.kind == .directory) continue;
	1752
	1753	const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
	1754	if (!filter.includePath(entry_pkg_path)) {
	1755	// Delete instead of including in hash calculation.
	1756	const fs_path = try arena.dupe(u8, entry.path);
	1757
	1758	// Also track the parent directory in case it becomes empty.
	1759	if (fs.path.dirname(fs_path)) \|parent\|
	1760	try sus_dirs.put(gpa, parent, {});
	1761
	1762	const deleted_file = try arena.create(DeletedFile);
	1763	deleted_file.* = .{
	1764	.fs_path = fs_path,
	1765	.failure = undefined, // to be populated by the worker
	1766	};
	1767	group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file });
	1768	try deleted_files.append(deleted_file);
	1769	continue;
	1770	}
	1771
	1772	const kind: HashedFile.Kind = switch (entry.kind) {
	1773	.directory => unreachable,
	1774	.file => .file,
	1775	.sym_link => .link,
	1776	else => return f.fail(f.location_tok, try eb.printString(
	1777	"package contains '{s}' which has illegal file type '{t}'",
	1778	.{ entry.path, entry.kind },
	1779	)),
	1780	};
	1781
	1782	if (std.mem.eql(u8, entry_pkg_path, std.zig.build_zig_basename))
	1783	f.has_build_zig = true;
	1784
	1785	const fs_path = try arena.dupe(u8, entry.path);
	1786	const hashed_file = try arena.create(HashedFile);
	1787	hashed_file.* = .{
	1788	.fs_path = fs_path,
	1789	.normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
	1790	.kind = kind,
	1791	.hash = undefined, // to be populated by the worker
	1792	.failure = undefined, // to be populated by the worker
	1793	.size = undefined, // to be populated by the worker
	1794	};
	1795	group.async(io, workerHashFile, .{ io, root_dir, hashed_file });
	1796	try all_files.append(hashed_file);
	1797	}
	1798
	1799	try group.await(io);
	1800	}
	1801
	1802	{
	1803	// Sort by length, descending, so that child directories get removed first.
	1804	sus_dirs.sortUnstable(@as(struct {
	1805	keys: []const []const u8,
	1806	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
	1807	return ctx.keys[b_index].len < ctx.keys[a_index].len;
	1808	}
	1809	}, .{ .keys = sus_dirs.keys() }));
	1810
	1811	// During this loop, more entries will be added, so we must loop by index.
	1812	var i: usize = 0;
	1813	while (i < sus_dirs.count()) : (i += 1) {
	1814	const sus_dir = sus_dirs.keys()[i];
	1815	root_dir.deleteDir(io, sus_dir) catch \|err\| switch (err) {
	1816	error.DirNotEmpty => continue,
	1817	error.FileNotFound => continue,
	1818	else => \|e\| {
	1819	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1820	"unable to delete empty directory '{s}': {s}",
	1821	.{ sus_dir, @errorName(e) },
	1822	) });
	1823	return error.FetchFailed;
	1824	},
	1825	};
	1826	if (fs.path.dirname(sus_dir)) \|parent\| {
	1827	try sus_dirs.put(gpa, parent, {});
	1828	}
	1829	}
	1830	}
	1831
	1832	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
	1833
	1834	var hasher = Package.Hash.Algo.init(.{});
	1835	var any_failures = false;
	1836	for (all_files.items) \|hashed_file\| {
	1837	hashed_file.failure catch \|err\| {
	1838	any_failures = true;
	1839	try eb.addRootErrorMessage(.{
	1840	.msg = try eb.printString("unable to hash '{s}': {s}", .{
	1841	hashed_file.fs_path, @errorName(err),
	1842	}),
	1843	});
	1844	};
	1845	hasher.update(&hashed_file.hash);
	1846	total_size += hashed_file.size;
	1847	}
	1848	for (deleted_files.items) \|deleted_file\| {
	1849	deleted_file.failure catch \|err\| {
	1850	any_failures = true;
	1851	try eb.addRootErrorMessage(.{
	1852	.msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
	1853	deleted_file.fs_path, @errorName(err),
	1854	}),
	1855	});
	1856	};
	1857	}
	1858
	1859	if (any_failures) return error.FetchFailed;
	1860
	1861	if (f.job_queue.debug_hash) {
	1862	assert(!f.job_queue.recursive);
	1863	// Print something to stdout that can be text diffed to figure out why
	1864	// the package hash is different.
	1865	dumpHashInfo(io, all_files.items) catch \|err\|
	1866	std.process.fatal("unable to write to stdout: {t}", .{err});
	1867	}
	1868
	1869	return .{
	1870	.digest = hasher.finalResult(),
	1871	.total_size = total_size,
	1872	};
	1873	}
	1874
	1875	fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void {
	1876	var stdout_buffer: [1024]u8 = undefined;
	1877	var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer);
	1878	dumpHashInfoWriter(&stdout_writer.interface, all_files) catch \|err\| switch (err) {
	1879	error.WriteFailed => return stdout_writer.err.?,
	1880	};
	1881	try stdout_writer.flush();
	1882	}
	1883
	1884	fn dumpHashInfoWriter(w: Io.Writer, all_files: []const const HashedFile) Io.Writer.Error!void {
	1885	for (all_files) \|hashed_file\| {
	1886	try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path });
	1887	}
	1888	}
	1889
	1890	fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void {
	1891	hashed_file.failure = hashFileFallible(io, dir, hashed_file);
	1892	}
	1893
	1894	fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void {
	1895	deleted_file.failure = deleteFileFallible(io, dir, deleted_file);
	1896	}
	1897
	1898	fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
	1899	var buf: [8000]u8 = undefined;
	1900	var hasher = Package.Hash.Algo.init(.{});
	1901	hasher.update(hashed_file.normalized_path);
	1902	var file_size: u64 = 0;
	1903
	1904	switch (hashed_file.kind) {
	1905	.file => {
	1906	var file = try dir.openFile(io, hashed_file.fs_path, .{});
	1907	defer file.close(io);
	1908	// Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
	1909	hasher.update(&.{ 0, 0 });
	1910	var file_header: FileHeader = .{};
	1911	while (true) {
	1912	const bytes_read = try file.readPositional(io, &.{&buf}, file_size);
	1913	if (bytes_read == 0) break;
	1914	file_size += bytes_read;
	1915	hasher.update(buf[0..bytes_read]);
	1916	file_header.update(buf[0..bytes_read]);
	1917	}
	1918	if (file_header.isExecutable()) {
	1919	try setExecutable(io, file);
	1920	}
	1921	},
	1922	.link => {
	1923	const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)];
	1924	if (fs.path.sep != canonical_sep) {
	1925	// Package hashes are intended to be consistent across
	1926	// platforms which means we must normalize path separators
	1927	// inside symlinks.
	1928	normalizePath(link_name);
	1929	}
	1930	hasher.update(link_name);
	1931	},
	1932	}
	1933	hasher.final(&hashed_file.hash);
	1934	hashed_file.size = file_size;
	1935	}
	1936
	1937	fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
	1938	try dir.deleteFile(io, deleted_file.fs_path);
	1939	}
	1940
	1941	fn setExecutable(io: Io, file: Io.File) !void {
	1942	if (!Io.File.Permissions.has_executable_bit) return;
	1943	try file.setPermissions(io, .executable_file);
	1944	}
	1945
	1946	const DeletedFile = struct {
	1947	fs_path: []const u8,
	1948	failure: Error!void,
	1949
	1950	const Error =
	1951	Io.Dir.DeleteFileError \|\|
	1952	Io.Dir.DeleteDirError;
	1953	};
	1954
	1955	const HashedFile = struct {
	1956	fs_path: []const u8,
	1957	normalized_path: []const u8,
	1958	hash: Package.Hash.Digest,
	1959	failure: Error!void,
	1960	kind: Kind,
	1961	size: u64,
	1962
	1963	const Error =
	1964	Io.File.OpenError \|\|
	1965	Io.File.ReadPositionalError \|\|
	1966	Io.File.StatError \|\|
	1967	Io.File.SetPermissionsError \|\|
	1968	Io.Dir.ReadLinkError;
	1969
	1970	const Kind = enum { file, link };
	1971
	1972	fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
	1973	_ = context;
	1974	return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
	1975	}
	1976	};
	1977
	1978	/// Strips root directory name from file system path.
	1979	fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
	1980	if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
	1981
	1982	if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) {
	1983	return fs_path[root_dir.len + 1 ..];
	1984	}
	1985
	1986	return fs_path;
	1987	}
	1988
	1989	/// Make a file system path identical independently of operating system path inconsistencies.
	1990	/// This converts backslashes into forward slashes.
	1991	fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
	1992	const normalized = try arena.dupe(u8, pkg_path);
	1993	if (fs.path.sep == canonical_sep) return normalized;
	1994	normalizePath(normalized);
	1995	return normalized;
	1996	}
	1997
	1998	const canonical_sep = fs.path.sep_posix;
	1999
	2000	fn normalizePath(bytes: []u8) void {
	2001	assert(fs.path.sep != canonical_sep);
	2002	std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
	2003	}
	2004
	2005	const Filter = struct {
	2006	include_paths: std.array_hash_map.String(void) = .empty,
	2007
	2008	/// sub_path is relative to the package root.
	2009	pub fn includePath(self: *const Filter, sub_path: []const u8) bool {
	2010	if (self.include_paths.count() == 0) return true;
	2011	if (self.include_paths.contains("")) return true;
	2012	if (self.include_paths.contains(".")) return true;
	2013	if (self.include_paths.contains(sub_path)) return true;
	2014
	2015	// Check if any included paths are parent directories of sub_path.
	2016	var dirname = sub_path;
	2017	while (std.fs.path.dirname(dirname)) \|next_dirname\| {
	2018	if (self.include_paths.contains(next_dirname)) return true;
	2019	dirname = next_dirname;
	2020	}
	2021
	2022	return false;
	2023	}
	2024
	2025	test includePath {
	2026	const gpa = std.testing.allocator;
	2027	var filter: Filter = .{};
	2028	defer filter.include_paths.deinit(gpa);
	2029
	2030	try filter.include_paths.put(gpa, "src", {});
	2031	try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c"));
	2032	try std.testing.expect(!filter.includePath(".gitignore"));
	2033	}
	2034	};
	2035
	2036	pub fn depDigest(pkg_root: Path, cache_root: Directory, dep: Manifest.Dependency) ?Package.Hash {
	2037	if (dep.hash) \|h\| return .fromSlice(h);
	2038
	2039	switch (dep.location) {
	2040	.url => return null,
	2041	.path => \|rel_path\| {
	2042	var buf: [fs.max_path_bytes]u8 = undefined;
	2043	var fba = std.heap.FixedBufferAllocator.init(&buf);
	2044	const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch
	2045	return null;
	2046	return relativePathDigest(new_root, cache_root);
	2047	},
	2048	}
	2049	}
	2050
	2051	// Detects executable header: ELF or Macho-O magic header or shebang line.
	2052	const FileHeader = struct {
	2053	header: [4]u8 = undefined,
	2054	bytes_read: usize = 0,
	2055
	2056	pub fn update(self: *FileHeader, buf: []const u8) void {
	2057	if (self.bytes_read >= self.header.len) return;
	2058	const n = @min(self.header.len - self.bytes_read, buf.len);
	2059	@memcpy(self.header[self.bytes_read..][0..n], buf[0..n]);
	2060	self.bytes_read += n;
	2061	}
	2062
	2063	fn isScript(self: *FileHeader) bool {
	2064	const shebang = "#!";
	2065	return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang);
	2066	}
	2067
	2068	fn isElf(self: *FileHeader) bool {
	2069	const elf_magic = std.elf.MAGIC;
	2070	return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic);
	2071	}
	2072
	2073	fn isMachO(self: *FileHeader) bool {
	2074	if (self.bytes_read < 4) return false;
	2075	const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian());
	2076	return magic_number == std.macho.MH_MAGIC or
	2077	magic_number == std.macho.MH_MAGIC_64 or
	2078	magic_number == std.macho.FAT_MAGIC or
	2079	magic_number == std.macho.FAT_MAGIC_64 or
	2080	magic_number == std.macho.MH_CIGAM or
	2081	magic_number == std.macho.MH_CIGAM_64 or
	2082	magic_number == std.macho.FAT_CIGAM or
	2083	magic_number == std.macho.FAT_CIGAM_64;
	2084	}
	2085
	2086	pub fn isExecutable(self: *FileHeader) bool {
	2087	return self.isScript() or self.isElf() or self.isMachO();
	2088	}
	2089	};
	2090
	2091	test FileHeader {
	2092	var h: FileHeader = .{};
	2093	try std.testing.expect(!h.isExecutable());
	2094
	2095	const elf_magic = std.elf.MAGIC;
	2096	h.update(elf_magic[0..2]);
	2097	try std.testing.expect(!h.isExecutable());
	2098	h.update(elf_magic[2..4]);
	2099	try std.testing.expect(h.isExecutable());
	2100
	2101	h.update(elf_magic[2..4]);
	2102	try std.testing.expect(h.isExecutable());
	2103
	2104	const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE };
	2105	h.bytes_read = 0;
	2106	h.update(&macho64_magic_bytes);
	2107	try std.testing.expect(h.isExecutable());
	2108
	2109	const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF };
	2110	h.bytes_read = 0;
	2111	h.update(&macho64_cigam_bytes);
	2112	try std.testing.expect(h.isExecutable());
	2113	}
	2114
	2115	// Result of the `unpackResource` operation. Enables collecting errors from
	2116	// tar/git diagnostic, filtering that errors by manifest inclusion rules and
	2117	// emitting remaining errors to an `ErrorBundle`.
	2118	const UnpackResult = struct {
	2119	errors: []Error = undefined,
	2120	errors_count: usize = 0,
	2121	root_error_message: []const u8 = "",
	2122
	2123	// A non empty value means that the package contents are inside a
	2124	// sub-directory indicated by the named path.
	2125	root_dir: []const u8 = "",
	2126
	2127	const Error = union(enum) {
	2128	unable_to_create_sym_link: struct {
	2129	code: anyerror,
	2130	file_name: []const u8,
	2131	link_name: []const u8,
	2132	},
	2133	unable_to_create_file: struct {
	2134	code: anyerror,
	2135	file_name: []const u8,
	2136	},
	2137	unsupported_file_type: struct {
	2138	file_name: []const u8,
	2139	file_type: u8,
	2140	},
	2141
	2142	fn excluded(self: Error, filter: Filter) bool {
	2143	const file_name = switch (self) {
	2144	.unable_to_create_file => \|info\| info.file_name,
	2145	.unable_to_create_sym_link => \|info\| info.file_name,
	2146	.unsupported_file_type => \|info\| info.file_name,
	2147	};
	2148	return !filter.includePath(file_name);
	2149	}
	2150	};
	2151
	2152	fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void {
	2153	self.root_error_message = try arena.dupe(u8, root_error_message);
	2154	self.errors = try arena.alloc(UnpackResult.Error, n);
	2155	}
	2156
	2157	fn hasErrors(self: *UnpackResult) bool {
	2158	return self.errors_count > 0;
	2159	}
	2160
	2161	fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void {
	2162	self.errors[self.errors_count] = .{ .unable_to_create_file = .{
	2163	.code = err,
	2164	.file_name = file_name,
	2165	} };
	2166	self.errors_count += 1;
	2167	}
	2168
	2169	fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void {
	2170	self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{
	2171	.code = err,
	2172	.file_name = file_name,
	2173	.link_name = link_name,
	2174	} };
	2175	self.errors_count += 1;
	2176	}
	2177
	2178	fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void {
	2179	self.errors[self.errors_count] = .{ .unsupported_file_type = .{
	2180	.file_name = file_name,
	2181	.file_type = file_type,
	2182	} };
	2183	self.errors_count += 1;
	2184	}
	2185
	2186	fn validate(self: UnpackResult, f: Fetch, filter: Filter) !void {
	2187	if (self.errors_count == 0) return;
	2188
	2189	var unfiltered_errors: u32 = 0;
	2190	for (self.errors) \|item\| {
	2191	if (item.excluded(filter)) continue;
	2192	unfiltered_errors += 1;
	2193	}
	2194	if (unfiltered_errors == 0) return;
	2195
	2196	// Emmit errors to an `ErrorBundle`.
	2197	const eb = &f.error_bundle;
	2198	try eb.addRootErrorMessage(.{
	2199	.msg = try eb.addString(self.root_error_message),
	2200	.src_loc = try f.srcLoc(f.location_tok),
	2201	.notes_len = unfiltered_errors,
	2202	});
	2203	var note_i: u32 = try eb.reserveNotes(unfiltered_errors);
	2204	for (self.errors) \|item\| {
	2205	if (item.excluded(filter)) continue;
	2206	switch (item) {
	2207	.unable_to_create_sym_link => \|info\| {
	2208	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	2209	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
	2210	info.file_name, info.link_name, @errorName(info.code),
	2211	}),
	2212	}));
	2213	},
	2214	.unable_to_create_file => \|info\| {
	2215	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	2216	.msg = try eb.printString("unable to create file '{s}': {s}", .{
	2217	info.file_name, @errorName(info.code),
	2218	}),
	2219	}));
	2220	},
	2221	.unsupported_file_type => \|info\| {
	2222	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	2223	.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
	2224	info.file_name, info.file_type,
	2225	}),
	2226	}));
	2227	},
	2228	}
	2229	note_i += 1;
	2230	}
	2231
	2232	return error.FetchFailed;
	2233	}
	2234
	2235	test validate {
	2236	const gpa = std.testing.allocator;
	2237	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	2238	defer arena_instance.deinit();
	2239	const arena = arena_instance.allocator();
	2240
	2241	// fill UnpackResult with errors
	2242	var res: UnpackResult = .{};
	2243	try res.allocErrors(arena, 4, "unable to unpack");
	2244	try std.testing.expectEqual(0, res.errors_count);
	2245	res.unableToCreateFile("dir1/file1", error.File1);
	2246	res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError);
	2247	res.unableToCreateFile("dir1/file3", error.File3);
	2248	res.unsupportedFileType("dir2/file4", 'x');
	2249	try std.testing.expectEqual(4, res.errors_count);
	2250
	2251	// create filter, includes dir2, excludes dir1
	2252	var filter: Filter = .{};
	2253	try filter.include_paths.put(arena, "dir2", {});
	2254
	2255	// init Fetch
	2256	var fetch: Fetch = undefined;
	2257	fetch.parent_manifest_ast = null;
	2258	fetch.location_tok = 0;
	2259	try fetch.error_bundle.init(gpa);
	2260	defer fetch.error_bundle.deinit();
	2261
	2262	// validate errors with filter
	2263	try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter));
	2264
	2265	// output errors to string
	2266	var errors = try fetch.error_bundle.toOwnedBundle("");
	2267	defer errors.deinit(gpa);
	2268	var aw: Io.Writer.Allocating = .init(gpa);
	2269	defer aw.deinit();
	2270	try errors.renderToWriter(.{}, &aw.writer);
	2271	try std.testing.expectEqualStrings(
	2272	\\error: unable to unpack
	2273	\\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError
	2274	\\ note: file 'dir2/file4' has unsupported type 'x'
	2275	\\
	2276	, aw.written());
	2277	}
	2278	};
	2279
	2280	test {
	2281	_ = Filter;
	2282	_ = FileType;
	2283	_ = UnpackResult;
	2284	}

lib/compiler/Maker/Fetch/git.zig created+1750

...	...	@@ -0,0 +1,1750 @@
	1	//! Git support for package fetching.
	2	//!
	3	//! This is not intended to support all features of Git: it is limited to the
	4	//! basic functionality needed to clone a repository for the purpose of fetching
	5	//! a package.
	6
	7	const std = @import("std");
	8	const Io = std.Io;
	9	const mem = std.mem;
	10	const testing = std.testing;
	11	const Allocator = mem.Allocator;
	12	const Sha1 = std.crypto.hash.Sha1;
	13	const Sha256 = std.crypto.hash.sha2.Sha256;
	14	const assert = std.debug.assert;
	15
	16	/// The ID of a Git object.
	17	pub const Oid = union(Format) {
	18	sha1: [Sha1.digest_length]u8,
	19	sha256: [Sha256.digest_length]u8,
	20
	21	pub const max_formatted_length = len: {
	22	var max: usize = 0;
	23	for (std.enums.values(Format)) \|f\| {
	24	max = @max(max, f.formattedLength());
	25	}
	26	break :len max;
	27	};
	28
	29	pub const Format = enum {
	30	sha1,
	31	sha256,
	32
	33	pub fn byteLength(f: Format) usize {
	34	return switch (f) {
	35	.sha1 => Sha1.digest_length,
	36	.sha256 => Sha256.digest_length,
	37	};
	38	}
	39
	40	pub fn formattedLength(f: Format) usize {
	41	return 2 * f.byteLength();
	42	}
	43	};
	44
	45	const Hasher = union(Format) {
	46	sha1: Sha1,
	47	sha256: Sha256,
	48
	49	fn init(oid_format: Format) Hasher {
	50	return switch (oid_format) {
	51	.sha1 => .{ .sha1 = Sha1.init(.{}) },
	52	.sha256 => .{ .sha256 = Sha256.init(.{}) },
	53	};
	54	}
	55
	56	// Must be public for use from HashedReader and HashedWriter.
	57	pub fn update(hasher: *Hasher, b: []const u8) void {
	58	switch (hasher.*) {
	59	inline else => \|*inner\| inner.update(b),
	60	}
	61	}
	62
	63	fn finalResult(hasher: *Hasher) Oid {
	64	return switch (hasher.*) {
	65	inline else => \|*inner, tag\| @unionInit(Oid, @tagName(tag), inner.finalResult()),
	66	};
	67	}
	68	};
	69
	70	const Hashing = union(Format) {
	71	sha1: Io.Writer.Hashing(Sha1),
	72	sha256: Io.Writer.Hashing(Sha256),
	73
	74	fn init(oid_format: Format, buffer: []u8) Hashing {
	75	return switch (oid_format) {
	76	.sha1 => .{ .sha1 = .init(buffer) },
	77	.sha256 => .{ .sha256 = .init(buffer) },
	78	};
	79	}
	80
	81	fn writer(h: @This()) Io.Writer {
	82	return switch (h.*) {
	83	inline else => \|*inner\| &inner.writer,
	84	};
	85	}
	86
	87	fn final(h: *@This()) Oid {
	88	switch (h.*) {
	89	inline else => \|*inner, tag\| {
	90	inner.writer.flush() catch unreachable; // hashers cannot fail
	91	return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult());
	92	},
	93	}
	94	}
	95	};
	96
	97	pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
	98	assert(bytes.len == oid_format.byteLength());
	99	return switch (oid_format) {
	100	inline else => \|tag\| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*),
	101	};
	102	}
	103
	104	pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid {
	105	return switch (oid_format) {
	106	inline else => \|tag\| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*),
	107	};
	108	}
	109
	110	pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid {
	111	switch (oid_format) {
	112	inline else => \|tag\| {
	113	if (s.len != tag.formattedLength()) return error.InvalidOid;
	114	var bytes: [tag.byteLength()]u8 = undefined;
	115	for (&bytes, 0..) \|*b, i\| {
	116	b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
	117	}
	118	return @unionInit(Oid, @tagName(tag), bytes);
	119	},
	120	}
	121	}
	122
	123	test parse {
	124	try testing.expectEqualSlices(
	125	u8,
	126	&.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
	127	&(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1,
	128	);
	129	try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588"));
	130	try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"));
	131	try testing.expectEqualSlices(
	132	u8,
	133	&.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A },
	134	&(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256,
	135	);
	136	try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf"));
	137	try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf"));
	138	try testing.expectError(error.InvalidOid, parse(.sha1, "master"));
	139	try testing.expectError(error.InvalidOid, parse(.sha256, "master"));
	140	try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD"));
	141	try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD"));
	142	}
	143
	144	pub fn parseAny(s: []const u8) error{InvalidOid}!Oid {
	145	return for (std.enums.values(Format)) \|f\| {
	146	if (s.len == f.formattedLength()) break parse(f, s);
	147	} else error.InvalidOid;
	148	}
	149
	150	pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void {
	151	try writer.print("{x}", .{oid.slice()});
	152	}
	153
	154	pub fn slice(oid: *const Oid) []const u8 {
	155	return switch (oid.*) {
	156	inline else => \|*bytes\| bytes,
	157	};
	158	}
	159	};
	160
	161	pub const Diagnostics = struct {
	162	allocator: Allocator,
	163	errors: std.ArrayList(Error) = .empty,
	164
	165	pub const Error = union(enum) {
	166	unable_to_create_sym_link: struct {
	167	code: anyerror,
	168	file_name: []const u8,
	169	link_name: []const u8,
	170	},
	171	unable_to_create_file: struct {
	172	code: anyerror,
	173	file_name: []const u8,
	174	},
	175	};
	176
	177	pub fn deinit(d: *Diagnostics) void {
	178	for (d.errors.items) \|item\| {
	179	switch (item) {
	180	.unable_to_create_sym_link => \|info\| {
	181	d.allocator.free(info.file_name);
	182	d.allocator.free(info.link_name);
	183	},
	184	.unable_to_create_file => \|info\| {
	185	d.allocator.free(info.file_name);
	186	},
	187	}
	188	}
	189	d.errors.deinit(d.allocator);
	190	d.* = undefined;
	191	}
	192	};
	193
	194	pub const Repository = struct {
	195	odb: Odb,
	196
	197	pub fn init(
	198	repo: *Repository,
	199	allocator: Allocator,
	200	format: Oid.Format,
	201	pack_file: *Io.File.Reader,
	202	index_file: *Io.File.Reader,
	203	) !void {
	204	repo.* = .{ .odb = undefined };
	205	try repo.odb.init(allocator, format, pack_file, index_file);
	206	}
	207
	208	pub fn deinit(repository: *Repository) void {
	209	repository.odb.deinit();
	210	repository.* = undefined;
	211	}
	212
	213	/// Checks out the repository at `commit_oid` to `worktree`.
	214	pub fn checkout(
	215	repository: *Repository,
	216	io: Io,
	217	worktree: Io.Dir,
	218	commit_oid: Oid,
	219	diagnostics: *Diagnostics,
	220	) !void {
	221	try repository.odb.seekOid(commit_oid);
	222	const tree_oid = tree_oid: {
	223	const commit_object = try repository.odb.readObject();
	224	if (commit_object.type != .commit) return error.NotACommit;
	225	break :tree_oid try getCommitTree(repository.odb.format, commit_object.data);
	226	};
	227	try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics);
	228	}
	229
	230	/// Checks out the tree at `tree_oid` to `worktree`.
	231	fn checkoutTree(
	232	repository: *Repository,
	233	io: Io,
	234	dir: Io.Dir,
	235	tree_oid: Oid,
	236	current_path: []const u8,
	237	diagnostics: *Diagnostics,
	238	) !void {
	239	try repository.odb.seekOid(tree_oid);
	240	const tree_object = try repository.odb.readObject();
	241	if (tree_object.type != .tree) return error.NotATree;
	242	// The tree object may be evicted from the object cache while we're
	243	// iterating over it, so we can make a defensive copy here to make sure
	244	// it remains valid until we're done with it
	245	const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
	246	defer repository.odb.allocator.free(tree_data);
	247
	248	var tree_iter: TreeIterator = .{
	249	.format = repository.odb.format,
	250	.data = tree_data,
	251	.pos = 0,
	252	};
	253	while (try tree_iter.next()) \|entry\| {
	254	switch (entry.type) {
	255	.directory => {
	256	try dir.createDir(io, entry.name, .default_dir);
	257	var subdir = try dir.openDir(io, entry.name, .{});
	258	defer subdir.close(io);
	259	const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
	260	defer repository.odb.allocator.free(sub_path);
	261	try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics);
	262	},
	263	.file => {
	264	try repository.odb.seekOid(entry.oid);
	265	const file_object = try repository.odb.readObject();
	266	if (file_object.type != .blob) return error.InvalidFile;
	267	var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch \|e\| {
	268	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
	269	errdefer diagnostics.allocator.free(file_name);
	270	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{
	271	.code = e,
	272	.file_name = file_name,
	273	} });
	274	continue;
	275	};
	276	defer file.close(io);
	277	try file.writePositionalAll(io, file_object.data, 0);
	278	},
	279	.symlink => {
	280	try repository.odb.seekOid(entry.oid);
	281	const symlink_object = try repository.odb.readObject();
	282	if (symlink_object.type != .blob) return error.InvalidFile;
	283	const link_name = symlink_object.data;
	284	dir.symLink(io, link_name, entry.name, .{}) catch \|e\| {
	285	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
	286	errdefer diagnostics.allocator.free(file_name);
	287	const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
	288	errdefer diagnostics.allocator.free(link_name_dup);
	289	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
	290	.code = e,
	291	.file_name = file_name,
	292	.link_name = link_name_dup,
	293	} });
	294	};
	295	},
	296	.gitlink => {
	297	// Consistent with git archive behavior, create the directory but
	298	// do nothing else
	299	try dir.createDir(io, entry.name, .default_dir);
	300	},
	301	}
	302	}
	303	}
	304
	305	/// Returns the ID of the tree associated with the given commit (provided as
	306	/// raw object data).
	307	fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid {
	308	if (!mem.startsWith(u8, commit_data, "tree ") or
	309	commit_data.len < "tree ".len + format.formattedLength() + "\n".len or
	310	commit_data["tree ".len + format.formattedLength()] != '\n')
	311	{
	312	return error.InvalidCommit;
	313	}
	314	return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]);
	315	}
	316
	317	const TreeIterator = struct {
	318	format: Oid.Format,
	319	data: []const u8,
	320	pos: usize,
	321
	322	const Entry = struct {
	323	type: Type,
	324	executable: bool,
	325	name: [:0]const u8,
	326	oid: Oid,
	327
	328	const Type = enum(u4) {
	329	directory = 0o4,
	330	file = 0o10,
	331	symlink = 0o12,
	332	gitlink = 0o16,
	333	};
	334	};
	335
	336	fn next(iterator: *TreeIterator) !?Entry {
	337	if (iterator.pos == iterator.data.len) return null;
	338
	339	const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
	340	const mode: packed struct {
	341	permission: u9,
	342	unused: u3,
	343	type: u4,
	344	} = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
	345	const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree;
	346	const executable = switch (mode.permission) {
	347	0 => if (@"type" == .file) return error.InvalidTree else false,
	348	0o644 => if (@"type" != .file) return error.InvalidTree else false,
	349	0o755 => if (@"type" != .file) return error.InvalidTree else true,
	350	else => return error.InvalidTree,
	351	};
	352	iterator.pos = mode_end + 1;
	353
	354	const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
	355	const name = iterator.data[iterator.pos..name_end :0];
	356	iterator.pos = name_end + 1;
	357
	358	const oid_length = iterator.format.byteLength();
	359	if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
	360	const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]);
	361	iterator.pos += oid_length;
	362
	363	return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
	364	}
	365	};
	366	};
	367
	368	/// A Git object database backed by a packfile. A packfile index is also used
	369	/// for efficient access to objects in the packfile.
	370	///
	371	/// The format of the packfile and its associated index are documented in
	372	/// [pack-format](https://git-scm.com/docs/pack-format).
	373	const Odb = struct {
	374	format: Oid.Format,
	375	pack_file: *Io.File.Reader,
	376	index_header: IndexHeader,
	377	index_file: *Io.File.Reader,
	378	cache: ObjectCache = .{},
	379	allocator: Allocator,
	380
	381	/// Initializes the database from open pack and index files.
	382	fn init(
	383	odb: *Odb,
	384	allocator: Allocator,
	385	format: Oid.Format,
	386	pack_file: *Io.File.Reader,
	387	index_file: *Io.File.Reader,
	388	) !void {
	389	try pack_file.seekTo(0);
	390	try index_file.seekTo(0);
	391	odb.* = .{
	392	.format = format,
	393	.pack_file = pack_file,
	394	.index_header = undefined,
	395	.index_file = index_file,
	396	.allocator = allocator,
	397	};
	398	try odb.index_header.read(&index_file.interface);
	399	}
	400
	401	fn deinit(odb: *Odb) void {
	402	odb.cache.deinit(odb.allocator);
	403	odb.* = undefined;
	404	}
	405
	406	/// Reads the object at the current position in the database.
	407	fn readObject(odb: *Odb) !Object {
	408	var base_offset = odb.pack_file.logicalPos();
	409	var base_header: EntryHeader = undefined;
	410	var delta_offsets: std.ArrayList(u64) = .empty;
	411	defer delta_offsets.deinit(odb.allocator);
	412	const base_object = while (true) {
	413	if (odb.cache.get(base_offset)) \|base_object\| break base_object;
	414
	415	base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface);
	416	switch (base_header) {
	417	.ofs_delta => \|ofs_delta\| {
	418	try delta_offsets.append(odb.allocator, base_offset);
	419	base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
	420	try odb.pack_file.seekTo(base_offset);
	421	},
	422	.ref_delta => \|ref_delta\| {
	423	try delta_offsets.append(odb.allocator, base_offset);
	424	try odb.seekOid(ref_delta.base_object);
	425	base_offset = odb.pack_file.logicalPos();
	426	},
	427	else => {
	428	const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
	429	errdefer odb.allocator.free(base_data);
	430	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
	431	try odb.cache.put(odb.allocator, base_offset, base_object);
	432	break base_object;
	433	},
	434	}
	435	};
	436
	437	const base_data = try resolveDeltaChain(
	438	odb.allocator,
	439	odb.format,
	440	odb.pack_file,
	441	base_object,
	442	delta_offsets.items,
	443	&odb.cache,
	444	);
	445
	446	return .{ .type = base_object.type, .data = base_data };
	447	}
	448
	449	/// Seeks to the beginning of the object with the given ID.
	450	fn seekOid(odb: *Odb, oid: Oid) !void {
	451	const oid_length = odb.format.byteLength();
	452	const key = oid.slice()[0];
	453	var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
	454	var end_index = odb.index_header.fan_out_table[key];
	455	const found_index = while (start_index < end_index) {
	456	const mid_index = start_index + (end_index - start_index) / 2;
	457	try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
	458	const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface);
	459	switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
	460	.lt => start_index = mid_index + 1,
	461	.gt => end_index = mid_index,
	462	.eq => break mid_index,
	463	}
	464	} else return error.ObjectNotFound;
	465
	466	const n_objects = odb.index_header.fan_out_table[255];
	467	const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
	468	try odb.index_file.seekTo(offset_values_start + found_index * 4);
	469	const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big));
	470	const pack_offset = pack_offset: {
	471	if (l1_offset.big) {
	472	const l2_offset_values_start = offset_values_start + n_objects * 4;
	473	try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
	474	break :pack_offset try odb.index_file.interface.takeInt(u64, .big);
	475	} else {
	476	break :pack_offset l1_offset.value;
	477	}
	478	};
	479
	480	try odb.pack_file.seekTo(pack_offset);
	481	}
	482	};
	483
	484	const Object = struct {
	485	type: Type,
	486	data: []const u8,
	487
	488	const Type = enum {
	489	commit,
	490	tree,
	491	blob,
	492	tag,
	493	};
	494	};
	495
	496	/// A cache for object data.
	497	///
	498	/// The purpose of this cache is to speed up resolution of deltas by caching the
	499	/// results of resolving delta objects, while maintaining a maximum cache size
	500	/// to avoid excessive memory usage. If the total size of the objects in the
	501	/// cache exceeds the maximum, the cache will begin evicting the least recently
	502	/// used objects: when resolving delta chains, the most recently used objects
	503	/// will likely be more helpful as they will be further along in the chain
	504	/// (skipping earlier reconstruction steps).
	505	///
	506	/// Object data stored in the cache is managed by the cache. It should not be
	507	/// freed by the caller at any point after inserting it into the cache. Any
	508	/// objects remaining in the cache will be freed when the cache itself is freed.
	509	const ObjectCache = struct {
	510	objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty,
	511	lru_nodes: std.DoublyLinkedList = .{},
	512	lru_nodes_len: usize = 0,
	513	byte_size: usize = 0,
	514
	515	const max_byte_size = 128 * 1024 * 1024; // 128MiB
	516	/// A list of offsets stored in the cache, with the most recently used
	517	/// entries at the end.
	518	const LruListNode = struct {
	519	data: u64,
	520	node: std.DoublyLinkedList.Node,
	521	};
	522	const CacheEntry = struct { object: Object, lru_node: *LruListNode };
	523
	524	fn deinit(cache: *ObjectCache, allocator: Allocator) void {
	525	var object_iterator = cache.objects.iterator();
	526	while (object_iterator.next()) \|object\| {
	527	allocator.free(object.value_ptr.object.data);
	528	allocator.destroy(object.value_ptr.lru_node);
	529	}
	530	cache.objects.deinit(allocator);
	531	cache.* = undefined;
	532	}
	533
	534	/// Gets an object from the cache, moving it to the most recently used
	535	/// position if it is present.
	536	fn get(cache: *ObjectCache, offset: u64) ?Object {
	537	if (cache.objects.get(offset)) \|entry\| {
	538	cache.lru_nodes.remove(&entry.lru_node.node);
	539	cache.lru_nodes.append(&entry.lru_node.node);
	540	return entry.object;
	541	} else {
	542	return null;
	543	}
	544	}
	545
	546	/// Puts an object in the cache, possibly evicting older entries if the
	547	/// cache exceeds its maximum size. Note that, although old objects may
	548	/// be evicted, the object just added to the cache with this function
	549	/// will not be evicted before the next call to `put` or `deinit` even if
	550	/// it exceeds the maximum cache size.
	551	fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
	552	const lru_node = try allocator.create(LruListNode);
	553	errdefer allocator.destroy(lru_node);
	554	lru_node.data = offset;
	555
	556	const gop = try cache.objects.getOrPut(allocator, offset);
	557	if (gop.found_existing) {
	558	cache.byte_size -= gop.value_ptr.object.data.len;
	559	cache.lru_nodes.remove(&gop.value_ptr.lru_node.node);
	560	cache.lru_nodes_len -= 1;
	561	allocator.destroy(gop.value_ptr.lru_node);
	562	allocator.free(gop.value_ptr.object.data);
	563	}
	564	gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
	565	cache.byte_size += object.data.len;
	566	cache.lru_nodes.append(&lru_node.node);
	567	cache.lru_nodes_len += 1;
	568
	569	while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) {
	570	// The > 1 check is to make sure that we don't evict the most
	571	// recently added node, even if it by itself happens to exceed the
	572	// maximum size of the cache.
	573	const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?));
	574	cache.lru_nodes_len -= 1;
	575	const evict_offset = evict_node.data;
	576	allocator.destroy(evict_node);
	577	const evict_object = cache.objects.get(evict_offset).?.object;
	578	cache.byte_size -= evict_object.data.len;
	579	allocator.free(evict_object.data);
	580	_ = cache.objects.remove(evict_offset);
	581	}
	582	}
	583	};
	584
	585	/// A single pkt-line in the Git protocol.
	586	///
	587	/// The format of a pkt-line is documented in
	588	/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
	589	/// meanings of the delimiter and response-end packets are documented in
	590	/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
	591	pub const Packet = union(enum) {
	592	flush,
	593	delimiter,
	594	response_end,
	595	data: []const u8,
	596
	597	pub const max_data_length = 65516;
	598
	599	/// Reads a packet in pkt-line format.
	600	fn read(reader: *Io.Reader) !Packet {
	601	const packet: Packet = try .peek(reader);
	602	switch (packet) {
	603	.data => \|data\| reader.toss(data.len),
	604	else => {},
	605	}
	606	return packet;
	607	}
	608
	609	/// Consumes the header of a pkt-line packet and reads any associated data
	610	/// into the reader's buffer, but does not consume the data.
	611	fn peek(reader: *Io.Reader) !Packet {
	612	const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket;
	613	switch (length) {
	614	0 => return .flush,
	615	1 => return .delimiter,
	616	2 => return .response_end,
	617	3 => return error.InvalidPacket,
	618	else => if (length - 4 > max_data_length) return error.InvalidPacket,
	619	}
	620	return .{ .data = try reader.peek(length - 4) };
	621	}
	622
	623	/// Writes a packet in pkt-line format.
	624	fn write(packet: Packet, writer: *Io.Writer) !void {
	625	switch (packet) {
	626	.flush => try writer.writeAll("0000"),
	627	.delimiter => try writer.writeAll("0001"),
	628	.response_end => try writer.writeAll("0002"),
	629	.data => \|data\| {
	630	assert(data.len <= max_data_length);
	631	try writer.print("{x:0>4}", .{data.len + 4});
	632	try writer.writeAll(data);
	633	},
	634	}
	635	}
	636
	637	/// Returns the normalized form of textual packet data, stripping any
	638	/// trailing '\n'.
	639	///
	640	/// As documented in
	641	/// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format),
	642	/// non-binary (textual) pkt-line data should contain a trailing '\n', but
	643	/// is not required to do so (implementations must support both forms).
	644	fn normalizeText(data: []const u8) []const u8 {
	645	return if (mem.endsWith(u8, data, "\n"))
	646	data[0 .. data.len - 1]
	647	else
	648	data;
	649	}
	650	};
	651
	652	/// A client session for the Git protocol, currently limited to an HTTP(S)
	653	/// transport. Only protocol version 2 is supported, as documented in
	654	/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
	655	pub const Session = struct {
	656	transport: *std.http.Client,
	657	location: Location,
	658	supports_agent: bool,
	659	supports_shallow: bool,
	660	object_format: Oid.Format,
	661	arena: Allocator,
	662
	663	const agent = "zig/" ++ @import("builtin").zig_version_string;
	664	const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
	665
	666	/// Initializes a client session and discovers the capabilities of the
	667	/// server for optimal transport.
	668	pub fn init(
	669	arena: Allocator,
	670	transport: *std.http.Client,
	671	uri: std.Uri,
	672	/// Asserted to be at least `Packet.max_data_length`
	673	response_buffer: []u8,
	674	) !Session {
	675	assert(response_buffer.len >= Packet.max_data_length);
	676	var session: Session = .{
	677	.transport = transport,
	678	.location = try .init(arena, uri),
	679	.supports_agent = false,
	680	.supports_shallow = false,
	681	.object_format = .sha1,
	682	.arena = arena,
	683	};
	684	var capability_iterator: CapabilityIterator = undefined;
	685	try session.getCapabilities(&capability_iterator, response_buffer);
	686	defer capability_iterator.deinit();
	687	while (try capability_iterator.next()) \|capability\| {
	688	if (mem.eql(u8, capability.key, "agent")) {
	689	session.supports_agent = true;
	690	} else if (mem.eql(u8, capability.key, "fetch")) {
	691	var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
	692	while (feature_iterator.next()) \|feature\| {
	693	if (mem.eql(u8, feature, "shallow")) {
	694	session.supports_shallow = true;
	695	}
	696	}
	697	} else if (mem.eql(u8, capability.key, "object-format")) {
	698	if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) \|format\| {
	699	session.object_format = format;
	700	}
	701	}
	702	}
	703	return session;
	704	}
	705
	706	/// An owned `std.Uri` representing the location of the server (base URI).
	707	const Location = struct {
	708	uri: std.Uri,
	709
	710	fn init(arena: Allocator, uri: std.Uri) !Location {
	711	const scheme = try arena.dupe(u8, uri.scheme);
	712	const user = if (uri.user) \|user\| try std.fmt.allocPrint(arena, "{f}", .{
	713	std.fmt.alt(user, .formatUser),
	714	}) else null;
	715	const password = if (uri.password) \|password\| try std.fmt.allocPrint(arena, "{f}", .{
	716	std.fmt.alt(password, .formatPassword),
	717	}) else null;
	718	const host = if (uri.host) \|host\| try std.fmt.allocPrint(arena, "{f}", .{
	719	std.fmt.alt(host, .formatHost),
	720	}) else null;
	721	const path = try std.fmt.allocPrint(arena, "{f}", .{
	722	std.fmt.alt(uri.path, .formatPath),
	723	});
	724	// The query and fragment are not used as part of the base server URI.
	725	return .{
	726	.uri = .{
	727	.scheme = scheme,
	728	.user = if (user) \|s\| .{ .percent_encoded = s } else null,
	729	.password = if (password) \|s\| .{ .percent_encoded = s } else null,
	730	.host = if (host) \|s\| .{ .percent_encoded = s } else null,
	731	.port = uri.port,
	732	.path = .{ .percent_encoded = path },
	733	},
	734	};
	735	}
	736	};
	737
	738	/// Returns an iterator over capabilities supported by the server.
	739	///
	740	/// The `session.location` is updated if the server returns a redirect, so
	741	/// that subsequent session functions do not need to handle redirects.
	742	fn getCapabilities(session: Session, it: CapabilityIterator, response_buffer: []u8) !void {
	743	const arena = session.arena;
	744	assert(response_buffer.len >= Packet.max_data_length);
	745	var info_refs_uri = session.location.uri;
	746	{
	747	const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	748	std.fmt.alt(session.location.uri.path, .formatPath),
	749	});
	750	info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{
	751	"/", session_uri_path, "info/refs",
	752	}) };
	753	}
	754	info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" };
	755	info_refs_uri.fragment = null;
	756
	757	const max_redirects = 3;
	758	it.* = .{
	759	.request = try session.transport.request(.GET, info_refs_uri, .{
	760	.redirect_behavior = .init(max_redirects),
	761	.extra_headers = &.{
	762	.{ .name = "Git-Protocol", .value = "version=2" },
	763	},
	764	}),
	765	.reader = undefined,
	766	.decompress = undefined,
	767	};
	768	errdefer it.deinit();
	769	const request = &it.request;
	770	try request.sendBodiless();
	771
	772	var redirect_buffer: [1024]u8 = undefined;
	773	var response = try request.receiveHead(&redirect_buffer);
	774	if (response.head.status != .ok) return error.ProtocolError;
	775	const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
	776	if (any_redirects_occurred) {
	777	const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	778	std.fmt.alt(request.uri.path, .formatPath),
	779	});
	780	if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect;
	781	var new_uri = request.uri;
	782	new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] };
	783	session.location = try .init(arena, new_uri);
	784	}
	785
	786	const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	787	it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer);
	788	var state: enum { response_start, response_content } = .response_start;
	789	while (true) {
	790	// Some Git servers (at least GitHub) include an additional
	791	// '# service=git-upload-pack' informative response before sending
	792	// the expected 'version 2' packet and capability information.
	793	// This is not universal: SourceHut, for example, does not do this.
	794	// Thus, we need to skip any such useless additional responses
	795	// before we get the one we're actually looking for. The responses
	796	// will be delimited by flush packets.
	797	const packet = Packet.read(it.reader) catch \|err\| switch (err) {
	798	error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
	799	else => \|e\| return e,
	800	};
	801	switch (packet) {
	802	.flush => state = .response_start,
	803	.data => \|data\| switch (state) {
	804	.response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) {
	805	return;
	806	} else {
	807	state = .response_content;
	808	},
	809	else => {},
	810	},
	811	else => return error.UnexpectedPacket,
	812	}
	813	}
	814	}
	815
	816	const CapabilityIterator = struct {
	817	request: std.http.Client.Request,
	818	reader: *Io.Reader,
	819	decompress: std.http.Decompress,
	820
	821	const Capability = struct {
	822	key: []const u8,
	823	value: ?[]const u8 = null,
	824
	825	fn parse(data: []const u8) Capability {
	826	return if (mem.indexOfScalar(u8, data, '=')) \|separator_pos\|
	827	.{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] }
	828	else
	829	.{ .key = data };
	830	}
	831	};
	832
	833	fn deinit(it: *CapabilityIterator) void {
	834	it.request.deinit();
	835	it.* = undefined;
	836	}
	837
	838	fn next(it: *CapabilityIterator) !?Capability {
	839	switch (try Packet.read(it.reader)) {
	840	.flush => return null,
	841	.data => \|data\| return Capability.parse(Packet.normalizeText(data)),
	842	else => return error.UnexpectedPacket,
	843	}
	844	}
	845	};
	846
	847	const ListRefsOptions = struct {
	848	/// The ref prefixes (if any) to use to filter the refs available on the
	849	/// server. Note that the client must still check the returned refs
	850	/// against its desired filters itself: the server is not required to
	851	/// respect these prefix filters and may return other refs as well.
	852	ref_prefixes: []const []const u8 = &.{},
	853	/// Whether to include symref targets for returned symbolic refs.
	854	include_symrefs: bool = false,
	855	/// Whether to include the peeled object ID for returned tag refs.
	856	include_peeled: bool = false,
	857	/// Asserted to be at least `Packet.max_data_length`.
	858	buffer: []u8,
	859	};
	860
	861	/// Returns an iterator over refs known to the server.
	862	pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void {
	863	const arena = session.arena;
	864	assert(options.buffer.len >= Packet.max_data_length);
	865	var upload_pack_uri = session.location.uri;
	866	{
	867	const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	868	std.fmt.alt(session.location.uri.path, .formatPath),
	869	});
	870	upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
	871	}
	872	upload_pack_uri.query = null;
	873	upload_pack_uri.fragment = null;
	874
	875	var body: Io.Writer = .fixed(options.buffer);
	876	try Packet.write(.{ .data = "command=ls-refs\n" }, &body);
	877	if (session.supports_agent) {
	878	try Packet.write(.{ .data = agent_capability }, &body);
	879	}
	880	{
	881	const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{
	882	session.object_format,
	883	});
	884	try Packet.write(.{ .data = object_format_packet }, &body);
	885	}
	886	try Packet.write(.delimiter, &body);
	887	for (options.ref_prefixes) \|ref_prefix\| {
	888	const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix});
	889	try Packet.write(.{ .data = ref_prefix_packet }, &body);
	890	}
	891	if (options.include_symrefs) {
	892	try Packet.write(.{ .data = "symrefs\n" }, &body);
	893	}
	894	if (options.include_peeled) {
	895	try Packet.write(.{ .data = "peel\n" }, &body);
	896	}
	897	try Packet.write(.flush, &body);
	898
	899	it.* = .{
	900	.request = try session.transport.request(.POST, upload_pack_uri, .{
	901	.redirect_behavior = .unhandled,
	902	.extra_headers = &.{
	903	.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
	904	.{ .name = "Git-Protocol", .value = "version=2" },
	905	},
	906	}),
	907	.reader = undefined,
	908	.format = session.object_format,
	909	.decompress = undefined,
	910	};
	911	const request = &it.request;
	912	errdefer request.deinit();
	913	try request.sendBodyComplete(body.buffered());
	914
	915	var response = try request.receiveHead(options.buffer);
	916	if (response.head.status != .ok) return error.ProtocolError;
	917	const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	918	it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer);
	919	}
	920
	921	pub const RefIterator = struct {
	922	format: Oid.Format,
	923	request: std.http.Client.Request,
	924	reader: *Io.Reader,
	925	decompress: std.http.Decompress,
	926
	927	pub const Ref = struct {
	928	oid: Oid,
	929	name: []const u8,
	930	symref_target: ?[]const u8,
	931	peeled: ?Oid,
	932	};
	933
	934	pub fn deinit(iterator: *RefIterator) void {
	935	iterator.request.deinit();
	936	iterator.* = undefined;
	937	}
	938
	939	pub fn next(it: *RefIterator) !?Ref {
	940	switch (try Packet.read(it.reader)) {
	941	.flush => return null,
	942	.data => \|data\| {
	943	const ref_data = Packet.normalizeText(data);
	944	const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;
	945	const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
	946
	947	const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;
	948	const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];
	949
	950	var symref_target: ?[]const u8 = null;
	951	var peeled: ?Oid = null;
	952	var last_sep_pos = name_sep_pos;
	953	while (last_sep_pos < ref_data.len) {
	954	const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len;
	955	const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos];
	956	if (mem.startsWith(u8, attribute, "symref-target:")) {
	957	symref_target = attribute["symref-target:".len..];
	958	} else if (mem.startsWith(u8, attribute, "peeled:")) {
	959	peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;
	960	}
	961	last_sep_pos = next_sep_pos;
	962	}
	963
	964	return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
	965	},
	966	else => return error.UnexpectedPacket,
	967	}
	968	}
	969	};
	970
	971	/// Fetches the given refs from the server. A shallow fetch (depth 1) is
	972	/// performed if the server supports it.
	973	pub fn fetch(
	974	session: Session,
	975	fs: *FetchStream,
	976	wants: []const []const u8,
	977	/// Asserted to be at least `Packet.max_data_length`.
	978	response_buffer: []u8,
	979	) !void {
	980	const arena = session.arena;
	981	assert(response_buffer.len >= Packet.max_data_length);
	982	var upload_pack_uri = session.location.uri;
	983	{
	984	const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	985	std.fmt.alt(session.location.uri.path, .formatPath),
	986	});
	987	upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
	988	}
	989	upload_pack_uri.query = null;
	990	upload_pack_uri.fragment = null;
	991
	992	var body: Io.Writer = .fixed(response_buffer);
	993	try Packet.write(.{ .data = "command=fetch\n" }, &body);
	994	if (session.supports_agent) {
	995	try Packet.write(.{ .data = agent_capability }, &body);
	996	}
	997	{
	998	const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)});
	999	try Packet.write(.{ .data = object_format_packet }, &body);
	1000	}
	1001	try Packet.write(.delimiter, &body);
	1002	// Our packfile parser supports the OFS_DELTA object type
	1003	try Packet.write(.{ .data = "ofs-delta\n" }, &body);
	1004	// We do not currently convey server progress information to the user
	1005	try Packet.write(.{ .data = "no-progress\n" }, &body);
	1006	if (session.supports_shallow) {
	1007	try Packet.write(.{ .data = "deepen 1\n" }, &body);
	1008	}
	1009	for (wants) \|want\| {
	1010	var buf: [Packet.max_data_length]u8 = undefined;
	1011	const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
	1012	try Packet.write(.{ .data = arg }, &body);
	1013	}
	1014	try Packet.write(.{ .data = "done\n" }, &body);
	1015	try Packet.write(.flush, &body);
	1016
	1017	fs.* = .{
	1018	.request = try session.transport.request(.POST, upload_pack_uri, .{
	1019	.redirect_behavior = .not_allowed,
	1020	.extra_headers = &.{
	1021	.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
	1022	.{ .name = "Git-Protocol", .value = "version=2" },
	1023	},
	1024	}),
	1025	.input = undefined,
	1026	.reader = undefined,
	1027	.remaining_len = undefined,
	1028	.decompress = undefined,
	1029	};
	1030	const request = &fs.request;
	1031	errdefer request.deinit();
	1032
	1033	try request.sendBodyComplete(body.buffered());
	1034
	1035	var response = try request.receiveHead(&.{});
	1036	if (response.head.status != .ok) return error.ProtocolError;
	1037
	1038	const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	1039	const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer);
	1040	// We are not interested in any of the sections of the returned fetch
	1041	// data other than the packfile section, since we aren't doing anything
	1042	// complex like ref negotiation (this is a fresh clone).
	1043	var state: enum { section_start, section_content } = .section_start;
	1044	while (true) {
	1045	const packet = try Packet.read(reader);
	1046	switch (state) {
	1047	.section_start => switch (packet) {
	1048	.data => \|data\| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) {
	1049	fs.input = reader;
	1050	fs.reader = .{
	1051	.buffer = &.{},
	1052	.vtable = &.{ .stream = FetchStream.stream },
	1053	.seek = 0,
	1054	.end = 0,
	1055	};
	1056	fs.remaining_len = 0;
	1057	return;
	1058	} else {
	1059	state = .section_content;
	1060	},
	1061	else => return error.UnexpectedPacket,
	1062	},
	1063	.section_content => switch (packet) {
	1064	.delimiter => state = .section_start,
	1065	.data => {},
	1066	else => return error.UnexpectedPacket,
	1067	},
	1068	}
	1069	}
	1070	}
	1071
	1072	pub const FetchStream = struct {
	1073	request: std.http.Client.Request,
	1074	input: *Io.Reader,
	1075	reader: Io.Reader,
	1076	err: ?Error = null,
	1077	remaining_len: usize,
	1078	decompress: std.http.Decompress,
	1079
	1080	pub fn deinit(fs: *FetchStream) void {
	1081	fs.request.deinit();
	1082	}
	1083
	1084	pub const Error = error{
	1085	InvalidPacket,
	1086	ProtocolError,
	1087	UnexpectedPacket,
	1088	WriteFailed,
	1089	ReadFailed,
	1090	EndOfStream,
	1091	};
	1092
	1093	const StreamCode = enum(u8) {
	1094	pack_data = 1,
	1095	progress = 2,
	1096	fatal_error = 3,
	1097	_,
	1098	};
	1099
	1100	pub fn stream(r: Io.Reader, w: Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
	1101	const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r));
	1102	const input = fs.input;
	1103	if (fs.remaining_len == 0) {
	1104	while (true) {
	1105	switch (Packet.peek(input) catch \|err\| {
	1106	fs.err = err;
	1107	return error.ReadFailed;
	1108	}) {
	1109	.flush => return error.EndOfStream,
	1110	.data => \|data\| switch (@as(StreamCode, @enumFromInt(data[0]))) {
	1111	.pack_data => {
	1112	input.toss(1);
	1113	fs.remaining_len = data.len - 1;
	1114	break;
	1115	},
	1116	.fatal_error => {
	1117	fs.err = error.ProtocolError;
	1118	return error.ReadFailed;
	1119	},
	1120	else => {
	1121	input.toss(data.len);
	1122	},
	1123	},
	1124	else => {
	1125	fs.err = error.UnexpectedPacket;
	1126	return error.ReadFailed;
	1127	},
	1128	}
	1129	}
	1130	}
	1131	const buf = limit.slice(try w.writableSliceGreedy(1));
	1132	const n = @min(buf.len, fs.remaining_len);
	1133	try input.readSliceAll(buf[0..n]);
	1134	w.advance(n);
	1135	fs.remaining_len -= n;
	1136	return n;
	1137	}
	1138	};
	1139	};
	1140
	1141	const PackHeader = struct {
	1142	total_objects: u32,
	1143
	1144	const signature = "PACK";
	1145	const supported_version = 2;
	1146
	1147	fn read(reader: *Io.Reader) !PackHeader {
	1148	const actual_signature = reader.take(4) catch \|e\| switch (e) {
	1149	error.EndOfStream => return error.InvalidHeader,
	1150	else => \|other\| return other,
	1151	};
	1152	if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
	1153	const version = reader.takeInt(u32, .big) catch \|e\| switch (e) {
	1154	error.EndOfStream => return error.InvalidHeader,
	1155	else => \|other\| return other,
	1156	};
	1157	if (version != supported_version) return error.UnsupportedVersion;
	1158	const total_objects = reader.takeInt(u32, .big) catch \|e\| switch (e) {
	1159	error.EndOfStream => return error.InvalidHeader,
	1160	else => \|other\| return other,
	1161	};
	1162	return .{ .total_objects = total_objects };
	1163	}
	1164	};
	1165
	1166	const EntryHeader = union(Type) {
	1167	commit: Undeltified,
	1168	tree: Undeltified,
	1169	blob: Undeltified,
	1170	tag: Undeltified,
	1171	ofs_delta: OfsDelta,
	1172	ref_delta: RefDelta,
	1173
	1174	const Type = enum(u3) {
	1175	commit = 1,
	1176	tree = 2,
	1177	blob = 3,
	1178	tag = 4,
	1179	ofs_delta = 6,
	1180	ref_delta = 7,
	1181	};
	1182
	1183	const Undeltified = struct {
	1184	uncompressed_length: u64,
	1185	};
	1186
	1187	const OfsDelta = struct {
	1188	offset: u64,
	1189	uncompressed_length: u64,
	1190	};
	1191
	1192	const RefDelta = struct {
	1193	base_object: Oid,
	1194	uncompressed_length: u64,
	1195	};
	1196
	1197	fn objectType(header: EntryHeader) Object.Type {
	1198	return switch (header) {
	1199	inline .commit, .tree, .blob, .tag => \|_, tag\| @field(Object.Type, @tagName(tag)),
	1200	else => unreachable,
	1201	};
	1202	}
	1203
	1204	fn uncompressedLength(header: EntryHeader) u64 {
	1205	return switch (header) {
	1206	inline else => \|entry\| entry.uncompressed_length,
	1207	};
	1208	}
	1209
	1210	fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader {
	1211	const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
	1212	const initial: InitialByte = @bitCast(reader.takeByte() catch \|e\| switch (e) {
	1213	error.EndOfStream => return error.InvalidFormat,
	1214	else => \|other\| return other,
	1215	});
	1216	const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0;
	1217	var uncompressed_length: u64 = initial.len;
	1218	uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
	1219	const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;
	1220	return switch (@"type") {
	1221	inline .commit, .tree, .blob, .tag => \|tag\| @unionInit(EntryHeader, @tagName(tag), .{
	1222	.uncompressed_length = uncompressed_length,
	1223	}),
	1224	.ofs_delta => .{ .ofs_delta = .{
	1225	.offset = try readOffsetVarInt(reader),
	1226	.uncompressed_length = uncompressed_length,
	1227	} },
	1228	.ref_delta => .{ .ref_delta = .{
	1229	.base_object = Oid.readBytes(format, reader) catch \|e\| switch (e) {
	1230	error.EndOfStream => return error.InvalidFormat,
	1231	else => \|other\| return other,
	1232	},
	1233	.uncompressed_length = uncompressed_length,
	1234	} },
	1235	};
	1236	}
	1237	};
	1238
	1239	fn readOffsetVarInt(r: *Io.Reader) !u64 {
	1240	const Byte = packed struct { value: u7, has_next: bool };
	1241	var b: Byte = @bitCast(try r.takeByte());
	1242	var value: u64 = b.value;
	1243	while (b.has_next) {
	1244	b = @bitCast(try r.takeByte());
	1245	value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
	1246	value \|= b.value;
	1247	}
	1248	return value;
	1249	}
	1250
	1251	const IndexHeader = struct {
	1252	fan_out_table: [256]u32,
	1253
	1254	const signature = "\xFFtOc";
	1255	const supported_version = 2;
	1256	const size = 4 + 4 + @sizeOf([256]u32);
	1257
	1258	fn read(index_header: IndexHeader, reader: Io.Reader) !void {
	1259	const sig = try reader.take(4);
	1260	if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
	1261	const version = try reader.takeInt(u32, .big);
	1262	if (version != supported_version) return error.UnsupportedVersion;
	1263	try reader.readSliceEndian(u32, &index_header.fan_out_table, .big);
	1264	}
	1265	};
	1266
	1267	const IndexEntry = struct {
	1268	offset: u64,
	1269	crc32: u32,
	1270	};
	1271
	1272	/// Writes out a version 2 index for the given packfile, as documented in
	1273	/// [pack-format](https://git-scm.com/docs/pack-format).
	1274	pub fn indexPack(
	1275	allocator: Allocator,
	1276	format: Oid.Format,
	1277	pack: *Io.File.Reader,
	1278	index_writer: *Io.File.Writer,
	1279	) !void {
	1280	try pack.seekTo(0);
	1281
	1282	var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
	1283	defer index_entries.deinit(allocator);
	1284	var pending_deltas: std.ArrayList(IndexEntry) = .empty;
	1285	defer pending_deltas.deinit(allocator);
	1286
	1287	const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas);
	1288
	1289	var cache: ObjectCache = .{};
	1290	defer cache.deinit(allocator);
	1291	var remaining_deltas = pending_deltas.items.len;
	1292	while (remaining_deltas > 0) {
	1293	var i: usize = remaining_deltas;
	1294	while (i > 0) {
	1295	i -= 1;
	1296	const delta = pending_deltas.items[i];
	1297	if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) \|oid\| {
	1298	try index_entries.put(allocator, oid, delta);
	1299	_ = pending_deltas.swapRemove(i);
	1300	}
	1301	}
	1302	if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
	1303	remaining_deltas = pending_deltas.items.len;
	1304	}
	1305
	1306	var oids: std.ArrayList(Oid) = .empty;
	1307	defer oids.deinit(allocator);
	1308	try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
	1309	var index_entries_iter = index_entries.iterator();
	1310	while (index_entries_iter.next()) \|entry\| {
	1311	oids.appendAssumeCapacity(entry.key_ptr.*);
	1312	}
	1313	mem.sortUnstable(Oid, oids.items, {}, struct {
	1314	fn lessThan(_: void, o1: Oid, o2: Oid) bool {
	1315	return mem.lessThan(u8, o1.slice(), o2.slice());
	1316	}
	1317	}.lessThan);
	1318
	1319	var fan_out_table: [256]u32 = undefined;
	1320	var count: u32 = 0;
	1321	var fan_out_index: u8 = 0;
	1322	for (oids.items) \|oid\| {
	1323	const key = oid.slice()[0];
	1324	if (key > fan_out_index) {
	1325	@memset(fan_out_table[fan_out_index..key], count);
	1326	fan_out_index = key;
	1327	}
	1328	count += 1;
	1329	}
	1330	@memset(fan_out_table[fan_out_index..], count);
	1331
	1332	var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{});
	1333	const writer = &index_hashed_writer.writer;
	1334	try writer.writeAll(IndexHeader.signature);
	1335	try writer.writeInt(u32, IndexHeader.supported_version, .big);
	1336	for (fan_out_table) \|fan_out_entry\| {
	1337	try writer.writeInt(u32, fan_out_entry, .big);
	1338	}
	1339
	1340	for (oids.items) \|oid\| {
	1341	try writer.writeAll(oid.slice());
	1342	}
	1343
	1344	for (oids.items) \|oid\| {
	1345	try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big);
	1346	}
	1347
	1348	var big_offsets: std.ArrayList(u64) = .empty;
	1349	defer big_offsets.deinit(allocator);
	1350	for (oids.items) \|oid\| {
	1351	const offset = index_entries.get(oid).?.offset;
	1352	if (offset <= std.math.maxInt(u31)) {
	1353	try writer.writeInt(u32, @intCast(offset), .big);
	1354	} else {
	1355	const index = big_offsets.items.len;
	1356	try big_offsets.append(allocator, offset);
	1357	try writer.writeInt(u32, @as(u32, @intCast(index)) \| (1 << 31), .big);
	1358	}
	1359	}
	1360	for (big_offsets.items) \|offset\| {
	1361	try writer.writeInt(u64, offset, .big);
	1362	}
	1363
	1364	try writer.writeAll(pack_checksum.slice());
	1365	const index_checksum = index_hashed_writer.hasher.finalResult();
	1366	try index_writer.interface.writeAll(index_checksum.slice());
	1367	try index_writer.end();
	1368	}
	1369
	1370	/// Performs the first pass over the packfile data for index construction.
	1371	/// This will index all non-delta objects, queue delta objects for further
	1372	/// processing, and return the pack checksum (which is part of the index
	1373	/// format).
	1374	fn indexPackFirstPass(
	1375	allocator: Allocator,
	1376	format: Oid.Format,
	1377	pack: *Io.File.Reader,
	1378	index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
	1379	pending_deltas: *std.ArrayList(IndexEntry),
	1380	) !Oid {
	1381	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	1382	var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
	1383	var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer);
	1384
	1385	const pack_header = try PackHeader.read(&pack_hashed.reader);
	1386
	1387	for (0..pack_header.total_objects) \|_\| {
	1388	const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen();
	1389	const entry_header = try EntryHeader.read(format, &pack_hashed.reader);
	1390	switch (entry_header) {
	1391	.commit, .tree, .blob, .tag => \|object\| {
	1392	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{});
	1393	var oid_hasher: Oid.Hashing = .init(format, &flate_buffer);
	1394	const oid_hasher_w = oid_hasher.writer();
	1395	// The object header is not included in the pack data but is
	1396	// part of the object's ID
	1397	try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length });
	1398	const n = try entry_decompress.reader.streamRemaining(oid_hasher_w);
	1399	if (n != object.uncompressed_length) return error.InvalidObject;
	1400	const oid = oid_hasher.final();
	1401	if (!skip_checksums) @compileError("TODO");
	1402	try index_entries.put(allocator, oid, .{
	1403	.offset = entry_offset,
	1404	.crc32 = 0,
	1405	});
	1406	},
	1407	inline .ofs_delta, .ref_delta => \|delta\| {
	1408	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer);
	1409	const n = try entry_decompress.reader.discardRemaining();
	1410	if (n != delta.uncompressed_length) return error.InvalidObject;
	1411	if (!skip_checksums) @compileError("TODO");
	1412	try pending_deltas.append(allocator, .{
	1413	.offset = entry_offset,
	1414	.crc32 = 0,
	1415	});
	1416	},
	1417	}
	1418	}
	1419
	1420	if (!skip_checksums) @compileError("TODO");
	1421	return pack_hashed.hasher.finalResult();
	1422	}
	1423
	1424	/// Attempts to determine the final object ID of the given deltified object.
	1425	/// May return null if this is not yet possible (if the delta is a ref-based
	1426	/// delta and we do not yet know the offset of the base object).
	1427	fn indexPackHashDelta(
	1428	allocator: Allocator,
	1429	format: Oid.Format,
	1430	pack: *Io.File.Reader,
	1431	delta: IndexEntry,
	1432	index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
	1433	cache: *ObjectCache,
	1434	) !?Oid {
	1435	// Figure out the chain of deltas to resolve
	1436	var base_offset = delta.offset;
	1437	var base_header: EntryHeader = undefined;
	1438	var delta_offsets: std.ArrayList(u64) = .empty;
	1439	defer delta_offsets.deinit(allocator);
	1440	const base_object = while (true) {
	1441	if (cache.get(base_offset)) \|base_object\| break base_object;
	1442
	1443	try pack.seekTo(base_offset);
	1444	base_header = try EntryHeader.read(format, &pack.interface);
	1445	switch (base_header) {
	1446	.ofs_delta => \|ofs_delta\| {
	1447	try delta_offsets.append(allocator, base_offset);
	1448	base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
	1449	},
	1450	.ref_delta => \|ref_delta\| {
	1451	try delta_offsets.append(allocator, base_offset);
	1452	base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
	1453	},
	1454	else => {
	1455	const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
	1456	errdefer allocator.free(base_data);
	1457	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
	1458	try cache.put(allocator, base_offset, base_object);
	1459	break base_object;
	1460	},
	1461	}
	1462	};
	1463
	1464	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
	1465
	1466	var entry_hasher_buffer: [64]u8 = undefined;
	1467	var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer);
	1468	const entry_hasher_w = entry_hasher.writer();
	1469	// Writes to hashers cannot fail.
	1470	entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable;
	1471	entry_hasher_w.writeAll(base_data) catch unreachable;
	1472	return entry_hasher.final();
	1473	}
	1474
	1475	/// Resolves a chain of deltas, returning the final base object data. `pack` is
	1476	/// assumed to be looking at the start of the object data for the base object of
	1477	/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
	1478	/// to obtain the final object.
	1479	fn resolveDeltaChain(
	1480	allocator: Allocator,
	1481	format: Oid.Format,
	1482	pack: *Io.File.Reader,
	1483	base_object: Object,
	1484	delta_offsets: []const u64,
	1485	cache: *ObjectCache,
	1486	) ![]const u8 {
	1487	var base_data = base_object.data;
	1488	var i: usize = delta_offsets.len;
	1489	while (i > 0) {
	1490	i -= 1;
	1491
	1492	const delta_offset = delta_offsets[i];
	1493	try pack.seekTo(delta_offset);
	1494	const delta_header = try EntryHeader.read(format, &pack.interface);
	1495	const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
	1496	defer allocator.free(delta_data);
	1497	var delta_reader: Io.Reader = .fixed(delta_data);
	1498	_ = try delta_reader.takeLeb128(u64); // base object size
	1499	const expanded_size = try delta_reader.takeLeb128(u64);
	1500
	1501	const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
	1502	const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
	1503	errdefer allocator.free(expanded_data);
	1504	var expanded_delta_stream: Io.Writer = .fixed(expanded_data);
	1505	try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
	1506	if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
	1507
	1508	try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
	1509	base_data = expanded_data;
	1510	}
	1511	return base_data;
	1512	}
	1513
	1514	/// Reads the complete contents of an object from `reader`. This function may
	1515	/// read more bytes than required from `reader`, so the reader position after
	1516	/// returning is not reliable.
	1517	fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 {
	1518	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
	1519	var aw: Io.Writer.Allocating = .init(allocator);
	1520	try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len);
	1521	defer aw.deinit();
	1522	var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
	1523	try decompress.reader.streamExact(&aw.writer, alloc_size);
	1524	return aw.toOwnedSlice();
	1525	}
	1526
	1527	/// Expands delta data from `delta_reader` to `writer`.
	1528	///
	1529	/// The format of the delta data is documented in
	1530	/// [pack-format](https://git-scm.com/docs/pack-format).
	1531	fn expandDelta(base_object: []const u8, delta_reader: Io.Reader, writer: Io.Writer) !void {
	1532	while (true) {
	1533	const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch \|e\| switch (e) {
	1534	error.EndOfStream => return,
	1535	else => \|other\| return other,
	1536	});
	1537	if (inst.copy) {
	1538	const available: packed struct {
	1539	offset1: bool,
	1540	offset2: bool,
	1541	offset3: bool,
	1542	offset4: bool,
	1543	size1: bool,
	1544	size2: bool,
	1545	size3: bool,
	1546	} = @bitCast(inst.value);
	1547	const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
	1548	.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
	1549	.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
	1550	.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
	1551	.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
	1552	};
	1553	const base_offset: u32 = @bitCast(offset_parts);
	1554	const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
	1555	.size1 = if (available.size1) try delta_reader.takeByte() else 0,
	1556	.size2 = if (available.size2) try delta_reader.takeByte() else 0,
	1557	.size3 = if (available.size3) try delta_reader.takeByte() else 0,
	1558	};
	1559	var size: u24 = @bitCast(size_parts);
	1560	if (size == 0) size = 0x10000;
	1561	try writer.writeAll(base_object[base_offset..][0..size]);
	1562	} else if (inst.value != 0) {
	1563	try delta_reader.streamExact(writer, inst.value);
	1564	} else {
	1565	return error.InvalidDeltaInstruction;
	1566	}
	1567	}
	1568	}
	1569
	1570	/// Runs the packfile indexing and checkout test.
	1571	///
	1572	/// The two testrepo repositories under testdata contain identical commit
	1573	/// histories and contents.
	1574	///
	1575	/// To verify the contents of the packfiles using Git alone, run the
	1576	/// following commands in an empty directory:
	1577	///
	1578	/// 1. `git init --object-format=(sha1\|sha256)`
	1579	/// 2. `git unpack-objects <path/to/testrepo.pack`
	1580	/// 3. `git fsck` - will print one "dangling commit":
	1581	/// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb`
	1582	/// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a`
	1583	/// 4. `git checkout $commit`
	1584	fn runRepositoryTest(io: Io, comptime format: Oid.Format, head_commit: []const u8) !void {
	1585	const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack");
	1586
	1587	var git_dir = testing.tmpDir(.{});
	1588	defer git_dir.cleanup();
	1589	var pack_file = try git_dir.dir.createFile(io, "testrepo.pack", .{ .read = true });
	1590	defer pack_file.close(io);
	1591	try pack_file.writeStreamingAll(io, testrepo_pack);
	1592
	1593	var pack_file_buffer: [2000]u8 = undefined;
	1594	var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
	1595
	1596	var index_file = try git_dir.dir.createFile(io, "testrepo.idx", .{ .read = true });
	1597	defer index_file.close(io);
	1598	var index_file_buffer: [2000]u8 = undefined;
	1599	var index_file_writer = index_file.writer(io, &index_file_buffer);
	1600	try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer);
	1601
	1602	// Arbitrary size limit on files read while checking the repository contents
	1603	// (all files in the test repo are known to be smaller than this)
	1604	const max_file_size = 8192;
	1605
	1606	if (!skip_checksums) {
	1607	const index_file_data = try git_dir.dir.readFileAlloc(io, "testrepo.idx", testing.allocator, .limited(max_file_size));
	1608	defer testing.allocator.free(index_file_data);
	1609	// testrepo.idx is generated by Git. The index created by this file should
	1610	// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
	1611	// this.
	1612	const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
	1613	try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
	1614	}
	1615
	1616	var index_file_reader = index_file.reader(io, &index_file_buffer);
	1617	var repository: Repository = undefined;
	1618	try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader);
	1619	defer repository.deinit();
	1620
	1621	var worktree = testing.tmpDir(.{ .iterate = true });
	1622	defer worktree.cleanup();
	1623
	1624	const commit_id = try Oid.parse(format, head_commit);
	1625
	1626	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
	1627	defer diagnostics.deinit();
	1628	try repository.checkout(io, worktree.dir, commit_id, &diagnostics);
	1629	try testing.expect(diagnostics.errors.items.len == 0);
	1630
	1631	const expected_files: []const []const u8 = &.{
	1632	"dir/file",
	1633	"dir/subdir/file",
	1634	"dir/subdir/file2",
	1635	"dir2/file",
	1636	"dir3/file",
	1637	"dir3/file2",
	1638	"file",
	1639	"file2",
	1640	"file3",
	1641	"file4",
	1642	"file5",
	1643	"file6",
	1644	"file7",
	1645	"file8",
	1646	"file9",
	1647	};
	1648	var actual_files: std.ArrayList([]u8) = .empty;
	1649	defer actual_files.deinit(testing.allocator);
	1650	defer for (actual_files.items) \|file\| testing.allocator.free(file);
	1651	var walker = try worktree.dir.walk(testing.allocator);
	1652	defer walker.deinit();
	1653	while (try walker.next(io)) \|entry\| {
	1654	if (entry.kind != .file) continue;
	1655	const path = try testing.allocator.dupe(u8, entry.path);
	1656	errdefer testing.allocator.free(path);
	1657	mem.replaceScalar(u8, path, std.fs.path.sep, '/');
	1658	try actual_files.append(testing.allocator, path);
	1659	}
	1660	mem.sortUnstable([]u8, actual_files.items, {}, struct {
	1661	fn lessThan(_: void, a: []u8, b: []u8) bool {
	1662	return mem.lessThan(u8, a, b);
	1663	}
	1664	}.lessThan);
	1665	try testing.expectEqualDeep(expected_files, actual_files.items);
	1666
	1667	const expected_file_contents =
	1668	\\revision 1
	1669	\\revision 2
	1670	\\revision 4
	1671	\\revision 5
	1672	\\revision 7
	1673	\\revision 8
	1674	\\revision 9
	1675	\\revision 10
	1676	\\revision 12
	1677	\\revision 13
	1678	\\revision 14
	1679	\\revision 18
	1680	\\revision 19
	1681	\\
	1682	;
	1683	const actual_file_contents = try worktree.dir.readFileAlloc(io, "file", testing.allocator, .limited(max_file_size));
	1684	defer testing.allocator.free(actual_file_contents);
	1685	try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
	1686	}
	1687
	1688	/// Checksum calculation is useful for troubleshooting and debugging, but it's
	1689	/// redundant since the package manager already does content hashing at the
	1690	/// end. Let's save time by not doing that work, but, I left a cookie crumb
	1691	/// trail here if you want to restore the functionality for tinkering purposes.
	1692	const skip_checksums = true;
	1693
	1694	test "SHA-1 packfile indexing and checkout" {
	1695	try runRepositoryTest(std.testing.io, .sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
	1696	}
	1697
	1698	test "SHA-256 packfile indexing and checkout" {
	1699	try runRepositoryTest(std.testing.io, .sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a");
	1700	}
	1701
	1702	/// Checks out a commit of a packfile. Intended for experimenting with and
	1703	/// benchmarking possible optimizations to the indexing and checkout behavior.
	1704	pub fn main() !void {
	1705	const allocator = std.heap.smp_allocator;
	1706
	1707	var threaded: Io.Threaded = .init(allocator, .{});
	1708	defer threaded.deinit();
	1709	const io = threaded.io();
	1710
	1711	const args = try std.process.argsAlloc(allocator);
	1712	defer std.process.argsFree(allocator, args);
	1713	if (args.len != 5) {
	1714	return error.InvalidArguments; // Arguments: format packfile commit worktree
	1715	}
	1716
	1717	const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat;
	1718
	1719	var pack_file = try Io.Dir.cwd().openFile(io, args[2], .{});
	1720	defer pack_file.close(io);
	1721	var pack_file_buffer: [4096]u8 = undefined;
	1722	var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
	1723
	1724	const commit = try Oid.parse(format, args[3]);
	1725	var worktree = try Io.Dir.cwd().createDirPathOpen(io, args[4], .{});
	1726	defer worktree.close(io);
	1727
	1728	var git_dir = try worktree.createDirPathOpen(io, ".git", .{});
	1729	defer git_dir.close(io);
	1730
	1731	std.debug.print("Starting index...\n", .{});
	1732	var index_file = try git_dir.createFile(io, "idx", .{ .read = true });
	1733	defer index_file.close(io);
	1734	var index_file_buffer: [4096]u8 = undefined;
	1735	var index_file_writer = index_file.writer(io, &index_file_buffer);
	1736	try indexPack(allocator, format, &pack_file_reader, &index_file_writer);
	1737
	1738	std.debug.print("Starting checkout...\n", .{});
	1739	var index_file_reader = index_file.reader(io, &index_file_buffer);
	1740	var repository: Repository = undefined;
	1741	try repository.init(allocator, format, &pack_file_reader, &index_file_reader);
	1742	defer repository.deinit();
	1743	var diagnostics: Diagnostics = .{ .allocator = allocator };
	1744	defer diagnostics.deinit();
	1745	try repository.checkout(io, worktree, commit, &diagnostics);
	1746
	1747	for (diagnostics.errors.items) \|err\| {
	1748	std.debug.print("Diagnostic: {}\n", .{err});
	1749	}
	1750	}

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.idx created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.idx differ

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.pack created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.pack differ

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.idx created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.idx differ

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.pack created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.pack differ

lib/compiler/Maker/Fuzz.zig+4-4

...	...	@@ -166,9 +166,9 @@ pub fn deinit(fuzz: *Fuzz) void {
166	166	fn rebuildTestsWorkerRun(
167	167	maker: *Maker,
168	168	run_index: Configuration.Step.Index,
169		parent_prog_node: std.Progress.Node,
	169	parent_progress_node: std.Progress.Node,
170	170	) void {
171		rebuildTestsWorkerRunFallible(maker, run_index, parent_prog_node) catch \|err\| {
	171	rebuildTestsWorkerRunFallible(maker, run_index, parent_progress_node) catch \|err\| {
172	172	const conf = &maker.scanned_config.configuration;
173	173	const conf_run = run_index.ptr(conf).extended.cast(conf, Configuration.Step.Run).?;
174	174	const comp_index = conf_run.producer.value.?;
...	...	@@ -180,7 +180,7 @@ fn rebuildTestsWorkerRun(
180	180	fn rebuildTestsWorkerRunFallible(
181	181	maker: *Maker,
182	182	run_index: Configuration.Step.Index,
183		parent_prog_node: std.Progress.Node,
	183	parent_progress_node: std.Progress.Node,
184	184	) !void {
185	185	const graph = maker.graph;
186	186	const io = graph.io;
...	...	@@ -196,7 +196,7 @@ fn rebuildTestsWorkerRunFallible(
196	196	const root_module = conf_comp.root_module.get(conf);
197	197	const target = root_module.resolved_target.get(conf).?.result.get(conf);
198	198
199		const prog_node = parent_prog_node.start(conf_comp_step.name.slice(conf), 0);
	199	const prog_node = parent_progress_node.start(conf_comp_step.name.slice(conf), 0);
200	200	defer prog_node.end();
201	201
202	202	const result = comp.rebuildInFuzzMode(maker, comp_index, prog_node);

lib/compiler/Maker/Graph.zig-1

...	...	@@ -4,7 +4,6 @@ const Graph = @This();
4	4	const std = @import("std");
5	5	const Io = std.Io;
6	6	const Allocator = std.mem.Allocator;
7		const Configuration = std.Build.Configuration;
8	7	const Path = std.Build.Cache.Path;
9	8	const Directory = std.Build.Cache.Directory;
10	9

lib/compiler/Maker/Package.zig created+207

...	...	@@ -0,0 +1,207 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3
	4	pub const Fetch = @import("Fetch.zig");
	5	pub const Manifest = @import("Package/Manifest.zig");
	6
	7	pub const Fingerprint = packed struct(u64) {
	8	id: u32,
	9	checksum: u32,
	10
	11	pub fn generate(rng: std.Random, name: []const u8) Fingerprint {
	12	return .{
	13	.id = rng.intRangeLessThan(u32, 1, 0xffffffff),
	14	.checksum = std.hash.Crc32.hash(name),
	15	};
	16	}
	17
	18	pub fn validate(n: Fingerprint, name: []const u8) bool {
	19	switch (n.id) {
	20	0x00000000, 0xffffffff => return false,
	21	else => return std.hash.Crc32.hash(name) == n.checksum,
	22	}
	23	}
	24
	25	pub fn int(n: Fingerprint) u64 {
	26	return @bitCast(n);
	27	}
	28	};
	29
	30	/// A user-readable, file system safe hash that identifies an exact package
	31	/// snapshot, including file contents.
	32	///
	33	/// The hash is not only to prevent collisions but must resist attacks where
	34	/// the adversary fully controls the contents being hashed. Thus, it contains
	35	/// a full SHA-256 digest.
	36	///
	37	/// This data structure can be used to store the legacy hash format too. Legacy
	38	/// hash format is scheduled to be removed after 0.14.0 is tagged.
	39	///
	40	/// There's also a third way this structure is used. When using path rather than
	41	/// hash, a unique hash is still needed, so one is computed based on the path.
	42	pub const Hash = struct {
	43	/// Maximum size of a package hash. Unused bytes at the end are
	44	/// filled with zeroes.
	45	///
	46	/// Assumed to be already validated.
	47	bytes: [max_len]u8,
	48
	49	pub const Algo = std.crypto.hash.sha2.Sha256;
	50	pub const Digest = [Algo.digest_length]u8;
	51
	52	/// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
	53	pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6;
	54
	55	/// Asserts `s` is valid.
	56	pub fn fromSlice(s: []const u8) Hash {
	57	assert(validate(s) == .ok);
	58	var result: Hash = undefined;
	59	@memcpy(result.bytes[0..s.len], s);
	60	@memset(result.bytes[s.len..], 0);
	61	return result;
	62	}
	63
	64	pub const Validation = enum { ok, short, long, incomplete };
	65
	66	pub fn validate(s: []const u8) Validation {
	67	if (s.len > max_len) return .long;
	68	if (s.len < 44) return .short;
	69	const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-');
	70	if (n_dashes < 2) return .incomplete;
	71	return .ok;
	72	}
	73
	74	test validate {
	75	try std.testing.expectEqual(.short, validate(""));
	76	}
	77
	78	pub fn toSlice(ph: *const Hash) []const u8 {
	79	var end: usize = ph.bytes.len;
	80	while (true) {
	81	end -= 1;
	82	if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
	83	}
	84	}
	85
	86	pub fn eql(a: const Hash, b: const Hash) bool {
	87	return std.mem.eql(u8, &a.bytes, &b.bytes);
	88	}
	89
	90	/// Produces "$name-$semver-$hashplus".
	91	/// * name is the name field from build.zig.zon, asserted to be at most 32
	92	/// bytes and assumed be a valid zig identifier
	93	/// * semver is the version field from build.zig.zon, asserted to be at
	94	/// most 32 bytes
	95	/// * hashplus is the following 33-byte array, base64 encoded using -_ to make
	96	/// it filesystem safe:
	97	/// - (4 bytes) LE u32 Package ID
	98	/// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
	99	/// - (25 bytes) truncated SHA-256 digest of hashed files of the package
	100	pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash {
	101	assert(name.len <= 32);
	102	assert(ver.len <= 32);
	103	var result: Hash = undefined;
	104	var buf: std.ArrayList(u8) = .initBuffer(&result.bytes);
	105	buf.appendSliceAssumeCapacity(name);
	106	buf.appendAssumeCapacity('-');
	107	buf.appendSliceAssumeCapacity(ver);
	108	buf.appendAssumeCapacity('-');
	109	var hashplus: [33]u8 = undefined;
	110	std.mem.writeInt(u32, hashplus[0..4], id, .little);
	111	std.mem.writeInt(u32, hashplus[4..8], size, .little);
	112	hashplus[8..].* = digest[0..25].*;
	113	_ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus);
	114	@memset(buf.unusedCapacitySlice(), 0);
	115	return result;
	116	}
	117
	118	/// Produces a unique hash based on the path provided. The result should
	119	/// not be user-visible.
	120	pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
	121	var result: Hash = .{ .bytes = @splat(0) };
	122	var i: usize = 0;
	123	if (is_global) {
	124	result.bytes[0] = '/';
	125	i += 1;
	126	}
	127	if (i + sub_path.len <= result.bytes.len) {
	128	@memcpy(result.bytes[i..][0..sub_path.len], sub_path);
	129	return result;
	130	}
	131	var bin_digest: [Algo.digest_length]u8 = undefined;
	132	Algo.hash(sub_path, &bin_digest, .{});
	133	_ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable;
	134	return result;
	135	}
	136
	137	pub fn projectId(hash: *const Hash) ProjectId {
	138	const bytes = hash.toSlice();
	139	const name = std.mem.sliceTo(bytes, '-');
	140	const encoded_hashplus = bytes[bytes.len - 44 ..];
	141	var hashplus: [33]u8 = undefined;
	142	std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable;
	143	const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little);
	144	return .init(name, fingerprint_id);
	145	}
	146
	147	test projectId {
	148	const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw");
	149	const project_id = hash.projectId();
	150
	151	var expected_name: [32]u8 = @splat(0);
	152	expected_name[0.."pulseaudio".len].* = "pulseaudio".*;
	153	try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
	154
	155	try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id);
	156	}
	157
	158	test "projectId with dashes in the base64" {
	159	const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC");
	160	const project_id = hash.projectId();
	161
	162	var expected_name: [32]u8 = @splat(0);
	163	expected_name[0.."dvui".len].* = "dvui".*;
	164	try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
	165
	166	try std.testing.expectEqual(0x99490101, project_id.fingerprint_id);
	167	}
	168	};
	169
	170	/// Minimum information required to identify whether a package is an artifact
	171	/// of a given project.
	172	pub const ProjectId = struct {
	173	/// Bytes after name.len are set to zero.
	174	padded_name: [32]u8,
	175	fingerprint_id: u32,
	176
	177	pub fn init(name: []const u8, fingerprint_id: u32) ProjectId {
	178	var padded_name: [32]u8 = @splat(0);
	179	@memcpy(padded_name[0..name.len], name);
	180	return .{
	181	.padded_name = padded_name,
	182	.fingerprint_id = fingerprint_id,
	183	};
	184	}
	185
	186	pub fn eql(a: const ProjectId, b: const ProjectId) bool {
	187	return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name);
	188	}
	189
	190	pub fn hash(a: *const ProjectId) u64 {
	191	const x: u64 = @bitCast(a.padded_name[0..8].*);
	192	return std.hash.int(x \| a.fingerprint_id);
	193	}
	194	};
	195
	196	test Hash {
	197	const example_digest: Hash.Digest = .{
	198	0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
	199	0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
	200	};
	201	const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024);
	202	try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice());
	203	}
	204
	205	test {
	206	_ = Fetch;
	207	}

lib/compiler/Maker/Package/Manifest.zig created+734

...	...	@@ -0,0 +1,734 @@
	1	const Manifest = @This();
	2
	3	const std = @import("std");
	4	const Io = std.Io;
	5	const mem = std.mem;
	6	const Allocator = std.mem.Allocator;
	7	const assert = std.debug.assert;
	8	const Ast = std.zig.Ast;
	9	const testing = std.testing;
	10
	11	const Package = @import("../Package.zig");
	12
	13	pub const max_bytes = 10 * 1024 * 1024;
	14	pub const basename = "build.zig.zon";
	15	pub const max_name_len = 32;
	16	pub const max_version_len = 32;
	17
	18	pub const Dependency = struct {
	19	location: Location,
	20	location_tok: Ast.TokenIndex,
	21	location_node: Ast.Node.Index,
	22	hash: ?[]const u8,
	23	hash_tok: Ast.OptionalTokenIndex,
	24	hash_node: Ast.Node.OptionalIndex,
	25	node: Ast.Node.Index,
	26	name_tok: Ast.TokenIndex,
	27	lazy: bool,
	28
	29	pub const Location = union(enum) {
	30	url: []const u8,
	31	path: []const u8,
	32	};
	33	};
	34
	35	pub const ErrorMessage = struct {
	36	msg: []const u8,
	37	tok: Ast.TokenIndex,
	38	off: u32,
	39	};
	40
	41	name: []const u8,
	42	id: u32,
	43	version: std.SemanticVersion,
	44	version_node: Ast.Node.Index,
	45	dependencies: std.array_hash_map.String(Dependency),
	46	dependencies_node: Ast.Node.OptionalIndex,
	47	paths: std.array_hash_map.String(void),
	48	minimum_zig_version: ?std.SemanticVersion,
	49
	50	errors: []ErrorMessage,
	51	arena_state: std.heap.ArenaAllocator.State,
	52
	53	pub const ParseOptions = struct {
	54	allow_missing_paths_field: bool = false,
	55	};
	56
	57	pub const Error = Allocator.Error;
	58
	59	pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
	60	const main_node_index = ast.nodeData(.root).node;
	61
	62	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	63	errdefer arena_instance.deinit();
	64
	65	var p: Parse = .{
	66	.gpa = gpa,
	67	.ast = ast.*,
	68	.arena = arena_instance.allocator(),
	69	.errors = .empty,
	70
	71	.name = undefined,
	72	.id = 0,
	73	.version = undefined,
	74	.version_node = undefined,
	75	.dependencies = .{},
	76	.dependencies_node = .none,
	77	.paths = .empty,
	78	.allow_missing_paths_field = options.allow_missing_paths_field,
	79	.minimum_zig_version = null,
	80	.buf = .empty,
	81	};
	82	defer p.buf.deinit(gpa);
	83	defer p.errors.deinit(gpa);
	84	defer p.dependencies.deinit(gpa);
	85	defer p.paths.deinit(gpa);
	86
	87	p.parseRoot(main_node_index, rng) catch \|err\| switch (err) {
	88	error.ParseFailure => assert(p.errors.items.len > 0),
	89	else => \|e\| return e,
	90	};
	91
	92	return .{
	93	.name = p.name,
	94	.id = p.id,
	95	.version = p.version,
	96	.version_node = p.version_node,
	97	.dependencies = try p.dependencies.clone(p.arena),
	98	.dependencies_node = p.dependencies_node,
	99	.paths = try p.paths.clone(p.arena),
	100	.minimum_zig_version = p.minimum_zig_version,
	101	.errors = try p.arena.dupe(ErrorMessage, p.errors.items),
	102	.arena_state = arena_instance.state,
	103	};
	104	}
	105
	106	pub fn deinit(man: *Manifest, gpa: Allocator) void {
	107	man.arena_state.promote(gpa).deinit();
	108	man.* = undefined;
	109	}
	110
	111	pub fn copyErrorsIntoBundle(
	112	man: Manifest,
	113	ast: Ast,
	114	/// ErrorBundle null-terminated string index
	115	src_path: u32,
	116	eb: *std.zig.ErrorBundle.Wip,
	117	) Allocator.Error!void {
	118	for (man.errors) \|msg\| {
	119	const start_loc = ast.tokenLocation(0, msg.tok);
	120
	121	try eb.addRootErrorMessage(.{
	122	.msg = try eb.addString(msg.msg),
	123	.src_loc = try eb.addSourceLocation(.{
	124	.src_path = src_path,
	125	.span_start = ast.tokenStart(msg.tok),
	126	.span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len),
	127	.span_main = ast.tokenStart(msg.tok) + msg.off,
	128	.line = @intCast(start_loc.line),
	129	.column = @intCast(start_loc.column),
	130	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	131	}),
	132	});
	133	}
	134	}
	135
	136	const Parse = struct {
	137	gpa: Allocator,
	138	ast: Ast,
	139	arena: Allocator,
	140	buf: std.ArrayList(u8),
	141	errors: std.ArrayList(ErrorMessage),
	142
	143	name: []const u8,
	144	id: u32,
	145	version: std.SemanticVersion,
	146	version_node: Ast.Node.Index,
	147	dependencies: std.array_hash_map.String(Dependency),
	148	dependencies_node: Ast.Node.OptionalIndex,
	149	paths: std.array_hash_map.String(void),
	150	allow_missing_paths_field: bool,
	151	minimum_zig_version: ?std.SemanticVersion,
	152
	153	const InnerError = error{ ParseFailure, OutOfMemory };
	154
	155	fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void {
	156	const ast = p.ast;
	157	const main_token = ast.nodeMainToken(node);
	158
	159	var buf: [2]Ast.Node.Index = undefined;
	160	const struct_init = ast.fullStructInit(&buf, node) orelse {
	161	return fail(p, main_token, "expected top level expression to be a struct", .{});
	162	};
	163
	164	var have_name = false;
	165	var have_version = false;
	166	var have_included_paths = false;
	167	var fingerprint: ?Package.Fingerprint = null;
	168
	169	for (struct_init.ast.fields) \|field_init\| {
	170	const name_token = ast.firstToken(field_init) - 2;
	171	const field_name = try identifierTokenString(p, name_token);
	172	// We could get fancy with reflection and comptime logic here but doing
	173	// things manually provides an opportunity to do any additional verification
	174	// that is desirable on a per-field basis.
	175	if (mem.eql(u8, field_name, "dependencies")) {
	176	p.dependencies_node = field_init.toOptional();
	177	try parseDependencies(p, field_init);
	178	} else if (mem.eql(u8, field_name, "paths")) {
	179	have_included_paths = true;
	180	try parseIncludedPaths(p, field_init);
	181	} else if (mem.eql(u8, field_name, "name")) {
	182	p.name = try parseName(p, field_init);
	183	have_name = true;
	184	} else if (mem.eql(u8, field_name, "fingerprint")) {
	185	fingerprint = try parseFingerprint(p, field_init);
	186	} else if (mem.eql(u8, field_name, "version")) {
	187	p.version_node = field_init;
	188	const version_text = try parseString(p, field_init);
	189	if (version_text.len > max_version_len) {
	190	try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len });
	191	}
	192	p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
	193	try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
	194	break :v undefined;
	195	};
	196	have_version = true;
	197	} else if (mem.eql(u8, field_name, "minimum_zig_version")) {
	198	const version_text = try parseString(p, field_init);
	199	p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
	200	try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
	201	break :v null;
	202	};
	203	} else {
	204	// Ignore unknown fields so that we can add fields in future zig
	205	// versions without breaking older zig versions.
	206	}
	207	}
	208
	209	if (!have_name) {
	210	try appendError(p, main_token, "missing top-level 'name' field", .{});
	211	} else {
	212	if (fingerprint) \|n\| {
	213	if (!n.validate(p.name)) {
	214	return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{
	215	n.int(), Package.Fingerprint.generate(rng, p.name).int(),
	216	});
	217	}
	218	p.id = n.id;
	219	} else {
	220	try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
	221	Package.Fingerprint.generate(rng, p.name).int(),
	222	});
	223	}
	224	}
	225
	226	if (!have_version) {
	227	try appendError(p, main_token, "missing top-level 'version' field", .{});
	228	}
	229
	230	if (!have_included_paths) {
	231	if (p.allow_missing_paths_field) {
	232	try p.paths.put(p.gpa, "", {});
	233	} else {
	234	try appendError(p, main_token, "missing top-level 'paths' field", .{});
	235	}
	236	}
	237	}
	238
	239	fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
	240	const ast = p.ast;
	241
	242	var buf: [2]Ast.Node.Index = undefined;
	243	const struct_init = ast.fullStructInit(&buf, node) orelse {
	244	const tok = ast.nodeMainToken(node);
	245	return fail(p, tok, "expected dependencies expression to be a struct", .{});
	246	};
	247
	248	for (struct_init.ast.fields) \|field_init\| {
	249	const name_token = ast.firstToken(field_init) - 2;
	250	const dep_name = try identifierTokenString(p, name_token);
	251	const dep = try parseDependency(p, field_init);
	252	try p.dependencies.put(p.gpa, dep_name, dep);
	253	}
	254	}
	255
	256	fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
	257	const ast = p.ast;
	258
	259	var buf: [2]Ast.Node.Index = undefined;
	260	const struct_init = ast.fullStructInit(&buf, node) orelse {
	261	const tok = ast.nodeMainToken(node);
	262	return fail(p, tok, "expected dependency expression to be a struct", .{});
	263	};
	264
	265	var dep: Dependency = .{
	266	.location = undefined,
	267	.location_tok = undefined,
	268	.location_node = undefined,
	269	.hash = null,
	270	.hash_tok = .none,
	271	.hash_node = .none,
	272	.node = node,
	273	.name_tok = undefined,
	274	.lazy = false,
	275	};
	276	var has_location = false;
	277
	278	for (struct_init.ast.fields) \|field_init\| {
	279	const name_token = ast.firstToken(field_init) - 2;
	280	dep.name_tok = name_token;
	281	const field_name = try identifierTokenString(p, name_token);
	282	// We could get fancy with reflection and comptime logic here but doing
	283	// things manually provides an opportunity to do any additional verification
	284	// that is desirable on a per-field basis.
	285	if (mem.eql(u8, field_name, "url")) {
	286	if (has_location) {
	287	return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
	288	}
	289	dep.location = .{
	290	.url = parseString(p, field_init) catch \|err\| switch (err) {
	291	error.ParseFailure => continue,
	292	else => \|e\| return e,
	293	},
	294	};
	295	has_location = true;
	296	dep.location_tok = ast.nodeMainToken(field_init);
	297	dep.location_node = field_init;
	298	} else if (mem.eql(u8, field_name, "path")) {
	299	if (has_location) {
	300	return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
	301	}
	302	dep.location = .{
	303	.path = parseString(p, field_init) catch \|err\| switch (err) {
	304	error.ParseFailure => continue,
	305	else => \|e\| return e,
	306	},
	307	};
	308	has_location = true;
	309	dep.location_tok = ast.nodeMainToken(field_init);
	310	dep.location_node = field_init;
	311	} else if (mem.eql(u8, field_name, "hash")) {
	312	dep.hash = parseHash(p, field_init) catch \|err\| switch (err) {
	313	error.ParseFailure => continue,
	314	else => \|e\| return e,
	315	};
	316	dep.hash_tok = .fromToken(ast.nodeMainToken(field_init));
	317	dep.hash_node = field_init.toOptional();
	318	} else if (mem.eql(u8, field_name, "lazy")) {
	319	dep.lazy = parseBool(p, field_init) catch \|err\| switch (err) {
	320	error.ParseFailure => continue,
	321	else => \|e\| return e,
	322	};
	323	} else {
	324	// Ignore unknown fields so that we can add fields in future zig
	325	// versions without breaking older zig versions.
	326	}
	327	}
	328
	329	if (!has_location) {
	330	try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{});
	331	}
	332
	333	return dep;
	334	}
	335
	336	fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
	337	const ast = p.ast;
	338
	339	var buf: [2]Ast.Node.Index = undefined;
	340	const array_init = ast.fullArrayInit(&buf, node) orelse {
	341	const tok = ast.nodeMainToken(node);
	342	return fail(p, tok, "expected paths expression to be a list of strings", .{});
	343	};
	344
	345	for (array_init.ast.elements) \|elem_node\| {
	346	const path_string = try parseString(p, elem_node);
	347	// This is normalized so that it can be used in string comparisons
	348	// against file system paths.
	349	const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
	350	try p.paths.put(p.gpa, normalized, {});
	351	}
	352	}
	353
	354	fn parseBool(p: *Parse, node: Ast.Node.Index) !bool {
	355	const ast = p.ast;
	356	if (ast.nodeTag(node) != .identifier) {
	357	return fail(p, ast.nodeMainToken(node), "expected identifier", .{});
	358	}
	359	const ident_token = ast.nodeMainToken(node);
	360	const token_bytes = ast.tokenSlice(ident_token);
	361	if (mem.eql(u8, token_bytes, "true")) {
	362	return true;
	363	} else if (mem.eql(u8, token_bytes, "false")) {
	364	return false;
	365	} else {
	366	return fail(p, ident_token, "expected boolean", .{});
	367	}
	368	}
	369
	370	fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint {
	371	const ast = p.ast;
	372	const main_token = ast.nodeMainToken(node);
	373	if (ast.nodeTag(node) != .number_literal) {
	374	return fail(p, main_token, "expected integer literal", .{});
	375	}
	376	const token_bytes = ast.tokenSlice(main_token);
	377	const parsed = std.zig.parseNumberLiteral(token_bytes);
	378	switch (parsed) {
	379	.int => \|n\| return @bitCast(n),
	380	.big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{
	381	@tagName(parsed),
	382	}),
	383	.failure => \|err\| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
	384	}
	385	}
	386
	387	fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	388	const ast = p.ast;
	389	const main_token = ast.nodeMainToken(node);
	390
	391	if (ast.nodeTag(node) != .enum_literal)
	392	return fail(p, main_token, "expected enum literal", .{});
	393
	394	const ident_name = ast.tokenSlice(main_token);
	395	if (mem.startsWith(u8, ident_name, "@"))
	396	return fail(p, main_token, "name must be a valid bare zig identifier", .{});
	397
	398	if (ident_name.len > max_name_len)
	399	return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{
	400	std.zig.fmtId(ident_name), max_name_len,
	401	});
	402
	403	return ident_name;
	404	}
	405
	406	fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	407	const ast = p.ast;
	408	if (ast.nodeTag(node) != .string_literal) {
	409	return fail(p, ast.nodeMainToken(node), "expected string literal", .{});
	410	}
	411	const str_lit_token = ast.nodeMainToken(node);
	412	const token_bytes = ast.tokenSlice(str_lit_token);
	413	p.buf.clearRetainingCapacity();
	414	try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
	415	const duped = try p.arena.dupe(u8, p.buf.items);
	416	return duped;
	417	}
	418
	419	fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	420	const ast = p.ast;
	421	const tok = ast.nodeMainToken(node);
	422	const h = try parseString(p, node);
	423	switch (Package.Hash.validate(h)) {
	424	.ok => return h,
	425	else => \|t\| return fail(p, tok, "invalid hash: {t}", .{t}),
	426	}
	427	}
	428
	429	/// TODO: try to DRY this with AstGen.identifierTokenString
	430	fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
	431	const ast = p.ast;
	432	assert(ast.tokenTag(token) == .identifier);
	433	const ident_name = ast.tokenSlice(token);
	434	if (!mem.startsWith(u8, ident_name, "@")) {
	435	return ident_name;
	436	}
	437	p.buf.clearRetainingCapacity();
	438	try parseStrLit(p, token, &p.buf, ident_name, 1);
	439	const duped = try p.arena.dupe(u8, p.buf.items);
	440	return duped;
	441	}
	442
	443	/// TODO: try to DRY this with AstGen.parseStrLit
	444	fn parseStrLit(
	445	p: *Parse,
	446	token: Ast.TokenIndex,
	447	buf: *std.ArrayList(u8),
	448	bytes: []const u8,
	449	offset: u32,
	450	) InnerError!void {
	451	const raw_string = bytes[offset..];
	452	const result = r: {
	453	var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf);
	454	defer buf.* = aw.toArrayList();
	455	break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch \|err\| switch (err) {
	456	error.WriteFailed => return error.OutOfMemory,
	457	};
	458	};
	459	switch (result) {
	460	.success => {},
	461	.failure => \|err\| try p.appendStrLitError(err, token, bytes, offset),
	462	}
	463	}
	464
	465	/// TODO: try to DRY this with AstGen.failWithStrLitError
	466	fn appendStrLitError(
	467	p: *Parse,
	468	err: std.zig.string_literal.Error,
	469	token: Ast.TokenIndex,
	470	bytes: []const u8,
	471	offset: u32,
	472	) Allocator.Error!void {
	473	const raw_string = bytes[offset..];
	474	switch (err) {
	475	.invalid_escape_character => \|bad_index\| {
	476	try p.appendErrorOff(
	477	token,
	478	offset + @as(u32, @intCast(bad_index)),
	479	"invalid escape character: '{c}'",
	480	.{raw_string[bad_index]},
	481	);
	482	},
	483	.expected_hex_digit => \|bad_index\| {
	484	try p.appendErrorOff(
	485	token,
	486	offset + @as(u32, @intCast(bad_index)),
	487	"expected hex digit, found '{c}'",
	488	.{raw_string[bad_index]},
	489	);
	490	},
	491	.empty_unicode_escape_sequence => \|bad_index\| {
	492	try p.appendErrorOff(
	493	token,
	494	offset + @as(u32, @intCast(bad_index)),
	495	"empty unicode escape sequence",
	496	.{},
	497	);
	498	},
	499	.expected_hex_digit_or_rbrace => \|bad_index\| {
	500	try p.appendErrorOff(
	501	token,
	502	offset + @as(u32, @intCast(bad_index)),
	503	"expected hex digit or '}}', found '{c}'",
	504	.{raw_string[bad_index]},
	505	);
	506	},
	507	.invalid_unicode_codepoint => \|bad_index\| {
	508	try p.appendErrorOff(
	509	token,
	510	offset + @as(u32, @intCast(bad_index)),
	511	"unicode escape does not correspond to a valid unicode scalar value",
	512	.{},
	513	);
	514	},
	515	.expected_lbrace => \|bad_index\| {
	516	try p.appendErrorOff(
	517	token,
	518	offset + @as(u32, @intCast(bad_index)),
	519	"expected '{{', found '{c}",
	520	.{raw_string[bad_index]},
	521	);
	522	},
	523	.expected_rbrace => \|bad_index\| {
	524	try p.appendErrorOff(
	525	token,
	526	offset + @as(u32, @intCast(bad_index)),
	527	"expected '}}', found '{c}",
	528	.{raw_string[bad_index]},
	529	);
	530	},
	531	.expected_single_quote => \|bad_index\| {
	532	try p.appendErrorOff(
	533	token,
	534	offset + @as(u32, @intCast(bad_index)),
	535	"expected single quote ('), found '{c}",
	536	.{raw_string[bad_index]},
	537	);
	538	},
	539	.invalid_character => \|bad_index\| {
	540	try p.appendErrorOff(
	541	token,
	542	offset + @as(u32, @intCast(bad_index)),
	543	"invalid byte in string or character literal: '{c}'",
	544	.{raw_string[bad_index]},
	545	);
	546	},
	547	.empty_char_literal => {
	548	try p.appendErrorOff(token, offset, "empty character literal", .{});
	549	},
	550	}
	551	}
	552
	553	fn fail(
	554	p: *Parse,
	555	tok: Ast.TokenIndex,
	556	comptime fmt: []const u8,
	557	args: anytype,
	558	) InnerError {
	559	try appendError(p, tok, fmt, args);
	560	return error.ParseFailure;
	561	}
	562
	563	fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
	564	return appendErrorOff(p, tok, 0, fmt, args);
	565	}
	566
	567	fn appendErrorOff(
	568	p: *Parse,
	569	tok: Ast.TokenIndex,
	570	byte_offset: u32,
	571	comptime fmt: []const u8,
	572	args: anytype,
	573	) Allocator.Error!void {
	574	try p.errors.append(p.gpa, .{
	575	.msg = try std.fmt.allocPrint(p.arena, fmt, args),
	576	.tok = tok,
	577	.off = byte_offset,
	578	});
	579	}
	580	};
	581
	582	pub fn load(
	583	io: Io,
	584	arena: Allocator,
	585	manifest_path: std.Build.Cache.Path,
	586	ast: *std.zig.Ast,
	587	error_bundle: *std.zig.ErrorBundle.Wip,
	588	manifest: *Manifest,
	589	allow_missing_paths_field: bool,
	590	) !void {
	591	const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions(
	592	io,
	593	manifest_path.sub_path,
	594	arena,
	595	.limited(max_bytes),
	596	.@"1",
	597	0,
	598	);
	599
	600	ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
	601
	602	if (ast.errors.len > 0) {
	603	const file_path = try manifest_path.joinString(arena, "");
	604	try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle);
	605	return error.ErrorsBundled;
	606	}
	607
	608	const rng: std.Random.IoSource = .{ .io = io };
	609
	610	manifest.* = try parse(arena, ast, rng.interface(), .{
	611	.allow_missing_paths_field = allow_missing_paths_field,
	612	});
	613
	614	if (manifest.errors.len > 0) {
	615	const src_path = try error_bundle.printString("{f}", .{manifest_path});
	616	try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle);
	617	return error.ErrorsBundled;
	618	}
	619	}
	620
	621	test "basic" {
	622	const gpa = testing.allocator;
	623
	624	const example =
	625	\\.{
	626	\\ .name = .foo,
	627	\\ .fingerprint = 0x8c736521490b23df,
	628	\\ .version = "3.2.1",
	629	\\ .paths = .{""},
	630	\\ .dependencies = .{
	631	\\ .bar = .{
	632	\\ .url = "https://example.com/baz.tar.gz",
	633	\\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
	634	\\ },
	635	\\ },
	636	\\}
	637	;
	638
	639	var ast = try Ast.parse(gpa, example, .zon);
	640	defer ast.deinit(gpa);
	641
	642	try testing.expect(ast.errors.len == 0);
	643
	644	var rng = std.Random.DefaultPrng.init(0);
	645
	646	var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
	647	defer manifest.deinit(gpa);
	648
	649	try testing.expect(manifest.errors.len == 0);
	650	try testing.expectEqualStrings("foo", manifest.name);
	651
	652	try testing.expectEqual(@as(std.SemanticVersion, .{
	653	.major = 3,
	654	.minor = 2,
	655	.patch = 1,
	656	}), manifest.version);
	657
	658	try testing.expect(manifest.dependencies.count() == 1);
	659	try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
	660	try testing.expectEqualStrings(
	661	"https://example.com/baz.tar.gz",
	662	manifest.dependencies.values()[0].location.url,
	663	);
	664	try testing.expectEqualStrings(
	665	"libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
	666	manifest.dependencies.values()[0].hash orelse return error.TestFailed,
	667	);
	668
	669	try testing.expect(manifest.minimum_zig_version == null);
	670	}
	671
	672	test "minimum_zig_version" {
	673	const gpa = testing.allocator;
	674
	675	const example =
	676	\\.{
	677	\\ .name = .foo,
	678	\\ .fingerprint = 0x8c736521490b23df,
	679	\\ .version = "3.2.1",
	680	\\ .paths = .{""},
	681	\\ .minimum_zig_version = "0.11.1",
	682	\\}
	683	;
	684
	685	var ast = try Ast.parse(gpa, example, .zon);
	686	defer ast.deinit(gpa);
	687
	688	try testing.expect(ast.errors.len == 0);
	689
	690	var rng = std.Random.DefaultPrng.init(0);
	691
	692	var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
	693	defer manifest.deinit(gpa);
	694
	695	try testing.expect(manifest.errors.len == 0);
	696	try testing.expect(manifest.dependencies.count() == 0);
	697
	698	try testing.expect(manifest.minimum_zig_version != null);
	699
	700	try testing.expectEqual(@as(std.SemanticVersion, .{
	701	.major = 0,
	702	.minor = 11,
	703	.patch = 1,
	704	}), manifest.minimum_zig_version.?);
	705	}
	706
	707	test "minimum_zig_version - invalid version" {
	708	const gpa = testing.allocator;
	709
	710	const example =
	711	\\.{
	712	\\ .name = .foo,
	713	\\ .fingerprint = 0x8c736521490b23df,
	714	\\ .version = "3.2.1",
	715	\\ .minimum_zig_version = "X.11.1",
	716	\\ .paths = .{""},
	717	\\}
	718	;
	719
	720	var ast = try Ast.parse(gpa, example, .zon);
	721	defer ast.deinit(gpa);
	722
	723	try testing.expect(ast.errors.len == 0);
	724
	725	var rng = std.Random.DefaultPrng.init(0);
	726
	727	var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
	728	defer manifest.deinit(gpa);
	729
	730	try testing.expect(manifest.errors.len == 1);
	731	try testing.expect(manifest.dependencies.count() == 0);
	732
	733	try testing.expect(manifest.minimum_zig_version == null);
	734	}

lib/compiler/Maker/ScannedConfig.zig+1-3

...	...	@@ -9,7 +9,7 @@ const Graph = @import("Graph.zig");
9	9
10	10	configuration: Configuration,
11	11	top_level_steps: std.array_hash_map.String(Configuration.Step.Index),
12		path: []const u8,
	12	path: std.Build.Cache.Path,
13	13
14	14	pub fn print(sc: const ScannedConfig, w: Writer) Writer.Error!void {
15	15	std.log.err("TODO also print paths", .{});
...	...	@@ -342,7 +342,6 @@ pub fn printUsage(sc: const ScannedConfig, graph: Graph, w: *Writer) !void {
342	342	\\ --build-file [file] Override path to build.zig
343	343	\\ --cache-dir [path] Override path to local Zig cache directory
344	344	\\ --global-cache-dir [path] Override path to global Zig cache directory
345		\\ --zig-lib-dir [arg] Override path to Zig lib directory
346	345	\\ --seed [integer] For shuffling dependency traversal order (default: random)
347	346	\\ --cache-poison[=mode] Override configuration caching behavior
348	347	\\ pure (default) Avoid false positive cache hits
...	...	@@ -360,7 +359,6 @@ pub fn printUsage(sc: const ScannedConfig, graph: Graph, w: *Writer) !void {
360	359	\\ none (default) No build ID
361	360	\\ --debug-log [scope] Enable debugging the compiler
362	361	\\ --debug-pkg-config Fail if unknown pkg-config flags encountered
363		\\ --maker-opt=[mode] Change maker executable optimization mode (default: ReleaseSafe)
364	362	\\ --verbose-link Enable compiler debug output for linking
365	363	\\ --verbose-air Enable compiler debug output for Zig AIR
366	364	\\ --verbose-llvm-ir Enable compiler debug output for LLVM IR

lib/compiler/Maker/Step.zig+2-2

...	...	@@ -584,7 +584,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: Maker, zp: Zi
584	584	if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
585	585	return s.fail(
586	586	maker,
587		"zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
	587	"zig version mismatch build runner vs compiler: {q} vs {q}",
588	588	.{ builtin.zig_version_string, body },
589	589	);
590	590	}
...	...	@@ -654,7 +654,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: Maker, zp: Zi
654	654	}
655	655	}
656	656	},
657		.time_report => if (maker.web_server) \|*ws\| {
	657	.time_report => if (maker.web_server) \|ws\| {
658	658	const TimeReport = std.zig.Server.Message.TimeReport;
659	659	const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]);
660	660	ws.updateTimeReportCompile(.{

lib/compiler/Maker/Step/Compile.zig+30-37

...	...	@@ -10,7 +10,6 @@ const Module = std.Build.Configuration.Module;
10	10	const Io = std.Io;
11	11	const Sha256 = std.crypto.hash.sha2.Sha256;
12	12	const assert = std.debug.assert;
13		const allocPrint = std.fmt.allocPrint;
14	13
15	14	const Step = @import("../Step.zig");
16	15	const Maker = @import("../../Maker.zig");
...	...	@@ -179,7 +178,7 @@ fn lowerZigArgs(
179	178	try zig_args.append(gpa, cmd);
180	179
181	180	if (graph.reference_trace) \|some\| {
182		try zig_args.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{some}));
	181	try zig_args.append(gpa, try arena.print("-freference-trace={d}", .{some}));
183	182	}
184	183	try addFlag(gpa, zig_args, "allow-so-scripts", conf_comp.flags2.allow_so_scripts.toBool() orelse graph.allow_so_scripts);
185	184
...	...	@@ -191,7 +190,7 @@ fn lowerZigArgs(
191	190
192	191	if (root_module.resolved_target.get(conf).?.query.unwrap()) \|query\| {
193	192	if (query.get(conf).flags.object_format.unwrap()) \|ofmt\| {
194		try zig_args.append(gpa, try allocPrint(arena, "-ofmt={t}", .{ofmt}));
	193	try zig_args.append(gpa, try arena.print("-ofmt={t}", .{ofmt}));
195	194	}
196	195	}
197	196
...	...	@@ -201,7 +200,7 @@ fn lowerZigArgs(
201	200	.enabled => try zig_args.append(gpa, "-fentry"),
202	201	.symbol_name => {
203	202	const symbol_name = conf_comp.entry.value.?.slice(conf);
204		try zig_args.append(gpa, try allocPrint(arena, "-fentry={s}", .{symbol_name}));
	203	try zig_args.append(gpa, try arena.print("-fentry={s}", .{symbol_name}));
205	204	},
206	205	}
207	206
...	...	@@ -210,7 +209,7 @@ fn lowerZigArgs(
210	209	}
211	210
212	211	if (conf_comp.stack_size.value) \|stack_size\| {
213		try zig_args.appendSlice(gpa, &.{ "--stack", try allocPrint(arena, "{d}", .{stack_size}) });
	212	try zig_args.appendSlice(gpa, &.{ "--stack", try arena.print("{d}", .{stack_size}) });
214	213	}
215	214
216	215	try addBool(gpa, zig_args, "-ffuzz", fuzz);
...	...	@@ -346,7 +345,7 @@ fn lowerZigArgs(
346	345	else => \|e\| return e,
347	346	}
348	347	}
349		try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{
	348	try zig_args.append(gpa, try arena.print("{s}{s}", .{
350	349	prefix, system_lib_name,
351	350	}));
352	351	}
...	...	@@ -526,7 +525,7 @@ fn lowerZigArgs(
526	525	if (mem.eql(u8, import_cli_name, name_slice)) {
527	526	zig_args.appendAssumeCapacity(import_cli_name);
528	527	} else {
529		zig_args.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{
	528	zig_args.appendAssumeCapacity(try arena.print("{s}={s}", .{
530	529	name_slice, import_cli_name,
531	530	}));
532	531	}
...	...	@@ -542,9 +541,9 @@ fn lowerZigArgs(
542	541	try zig_args.ensureUnusedCapacity(gpa, 1);
543	542	if (mod.root_source_file.unwrap()) \|lp\| {
544	543	const src = try maker.resolveLazyPathIndexAbs(arena, lp, compile_index);
545		zig_args.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}", .{ module_cli_name, src }));
	544	zig_args.appendAssumeCapacity(try arena.print("-M{s}={s}", .{ module_cli_name, src }));
546	545	} else if (moduleNeedsCliArg(&mod, conf)) {
547		zig_args.appendAssumeCapacity(try allocPrint(arena, "-M{s}", .{module_cli_name}));
	546	zig_args.appendAssumeCapacity(try arena.print("-M{s}", .{module_cli_name}));
548	547	}
549	548	}
550	549	}
...	...	@@ -583,7 +582,7 @@ fn lowerZigArgs(
583	582
584	583	if (conf_comp.image_base.value) \|image_base\| {
585	584	(try zig_args.addManyAsArray(gpa, 2)).* = .{
586		"--image-base", try allocPrint(arena, "0x{x}", .{image_base}),
	585	"--image-base", try arena.print("0x{x}", .{image_base}),
587	586	};
588	587	}
589	588
...	...	@@ -643,10 +642,10 @@ fn lowerZigArgs(
643	642	if (!conf_comp.flags.link_z_relro) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "norelro" };
644	643	if (conf_comp.flags.link_z_lazy) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "lazy" };
645	644	if (conf_comp.link_z_common_page_size.value) \|size\| (try zig_args.addManyAsArray(gpa, 2)).* = .{
646		"-z", try allocPrint(arena, "common-page-size={d}", .{size}),
	645	"-z", try arena.print("common-page-size={d}", .{size}),
647	646	};
648	647	if (conf_comp.link_z_max_page_size.value) \|size\| (try zig_args.addManyAsArray(gpa, 2)).* = .{
649		"-z", try allocPrint(arena, "max-page-size={d}", .{size}),
	648	"-z", try arena.print("max-page-size={d}", .{size}),
650	649	};
651	650	if (conf_comp.flags.link_z_defs) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "defs" };
652	651
...	...	@@ -667,7 +666,7 @@ fn lowerZigArgs(
667	666	try zig_args.ensureUnusedCapacity(gpa, 1);
668	667	if (graph.debug_compiler_runtime_libs) \|mode\| switch (mode) {
669	668	.Debug => zig_args.appendAssumeCapacity("--debug-rt"),
670		else => zig_args.appendAssumeCapacity(try allocPrint(arena, "--debug-rt={t}", .{mode})),
	669	else => zig_args.appendAssumeCapacity(try arena.print("--debug-rt={t}", .{mode})),
671	670	};
672	671
673	672	{
...	...	@@ -691,15 +690,9 @@ fn lowerZigArgs(
691	690	const abi = root_module_target.flags.abi.unwrap().?;
692	691	zig_args.addManyAsArrayAssumeCapacity(2).* = .{
693	692	"-install_name",
694		if (conf_comp.install_name.value) \|s\| s.slice(conf) else try allocPrint(
695		arena,
696		"@rpath/{s}{s}{s}",
697		.{
698		os_tag.libPrefix(abi),
699		conf_comp.root_name.slice(conf),
700		os_tag.dynamicLibSuffix(),
701		},
702		),
	693	if (conf_comp.install_name.value) \|s\| s.slice(conf) else try arena.print("@rpath/{s}{s}{s}", .{
	694	os_tag.libPrefix(abi), conf_comp.root_name.slice(conf), os_tag.dynamicLibSuffix(),
	695	}),
703	696	};
704	697	}
705	698	}
...	...	@@ -712,12 +705,12 @@ fn lowerZigArgs(
712	705	}
713	706	if (conf_comp.pagezero_size.value) \|pagezero_size\| {
714	707	(try zig_args.addManyAsArray(gpa, 2)).* = .{
715		"-pagezero_size", try allocPrint(arena, "{x}", .{pagezero_size}),
	708	"-pagezero_size", try arena.print("{x}", .{pagezero_size}),
716	709	};
717	710	}
718	711	if (conf_comp.headerpad_size.value) \|headerpad_size\| {
719	712	(try zig_args.addManyAsArray(gpa, 2)).* = .{
720		"-headerpad", try allocPrint(arena, "{x}", .{headerpad_size}),
	713	"-headerpad", try arena.print("{x}", .{headerpad_size}),
721	714	};
722	715	}
723	716	try addBool(gpa, zig_args, "-headerpad_max_install_names", conf_comp.flags.headerpad_max_install_names);
...	...	@@ -740,13 +733,13 @@ fn lowerZigArgs(
740	733	{
741	734	try zig_args.ensureUnusedCapacity(gpa, 4);
742	735	if (conf_comp.initial_memory.value) \|initial_memory\| {
743		zig_args.appendAssumeCapacity(try allocPrint(arena, "--initial-memory={d}", .{initial_memory}));
	736	zig_args.appendAssumeCapacity(try arena.print("--initial-memory={d}", .{initial_memory}));
744	737	}
745	738	if (conf_comp.max_memory.value) \|max_memory\| {
746		zig_args.appendAssumeCapacity(try allocPrint(arena, "--max-memory={d}", .{max_memory}));
	739	zig_args.appendAssumeCapacity(try arena.print("--max-memory={d}", .{max_memory}));
747	740	}
748	741	if (conf_comp.global_base.value) \|global_base\| {
749		zig_args.appendAssumeCapacity(try allocPrint(arena, "--global-base={d}", .{global_base}));
	742	zig_args.appendAssumeCapacity(try arena.print("--global-base={d}", .{global_base}));
750	743	}
751	744	switch (conf_comp.flags3.wasi_exec_model) {
752	745	.default => {},
...	...	@@ -777,7 +770,7 @@ fn lowerZigArgs(
777	770
778	771	for (graph.search_prefixes.items) \|search_prefix\| {
779	772	var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch \|err\| {
780		return step.fail(maker, "unable to open prefix directory '{s}': {t}", .{ search_prefix, err });
	773	return step.fail(maker, "unable to open prefix directory {q}: {t}", .{ search_prefix, err });
781	774	};
782	775	defer prefix_dir.close(io);
783	776
...	...	@@ -791,7 +784,7 @@ fn lowerZigArgs(
791	784	});
792	785	} else \|err\| switch (err) {
793	786	error.FileNotFound => {},
794		else => \|e\| return step.fail(maker, "unable to access '{s}/lib' directory: {t}", .{ search_prefix, e }),
	787	else => \|e\| return step.fail(maker, "unable to access {s}/lib directory: {t}", .{ search_prefix, e }),
795	788	}
796	789
797	790	if (prefix_dir.access(io, "include", .{})) \|_\| {
...	...	@@ -800,7 +793,7 @@ fn lowerZigArgs(
800	793	});
801	794	} else \|err\| switch (err) {
802	795	error.FileNotFound => {},
803		else => \|e\| return step.fail(maker, "unable to access '{s}/include' directory: {t}", .{ search_prefix, e }),
	796	else => \|e\| return step.fail(maker, "unable to access {s}/include directory: {t}", .{ search_prefix, e }),
804	797	}
805	798	}
806	799
...	...	@@ -812,15 +805,15 @@ fn lowerZigArgs(
812	805
813	806	if (conf_comp.flags3.build_id.unwrap(conf_comp.build_id.value, conf) orelse graph.build_id) \|build_id\| {
814	807	try zig_args.append(gpa, switch (build_id) {
815		.hexstring => \|hs\| try allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()}),
816		.none, .fast, .uuid, .sha1, .md5 => try allocPrint(arena, "--build-id={t}", .{build_id}),
	808	.hexstring => \|hs\| try arena.print("--build-id=0x{x}", .{hs.toSlice()}),
	809	.none, .fast, .uuid, .sha1, .md5 => try arena.print("--build-id={t}", .{build_id}),
817	810	});
818	811	}
819	812
820	813	const opt_zig_lib_dir: ?[]const u8 = if (conf_comp.zig_lib_dir.value) \|dir\|
821	814	try maker.resolveLazyPathIndexAbs(arena, dir, compile_index)
822	815	else if (graph.zig_lib_directory.path) \|_\|
823		try allocPrint(arena, "{f}", .{graph.zig_lib_directory})
	816	try arena.print("{f}", .{graph.zig_lib_directory})
824	817	else
825	818	null;
826	819
...	...	@@ -848,7 +841,7 @@ fn lowerZigArgs(
848	841	try addBool(gpa, zig_args, "-municode", conf_comp.flags.mingw_unicode_entry_point);
849	842
850	843	if (conf_comp.error_limit.value orelse graph.error_limit) \|err_limit\| (try zig_args.addManyAsArray(gpa, 2)).* = .{
851		"--error-limit", try allocPrint(arena, "{d}", .{err_limit}),
	844	"--error-limit", try arena.print("{d}", .{err_limit}),
852	845	};
853	846
854	847	try addFlag(gpa, zig_args, "incremental", conf_comp.flags4.incremental.toBool() orelse graph.incremental);
...	...	@@ -1152,7 +1145,7 @@ const CliNamedModules = struct {
1152	1145	try result.modules.putNoClobber(arena, mod, {});
1153	1146	break;
1154	1147	}
1155		name = try allocPrint(arena, "{s}{d}", .{ orig_name_slice, n });
	1148	name = try arena.print("{s}{d}", .{ orig_name_slice, n });
1156	1149	n += 1;
1157	1150	}
1158	1151	}
...	...	@@ -1307,7 +1300,7 @@ fn appendModuleFlags(
1307	1300	}
1308	1301
1309	1302	for (m.export_symbol_names.slice) \|symbol_name\| {
1310		try zig_args.append(gpa, try allocPrint(arena, "--export={s}", .{symbol_name.slice(conf)}));
	1303	try zig_args.append(gpa, try arena.print("--export={s}", .{symbol_name.slice(conf)}));
1311	1304	}
1312	1305
1313	1306	try zig_args.ensureUnusedCapacity(gpa, 2 * m.include_dirs.len);
...	...	@@ -1375,7 +1368,7 @@ pub fn appendIncludeDirFlags(
1375	1368	zig_args.appendAssumeCapacity(try path.toString(arena));
1376	1369	},
1377	1370	.embed_path => \|lazy_path\| {
1378		zig_args.appendAssumeCapacity(try allocPrint(arena, "--embed-dir={f}", .{
	1371	zig_args.appendAssumeCapacity(try arena.print("--embed-dir={f}", .{
1379	1372	try maker.resolveLazyPathIndex(arena, lazy_path, asking_step),
1380	1373	}));
1381	1374	},

lib/compiler/Maker/Step/ObjCopy.zig+9-10

...	...	@@ -3,7 +3,6 @@ const ObjCopy = @This();
3	3	const std = @import("std");
4	4	const Io = std.Io;
5	5	const Path = std.Build.Cache.Path;
6		const allocPrint = std.fmt.allocPrint;
7	6	const Configuration = std.Build.Configuration;
8	7
9	8	const Step = @import("../Step.zig");
...	...	@@ -55,7 +54,7 @@ pub fn make(
55	54	.sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }),
56	55	};
57	56	if (conf_oc.debug_file.value) \|debug_file\| {
58		const debug_basename = opt_debug_basename orelse try allocPrint(arena, "{s}.debug", .{
	57	const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{
59	58	Io.Dir.path.basename(input_path.sub_path),
60	59	});
61	60	maker.generatedPath(debug_file).* = .{
...	...	@@ -92,7 +91,7 @@ pub fn make(
92	91
93	92	if (conf_oc.pad_to.value) \|pad_to\| {
94	93	argv.addManyAsArrayAssumeCapacity(2).* = .{
95		"--pad-to", try allocPrint(arena, "{d}", .{pad_to}),
	94	"--pad-to", try arena.print("{d}", .{pad_to}),
96	95	};
97	96	}
98	97
...	...	@@ -105,14 +104,14 @@ pub fn make(
105	104	argv.appendAssumeCapacity("--compress-debug-sections");
106	105
107	106	if (conf_oc.debug_file.value) \|debug_file\| {
108		const debug_basename = opt_debug_basename orelse try allocPrint(arena, "{s}.debug", .{
	107	const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{
109	108	Io.Dir.path.basename(input_path.sub_path),
110	109	});
111	110	const debug_dest_path: Path = .{
112	111	.root_dir = cache_root,
113	112	.sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }),
114	113	};
115		argv.appendAssumeCapacity(try allocPrint(arena, "--extract-to={f}", .{debug_dest_path}));
	114	argv.appendAssumeCapacity(try arena.print("--extract-to={f}", .{debug_dest_path}));
116	115	maker.generatedPath(debug_file).* = debug_dest_path;
117	116	}
118	117
...	...	@@ -120,7 +119,7 @@ pub fn make(
120	119
121	120	for (conf_oc.add_section.slice) \|section\| {
122	121	argv.appendAssumeCapacity("--add-section");
123		argv.appendAssumeCapacity(try allocPrint(arena, "{s}={f}", .{
	122	argv.appendAssumeCapacity(try arena.print("{s}={f}", .{
124	123	section.section_name.slice(conf),
125	124	try maker.resolveLazyPathIndex(arena, section.file_path, step_index),
126	125	}));
...	...	@@ -133,14 +132,14 @@ pub fn make(
133	132
134	133	if (update.flags.alignment.toBytes()) \|a\| {
135	134	argv.appendAssumeCapacity("--set-section-alignment");
136		argv.appendAssumeCapacity(try allocPrint(arena, "{s}={d}", .{ name, a }));
	135	argv.appendAssumeCapacity(try arena.print("{s}={d}", .{ name, a }));
137	136	}
138	137
139	138	const f = update.flags.section_flags;
140	139	if (f != Configuration.Step.ObjCopy.SectionFlags.default) {
141	140	// trailing comma is allowed
142	141	argv.appendAssumeCapacity("--set-section-flags");
143		argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{
	142	argv.appendAssumeCapacity(try arena.print("{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{
144	143	name,
145	144	if (f.alloc) "alloc," else "",
146	145	if (f.contents) "contents," else "",
...	...	@@ -155,8 +154,8 @@ pub fn make(
155	154	}
156	155	}
157	156
158		argv.appendAssumeCapacity(try allocPrint(arena, "{f}", .{input_path}));
159		argv.appendAssumeCapacity(try allocPrint(arena, "{f}", .{dest_path}));
	157	argv.appendAssumeCapacity(try arena.print("{f}", .{input_path}));
	158	argv.appendAssumeCapacity(try arena.print("{f}", .{dest_path}));
160	159
161	160	argv.appendAssumeCapacity("--listen=-");
162	161	_ = Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch \|err\| switch (err) {

lib/compiler/Maker/Step/Run.zig+7-8

...	...	@@ -12,7 +12,6 @@ const Path = std.Build.Cache.Path;
12	12	const assert = std.debug.assert;
13	13	const mem = std.mem;
14	14	const process = std.process;
15		const allocPrint = std.fmt.allocPrint;
16	15	const Allocator = std.mem.Allocator;
17	16
18	17	const Step = @import("../Step.zig");
...	...	@@ -196,8 +195,8 @@ pub fn make(
196	195	const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }, false);
197	196
198	197	try argv_list.ensureUnusedCapacity(gpa, 3);
199		argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string}));
200		argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed}));
	198	argv_list.appendAssumeCapacity(try arena.print("--cache-dir={s}", .{cache_dir_string}));
	199	argv_list.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{graph.random_seed}));
201	200	argv_list.appendAssumeCapacity("--listen=-");
202	201	}
203	202
...	...	@@ -1242,7 +1241,7 @@ fn evalZigTest(
1242	1241	step.test_results = test_results;
1243	1242	if (test_metadata) \|tm\| {
1244	1243	run.cached_test_metadata = tm.toCachedTestMetadata();
1245		if (maker.web_server) \|*ws\| {
	1244	if (maker.web_server) \|ws\| {
1246	1245	if (graph.time_report) {
1247	1246	ws.updateTimeReportRunTest(
1248	1247	run_index,
...	...	@@ -1627,8 +1626,8 @@ pub fn rerunInFuzzMode(
1627	1626	const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }, false);
1628	1627
1629	1628	try argv_list.ensureUnusedCapacity(gpa, 3);
1630		argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string}));
1631		argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed}));
	1629	argv_list.appendAssumeCapacity(try arena.print("--cache-dir={s}", .{cache_dir_string}));
	1630	argv_list.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{graph.random_seed}));
1632	1631	argv_list.appendAssumeCapacity("--listen=-");
1633	1632	}
1634	1633
...	...	@@ -1925,7 +1924,7 @@ fn runCommand(
1925	1924	const path = try maker.resolveLazyPath(arena, lazy_path.get(conf), run_index);
1926	1925	path.root_dir.handle.createDirPath(io, path.subPathOrDot()) catch \|e\|
1927	1926	return step.fail(maker, "failed creating directory {f}: {t}", .{ path, e });
1928		interp_argv.appendAssumeCapacity(try allocPrint(arena, "--dir={f}::{s}", .{ path, name.slice(conf) }));
	1927	interp_argv.appendAssumeCapacity(try arena.print("--dir={f}::{s}", .{ path, name.slice(conf) }));
1929	1928	}
1930	1929	// Wasmtime doeesn't inherit environment variables from the parent process
1931	1930	// by default. '-S inherit-env' was added in Wasmtime version 20.
...	...	@@ -2479,7 +2478,7 @@ fn addPathForDynLibs(
2479	2478	const dll_path = try maker.generatedPath(conf_comp.generated_bin.value.?).toString(arena);
2480	2479	const search_path = Dir.path.dirname(dll_path).?;
2481	2480	if (environ_map.get(path_key)) \|prev_path\| {
2482		const new_path = try allocPrint(arena, "{s}{c}{s}", .{ prev_path, path_delimiter, search_path });
	2481	const new_path = try arena.print("{s}{c}{s}", .{ prev_path, path_delimiter, search_path });
2483	2482	try environ_map.put(path_key, new_path);
2484	2483	} else {
2485	2484	try environ_map.put(path_key, search_path);

lib/compiler/Maker/Step/TranslateC.zig+3-4

...	...	@@ -3,7 +3,6 @@ const TranslateC = @This();
3	3	const std = @import("std");
4	4	const Io = std.Io;
5	5	const Configuration = std.Build.Configuration;
6		const allocPrint = std.fmt.allocPrint;
7	6	const assert = std.debug.assert;
8	7	const OptimizeMode = std.lang.OptimizeMode;
9	8
...	...	@@ -54,7 +53,7 @@ pub fn make(
54	53	.fast => .ReleaseFast,
55	54	.small => .ReleaseSmall,
56	55	};
57		if (opt) \|o\| argv.appendAssumeCapacity(try allocPrint(arena, "-O{t}", .{o}));
	56	if (opt) \|o\| argv.appendAssumeCapacity(try arena.print("-O{t}", .{o}));
58	57
59	58	try argv.ensureUnusedCapacity(arena, conf_tc.include_dirs.len * 2);
60	59	for (0..conf_tc.include_dirs.len) \|i\|
...	...	@@ -133,7 +132,7 @@ pub fn make(
133	132	else => \|e\| return e,
134	133	}
135	134	}
136		try argv.append(arena, try allocPrint(arena, "{s}{s}", .{
	135	try argv.append(arena, try arena.print("{s}{s}", .{
137	136	prefix, system_lib_name,
138	137	}));
139	138	}
...	...	@@ -151,7 +150,7 @@ pub fn make(
151	150	}).?;
152	151
153	152	const stem = Io.Dir.path.stem(Io.Dir.path.basename(c_source_path));
154		const out_basename = try allocPrint(arena, "{s}.zig", .{stem});
	153	const out_basename = try arena.print("{s}.zig", .{stem});
155	154
156	155	maker.generatedPath(conf_tc.output_file).* = try output_dir_path.join(arena, out_basename);
157	156	}

lib/compiler/Maker/Step/UpdateSourceFiles.zig-1

...	...	@@ -3,7 +3,6 @@ const UpdateSourceFiles = @This();
3	3	const std = @import("std");
4	4	const Io = std.Io;
5	5	const Path = std.Build.Cache.Path;
6		const allocPrint = std.fmt.allocPrint;
7	6	const Configuration = std.Build.Configuration;
8	7
9	8	const Step = @import("../Step.zig");

lib/compiler/Maker/Step/WriteFile.zig-1

...	...	@@ -4,7 +4,6 @@ const std = @import("std");
4	4	const Io = std.Io;
5	5	const assert = std.debug.assert;
6	6	const Path = std.Build.Cache.Path;
7		const allocPrint = std.fmt.allocPrint;
8	7	const Configuration = std.Build.Configuration;
9	8
10	9	const Step = @import("../Step.zig");

lib/compiler/Maker/WebServer.zig+176-276

...	...	@@ -19,7 +19,7 @@ const Fuzz = @import("Fuzz.zig");
19	19	const Graph = @import("Graph.zig");
20	20	const Step = @import("Step.zig");
21	21
22		maker: *Maker,
	22	graph: *const Graph,
23	23	listen_address: net.IpAddress,
24	24	root_prog_node: std.Progress.Node,
25	25
...	...	@@ -28,17 +28,8 @@ serve_task: ?Io.Future(Io.Cancelable!void),
28	28
29	29	/// Uses `Io.Clock.awake`.
30	30	base_timestamp: Io.Timestamp,
31		/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`.
32		step_names_trailing: []u8,
33
34		/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps.
35		/// Accessed atomically.
36		step_status_bits: []u8,
37	31
38	32	fuzz: ?Fuzz,
39		time_report_mutex: Io.Mutex,
40		time_report_msgs: [][]u8,
41		time_report_update_times: []i64,
42	33
43	34	build_status: std.atomic.Value(abi.BuildStatus),
44	35	/// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate`
...	...	@@ -55,6 +46,21 @@ runner_request_ready_cond: Io.Condition,
55	46	runner_request_empty_cond: Io.Condition,
56	47	runner_request: ?RunnerRequest,
57	48
	49	configured: ?Configured,
	50
	51	const Configured = struct {
	52	maker: *Maker,
	53	/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`.
	54	step_names_trailing: []u8,
	55	/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps.
	56	/// Accessed atomically.
	57	step_status_bits: []u8,
	58
	59	time_report_mutex: Io.Mutex,
	60	time_report_msgs: [][]u8,
	61	time_report_update_times: []i64,
	62	};
	63
58	64	/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates
59	65	/// on a fixed interval of this many milliseconds.
60	66	const default_update_interval_ms = 500;
...	...	@@ -63,34 +69,88 @@ pub const base_clock: Io.Clock = .awake;
63	69
64	70	/// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`.
65	71	pub fn notifyUpdate(ws: *WebServer) void {
66		const io = ws.maker.graph.io;
	72	const io = ws.graph.io;
67	73	_ = ws.update_id.rmw(.Add, 1, .release);
68	74	io.futexWake(u32, &ws.update_id.raw, 16);
69	75	}
70	76
71	77	pub const Options = struct {
72		maker: *Maker,
	78	graph: *const Graph,
73	79	root_prog_node: std.Progress.Node,
74	80	listen_address: net.IpAddress,
75	81	base_timestamp: Io.Clock.Timestamp,
76	82	};
	83
77	84	pub fn init(opts: Options) WebServer {
78	85	// The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent`
79	86	// instead of threads, so that the web server can function in single-threaded builds.
80	87	comptime assert(!builtin.single_threaded);
81	88	assert(opts.base_timestamp.clock == base_clock);
	89	return .{
	90	.graph = opts.graph,
	91	.listen_address = opts.listen_address,
	92	.root_prog_node = opts.root_prog_node,
	93
	94	.tcp_server = null,
	95	.serve_task = null,
	96
	97	.base_timestamp = opts.base_timestamp.raw,
	98
	99	.fuzz = null,
	100
	101	.build_status = .init(.idle),
	102	.update_id = .init(0),
	103
	104	.runner_request_mutex = .init,
	105	.runner_request_ready_cond = .init,
	106	.runner_request_empty_cond = .init,
	107	.runner_request = null,
	108
	109	.configured = null,
	110	};
	111	}
	112
	113	pub fn deinit(ws: *WebServer) void {
	114	const graph = ws.graph;
	115	const io = graph.io;
	116
	117	if (ws.fuzz) \|*f\| f.deinit();
82	118
83		const maker = opts.maker;
	119	ws.releaseConfigured();
	120
	121	if (ws.serve_task) \|t\| {
	122	if (ws.tcp_server) \|*s\| s.stream.close(io);
	123	t.await();
	124	}
	125	if (ws.tcp_server) \|*s\| s.deinit();
	126	}
	127
	128	fn releaseConfigured(ws: *WebServer) void {
	129	if (ws.configured) \|*configured\| {
	130	const gpa = configured.maker.gpa;
	131	gpa.free(configured.step_names_trailing);
	132	gpa.free(configured.step_status_bits);
	133	for (configured.time_report_msgs) \|msg\| gpa.free(msg);
	134	gpa.free(configured.time_report_msgs);
	135	gpa.free(configured.time_report_update_times);
	136	gpa.free(configured.step_names_trailing);
	137	ws.configured = null;
	138	}
	139	}
	140
	141	pub fn updateConfiguration(ws: WebServer, maker: Maker) !void {
	142	const graph = ws.graph;
	143	const gpa = maker.gpa;
84	144	const all_steps = maker.step_stack.keys();
85	145	const c = &maker.scanned_config.configuration;
86		const gpa = maker.gpa;
87		const graph = maker.graph;
88	146
89		const step_names_trailing = gpa.alloc(u8, len: {
	147	const step_names_trailing = try gpa.alloc(u8, len: {
90	148	var name_bytes: usize = 0;
91	149	for (all_steps) \|step_index\| name_bytes += step_index.ptr(c).name.slice(c).len;
92	150	break :len name_bytes + all_steps.len * 4;
93		}) catch @panic("out of memory");
	151	});
	152	errdefer gpa.free(step_names_trailing);
	153
94	154	{
95	155	const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]);
96	156	var idx: usize = all_steps.len * 4;
...	...	@@ -103,71 +163,35 @@ pub fn init(opts: Options) WebServer {
103	163	assert(idx == step_names_trailing.len);
104	164	}
105	165
106		const step_status_bits = gpa.alloc(
107		u8,
108		std.math.divCeil(usize, all_steps.len, 4) catch unreachable,
109		) catch @panic("out of memory");
	166	const step_status_bits = try gpa.alloc(u8, std.math.divCeil(usize, all_steps.len, 4) catch unreachable);
	167	errdefer gpa.free(step_status_bits);
110	168	@memset(step_status_bits, 0);
111	169
112	170	const time_reports_len: usize = if (graph.time_report) all_steps.len else 0;
113		const time_report_msgs = gpa.alloc([]u8, time_reports_len) catch @panic("out of memory");
114		const time_report_update_times = gpa.alloc(i64, time_reports_len) catch @panic("out of memory");
	171	const time_report_msgs = try gpa.alloc([]u8, time_reports_len);
	172	errdefer gpa.free(time_report_msgs);
	173	const time_report_update_times = try gpa.alloc(i64, time_reports_len);
	174	errdefer gpa.free(time_report_update_times);
115	175	@memset(time_report_msgs, &.{});
116	176	@memset(time_report_update_times, std.math.minInt(i64));
117	177
118		return .{
119		.maker = maker,
120		.listen_address = opts.listen_address,
121		.root_prog_node = opts.root_prog_node,
122
123		.tcp_server = null,
124		.serve_task = null,
	178	ws.releaseConfigured();
125	179
126		.base_timestamp = opts.base_timestamp.raw,
	180	ws.configured = .{
	181	.maker = maker,
127	182	.step_names_trailing = step_names_trailing,
128
129	183	.step_status_bits = step_status_bits,
130
131		.fuzz = null,
132	184	.time_report_mutex = .init,
133	185	.time_report_msgs = time_report_msgs,
134	186	.time_report_update_times = time_report_update_times,
135
136		.build_status = .init(.idle),
137		.update_id = .init(0),
138
139		.runner_request_mutex = .init,
140		.runner_request_ready_cond = .init,
141		.runner_request_empty_cond = .init,
142		.runner_request = null,
143	187	};
144	188	}
145		pub fn deinit(ws: *WebServer) void {
146		const maker = ws.maker;
147		const gpa = maker.gpa;
148		const io = maker.graph.io;
149
150		gpa.free(ws.step_names_trailing);
151		gpa.free(ws.step_status_bits);
152	189
153		if (ws.fuzz) \|*f\| f.deinit();
154		for (ws.time_report_msgs) \|msg\| gpa.free(msg);
155		gpa.free(ws.time_report_msgs);
156		gpa.free(ws.time_report_update_times);
157
158		if (ws.serve_task) \|t\| {
159		if (ws.tcp_server) \|*s\| s.stream.close(io);
160		t.await();
161		}
162		if (ws.tcp_server) \|*s\| s.deinit();
163
164		gpa.free(ws.step_names_trailing);
165		}
166	190	pub fn start(ws: *WebServer) error{AlreadyReported}!void {
167	191	assert(ws.tcp_server == null);
168	192	assert(ws.serve_task == null);
169		const maker = ws.maker;
170		const io = maker.graph.io;
	193	const graph = ws.graph;
	194	const io = graph.io;
171	195
172	196	ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch \|err\| {
173	197	log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err });
...	...	@@ -186,8 +210,8 @@ pub fn start(ws: *WebServer) error{AlreadyReported}!void {
186	210	}
187	211	}
188	212	fn serve(ws: *WebServer) Io.Cancelable!void {
189		const maker = ws.maker;
190		const io = maker.graph.io;
	213	const graph = ws.graph;
	214	const io = graph.io;
191	215
192	216	var group: Io.Group = .init;
193	217	defer group.cancel(io);
...	...	@@ -213,7 +237,8 @@ pub fn startBuild(ws: *WebServer) void {
213	237	fuzz.deinit();
214	238	ws.fuzz = null;
215	239	}
216		for (ws.step_status_bits) \|*bits\| @atomicStore(u8, bits, 0, .monotonic);
	240	const configured = &ws.configured.?;
	241	for (configured.step_status_bits) \|*bits\| @atomicStore(u8, bits, 0, .monotonic);
217	242	ws.build_status.store(.running, .monotonic);
218	243	ws.notifyUpdate();
219	244	}
...	...	@@ -223,12 +248,13 @@ pub fn updateStepStatus(
223	248	step_index: Configuration.Step.Index,
224	249	new_status: abi.StepUpdate.Status,
225	250	) void {
226		const maker = ws.maker;
	251	const configured = &ws.configured.?;
	252	const maker = configured.maker;
227	253	const all_steps = maker.step_stack.keys();
228	254	const step_idx: u32 = for (all_steps, 0..) \|s, i\| {
229	255	if (s == step_index) break @intCast(i);
230	256	} else unreachable;
231		const ptr = &ws.step_status_bits[step_idx / 4];
	257	const ptr = &configured.step_status_bits[step_idx / 4];
232	258	const bit_offset: u3 = @intCast((step_idx % 4) * 2);
233	259	const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset);
234	260	const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset;
...	...	@@ -239,7 +265,8 @@ pub fn updateStepStatus(
239	265	pub fn finishBuild(ws: *WebServer, opts: struct {
240	266	fuzz: bool,
241	267	}) void {
242		const maker = ws.maker;
	268	const configured = &ws.configured.?;
	269	const maker = configured.maker;
243	270	const all_steps = maker.step_stack.keys();
244	271
245	272	if (opts.fuzz) {
...	...	@@ -274,15 +301,15 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
274	301	}
275	302
276	303	pub fn now(ws: *const WebServer) i64 {
277		const maker = ws.maker;
278		const io = maker.graph.io;
	304	const graph = ws.graph;
	305	const io = graph.io;
279	306	const ts = base_clock.now(io);
280	307	return @intCast(ws.base_timestamp.durationTo(ts).toNanoseconds());
281	308	}
282	309
283	310	fn accept(ws: *WebServer, stream: net.Stream) void {
284		const maker = ws.maker;
285		const io = maker.graph.io;
	311	const graph = ws.graph;
	312	const io = graph.io;
286	313
287	314	defer {
288	315	// `net.Stream.close` wants to helpfully overwrite `stream` with
...	...	@@ -328,17 +355,19 @@ fn accept(ws: *WebServer, stream: net.Stream) void {
328	355	}
329	356
330	357	fn serveWebSocket(ws: WebServer, sock: http.Server.WebSocket) !noreturn {
331		const maker = ws.maker;
332		const gpa = maker.gpa;
333		const graph = maker.graph;
	358	const graph = ws.graph;
	359	const gpa = graph.cache.gpa;
334	360	const io = graph.io;
	361	log.err("TODO serve a different message when the configuration changes", .{});
	362	const configured = &ws.configured.?;
	363	const maker = configured.maker;
335	364	const all_steps = maker.step_stack.keys();
336	365
337	366	var prev_build_status = ws.build_status.load(.monotonic);
338	367
339		const prev_step_status_bits = try gpa.alloc(u8, ws.step_status_bits.len);
	368	const prev_step_status_bits = try gpa.alloc(u8, configured.step_status_bits.len);
340	369	defer gpa.free(prev_step_status_bits);
341		for (prev_step_status_bits, ws.step_status_bits) \|copy, shared\| {
	370	for (prev_step_status_bits, configured.step_status_bits) \|copy, shared\| {
342	371	copy.* = @atomicLoad(u8, shared, .monotonic);
343	372	}
344	373
...	...	@@ -354,7 +383,7 @@ fn serveWebSocket(ws: WebServer, sock: http.Server.WebSocket) !noreturn {
354	383	.timestamp = ws.now(),
355	384	.steps_len = @intCast(all_steps.len),
356	385	};
357		var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits };
	386	var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), configured.step_names_trailing, prev_step_status_bits };
358	387	try sock.writeMessageVec(&bufs, .binary);
359	388	}
360	389
...	...	@@ -369,17 +398,17 @@ fn serveWebSocket(ws: WebServer, sock: http.Server.WebSocket) !noreturn {
369	398	}
370	399
371	400	{
372		try ws.time_report_mutex.lock(io);
373		defer ws.time_report_mutex.unlock(io);
374		for (ws.time_report_msgs, ws.time_report_update_times) \|msg, update_time\| {
	401	try configured.time_report_mutex.lock(io);
	402	defer configured.time_report_mutex.unlock(io);
	403	for (configured.time_report_msgs, configured.time_report_update_times) \|msg, update_time\| {
375	404	if (update_time <= prev_time) continue;
376		// We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so
	405	// We want to send `msg`, but shouldn't block `configured.time_report_mutex` while we do, so
377	406	// that we don't hold up the build system on the client accepting this packet.
378	407	const owned_msg = try gpa.dupe(u8, msg);
379	408	defer gpa.free(owned_msg);
380	409	// Temporarily unlock, then re-lock after the message is sent.
381		ws.time_report_mutex.unlock(io);
382		defer ws.time_report_mutex.lockUncancelable(io);
	410	configured.time_report_mutex.unlock(io);
	411	defer configured.time_report_mutex.lockUncancelable(io);
383	412	try sock.writeMessage(owned_msg, .binary);
384	413	}
385	414	}
...	...	@@ -393,7 +422,7 @@ fn serveWebSocket(ws: WebServer, sock: http.Server.WebSocket) !noreturn {
393	422	}
394	423	}
395	424
396		for (prev_step_status_bits, ws.step_status_bits, 0..) \|prev_byte, shared, byte_idx\| {
	425	for (prev_step_status_bits, configured.step_status_bits, 0..) \|prev_byte, shared, byte_idx\| {
397	426	const cur_byte = @atomicLoad(u8, shared, .monotonic);
398	427	if (prev_byte.* == cur_byte) continue;
399	428	const cur: [4]abi.StepUpdate.Status = .{
...	...	@@ -433,8 +462,8 @@ fn serveWebSocket(ws: WebServer, sock: http.Server.WebSocket) !noreturn {
433	462	}
434	463	}
435	464	fn recvWebSocketMessages(ws: WebServer, sock: http.Server.WebSocket) void {
436		const maker = ws.maker;
437		const io = maker.graph.io;
	465	const graph = ws.graph;
	466	const io = graph.io;
438	467
439	468	while (true) {
440	469	const msg = sock.readSmallMessage() catch return;
...	...	@@ -492,8 +521,7 @@ fn serveLibFile(
492	521	sub_path: []const u8,
493	522	content_type: []const u8,
494	523	) !void {
495		const maker = ws.maker;
496		const graph = maker.graph;
	524	const graph = ws.graph;
497	525
498	526	return serveFile(ws, request, .{
499	527	.root_dir = graph.zig_lib_directory,
...	...	@@ -505,7 +533,7 @@ fn serveClientWasm(
505	533	req: *http.Server.Request,
506	534	optimize_mode: std.builtin.OptimizeMode,
507	535	) !void {
508		const gpa = ws.maker.gpa;
	536	const gpa = ws.graph.cache.gpa;
509	537
510	538	var arena_state: std.heap.ArenaAllocator = .init(gpa);
511	539	defer arena_state.deinit();
...	...	@@ -522,9 +550,9 @@ pub fn serveFile(
522	550	path: Cache.Path,
523	551	content_type: []const u8,
524	552	) !void {
525		const maker = ws.maker;
526		const gpa = ws.maker.gpa;
527		const io = maker.graph.io;
	553	const graph = ws.graph;
	554	const gpa = graph.cache.gpa;
	555	const io = graph.io;
528	556
529	557	// The desired API is actually sendfile, which will require enhancing http.Server.
530	558	// We load the file with every request so that the user can make changes to the file
...	...	@@ -542,8 +570,7 @@ pub fn serveFile(
542	570	});
543	571	}
544	572	pub fn serveTarFile(ws: WebServer, request: http.Server.Request, paths: []const Cache.Path) !void {
545		const maker = ws.maker;
546		const graph = maker.graph;
	573	const graph = ws.graph;
547	574	const io = graph.io;
548	575
549	576	var send_buffer: [0x4000]u8 = undefined;
...	...	@@ -581,9 +608,8 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
581	608	const arch_os_abi = "wasm32-freestanding";
582	609	const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
583	610
584		const maker = ws.maker;
585		const gpa = maker.gpa;
586		const graph = maker.graph;
	611	const graph = ws.graph;
	612	const gpa = graph.cache.gpa;
587	613	const io = graph.io;
588	614
589	615	const main_src_path: Cache.Path = .{
...	...	@@ -622,151 +648,19 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim
622	648	"--listen=-",
623	649	});
624	650
625		var child = try std.process.spawn(io, .{
626		.argv = argv.items,
627		.environ_map = &graph.environ_map,
628		.stdin = .pipe,
629		.stdout = .pipe,
630		.stderr = .pipe,
631		});
632		defer child.kill(io);
633
634		var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited });
635		defer if (stderr_task.cancel(io)) \|slice\| gpa.free(slice) else \|_\| {};
636
637		var stdout_buffer: [512]u8 = undefined;
638		var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer);
639		const stdout = &stdout_reader.interface;
640
641		{
642		var w = child.stdin.?.writer(io, &.{});
643		w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch \|err\| switch (err) {
644		error.WriteFailed => return w.err.?,
645		};
646		w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch \|err\| switch (err) {
647		error.WriteFailed => return w.err.?,
648		};
649		}
650
651		const Header = std.zig.Server.Message.Header;
652
653		var result: ?Cache.Path = null;
654		var result_error_bundle = std.zig.ErrorBundle.empty;
655		var body_buffer: std.ArrayList(u8) = .empty;
656		defer body_buffer.deinit(gpa);
	651	const compile_prog_node = ws.root_prog_node.start("Compile WebAssembly Component", 0);
	652	defer compile_prog_node.end();
657	653
658		while (true) {
659		const header = stdout.takeStruct(Header, .little) catch \|err\| switch (err) {
660		error.ReadFailed => \|e\| return e,
661		error.EndOfStream => break,
662		};
663		body_buffer.clearRetainingCapacity();
664		try stdout.appendExact(gpa, &body_buffer, header.bytes_len);
665		const body = body_buffer.items;
666
667		switch (header.tag) {
668		.zig_version => {
669		if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
670		return error.ZigProtocolVersionMismatch;
671		}
672		},
673		.error_bundle => {
674		result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body);
675		},
676		.emit_digest => {
677		const EmitDigest = std.zig.Server.Message.EmitDigest;
678		const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body);
679		if (!ebp_hdr.flags.cache_hit) {
680		log.info("source changes detected; rebuilt wasm component", .{});
681		}
682		const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
683		result = .{
684		.root_dir = graph.global_cache_root,
685		.sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
686		};
687		},
688		else => {}, // ignore other messages
689		}
690		}
691
692		const stderr_contents = try stderr_task.await(io);
693		if (stderr_contents.len > 0) {
694		std.debug.print("{s}", .{stderr_contents});
695		}
696
697		// Send EOF to stdin.
698		child.stdin.?.close(io);
699		child.stdin = null;
700
701		switch (try child.wait(io)) {
702		.exited => \|code\| {
703		if (code != 0) {
704		log.err(
705		"the following command exited with error code {d}:\n{s}",
706		.{ code, try std.zig.allocPrintCmd(arena, argv.items, .{}) },
707		);
708		return error.WasmCompilationFailed;
709		}
710		},
711		.signal => \|sig\| {
712		log.err(
713		"the following command terminated with signal {t}:\n{s}",
714		.{ sig, try std.zig.allocPrintCmd(arena, argv.items, .{}) },
715		);
716		return error.WasmCompilationFailed;
717		},
718		.stopped => \|sig\| {
719		log.err(
720		"the following command stopped unexpectedly with signal {t}:\n{s}",
721		.{ sig, try std.zig.allocPrintCmd(arena, argv.items, .{}) },
722		);
723		return error.WasmCompilationFailed;
724		},
725		.unknown => {
726		log.err(
727		"the following command terminated unexpectedly:\n{s}",
728		.{try std.zig.allocPrintCmd(arena, argv.items, .{})},
729		);
730		return error.WasmCompilationFailed;
731		},
732		}
733
734		if (result_error_bundle.errorMessageCount() > 0) {
735		try result_error_bundle.renderToStderr(io, .{}, .auto);
736		log.err("the following command failed with {d} compilation errors:\n{s}", .{
737		result_error_bundle.errorMessageCount(),
738		try std.zig.allocPrintCmd(arena, argv.items, .{}),
739		});
740		return error.WasmCompilationFailed;
741		}
742
743		const base_path = result orelse {
744		log.err("child process failed to report result\n{s}", .{
745		try std.zig.allocPrintCmd(arena, argv.items, .{}),
746		});
747		return error.WasmCompilationFailed;
748		};
749		const target = std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{
	654	const result = try std.zig.buildExeSubprocess(gpa, io, .{
	655	.argv = argv.items,
	656	.cache_root = graph.global_cache_root,
	657	.root_name = root_name,
750	658	.arch_os_abi = arch_os_abi,
751	659	.cpu_features = cpu_features,
752		}) catch unreachable) catch unreachable;
753		const bin_name = try std.zig.binNameAlloc(arena, .{
754		.root_name = root_name,
755		.cpu_arch = target.cpu.arch,
756		.os_tag = target.os.tag,
757		.ofmt = target.ofmt,
758		.abi = target.abi,
759		.output_mode = .Exe,
	660	.progress_node = compile_prog_node,
760	661	});
761		return base_path.join(arena, bin_name);
762		}
763
764		fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 {
765		var file_reader: Io.File.Reader = .initStreaming(file, io, &.{});
766		return file_reader.interface.allocRemaining(gpa, limit) catch \|err\| switch (err) {
767		error.ReadFailed => return file_reader.err.?,
768		else => \|e\| return e,
769		};
	662	if (!result.cache_hit) log.info("source changes detected; rebuilt wasm component", .{});
	663	return result.path;
770	664	}
771	665
772	666	pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
...	...	@@ -783,9 +677,11 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
783	677	/// The trailing data of `abi.time_report.CompileResult`, except the step name.
784	678	trailing: []const u8,
785	679	}) void {
786		const maker = ws.maker;
	680	const graph = ws.graph;
	681	const io = graph.io;
	682	const configured = &ws.configured.?;
	683	const maker = configured.maker;
787	684	const gpa = maker.gpa;
788		const io = maker.graph.io;
789	685	const all_steps = maker.step_stack.keys();
790	686
791	687	const step_idx: u32 = for (all_steps, 0..) \|s, i\| {
...	...	@@ -793,10 +689,10 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
793	689	} else unreachable;
794	690
795	691	const old_buf = old: {
796		ws.time_report_mutex.lock(io) catch return;
797		defer ws.time_report_mutex.unlock(io);
798		const old = ws.time_report_msgs[step_idx];
799		ws.time_report_msgs[step_idx] = &.{};
	692	configured.time_report_mutex.lock(io) catch return;
	693	defer configured.time_report_mutex.unlock(io);
	694	const old = configured.time_report_msgs[step_idx];
	695	configured.time_report_msgs[step_idx] = &.{};
800	696	break :old old;
801	697	};
802	698	const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory");
...	...	@@ -816,19 +712,21 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
816	712	@memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing);
817	713
818	714	{
819		ws.time_report_mutex.lock(io) catch return;
820		defer ws.time_report_mutex.unlock(io);
821		assert(ws.time_report_msgs[step_idx].len == 0);
822		ws.time_report_msgs[step_idx] = buf;
823		ws.time_report_update_times[step_idx] = ws.now();
	715	configured.time_report_mutex.lock(io) catch return;
	716	defer configured.time_report_mutex.unlock(io);
	717	assert(configured.time_report_msgs[step_idx].len == 0);
	718	configured.time_report_msgs[step_idx] = buf;
	719	configured.time_report_update_times[step_idx] = ws.now();
824	720	}
825	721	ws.notifyUpdate();
826	722	}
827	723
828	724	pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void {
829		const maker = ws.maker;
	725	const graph = ws.graph;
	726	const io = graph.io;
	727	const configured = &ws.configured.?;
	728	const maker = configured.maker;
830	729	const gpa = maker.gpa;
831		const io = maker.graph.io;
832	730	const all_steps = maker.step_stack.keys();
833	731
834	732	const step_idx: u32 = for (all_steps, 0..) \|s, i\| {
...	...	@@ -836,10 +734,10 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.In
836	734	} else unreachable;
837	735
838	736	const old_buf = old: {
839		ws.time_report_mutex.lock(io) catch return;
840		defer ws.time_report_mutex.unlock(io);
841		const old = ws.time_report_msgs[step_idx];
842		ws.time_report_msgs[step_idx] = &.{};
	737	configured.time_report_mutex.lock(io) catch return;
	738	defer configured.time_report_mutex.unlock(io);
	739	const old = configured.time_report_msgs[step_idx];
	740	configured.time_report_msgs[step_idx] = &.{};
843	741	break :old old;
844	742	};
845	743	const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory");
...	...	@@ -849,11 +747,11 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.In
849	747	.ns_total = @intCast(duration.toNanoseconds()),
850	748	};
851	749	{
852		ws.time_report_mutex.lock(io) catch return;
853		defer ws.time_report_mutex.unlock(io);
854		assert(ws.time_report_msgs[step_idx].len == 0);
855		ws.time_report_msgs[step_idx] = buf;
856		ws.time_report_update_times[step_idx] = ws.now();
	750	configured.time_report_mutex.lock(io) catch return;
	751	defer configured.time_report_mutex.unlock(io);
	752	assert(configured.time_report_msgs[step_idx].len == 0);
	753	configured.time_report_msgs[step_idx] = buf;
	754	configured.time_report_update_times[step_idx] = ws.now();
857	755	}
858	756	ws.notifyUpdate();
859	757	}
...	...	@@ -864,9 +762,11 @@ pub fn updateTimeReportRunTest(
864	762	tests: *const Step.Run.CachedTestMetadata,
865	763	ns_per_test: []const u64,
866	764	) void {
867		const maker = ws.maker;
	765	const graph = ws.graph;
	766	const io = graph.io;
	767	const configured = &ws.configured.?;
	768	const maker = configured.maker;
868	769	const gpa = maker.gpa;
869		const io = maker.graph.io;
870	770	const all_steps = maker.step_stack.keys();
871	771
872	772	const step_idx: u32 = for (all_steps, 0..) \|s, i\| {
...	...	@@ -884,10 +784,10 @@ pub fn updateTimeReportRunTest(
884	784	break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len;
885	785	};
886	786	const old_buf = old: {
887		ws.time_report_mutex.lock(io) catch return;
888		defer ws.time_report_mutex.unlock(io);
889		const old = ws.time_report_msgs[step_idx];
890		ws.time_report_msgs[step_idx] = &.{};
	787	configured.time_report_mutex.lock(io) catch return;
	788	defer configured.time_report_mutex.unlock(io);
	789	const old = configured.time_report_msgs[step_idx];
	790	configured.time_report_msgs[step_idx] = &.{};
891	791	break :old old;
892	792	};
893	793	const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory");
...	...	@@ -910,11 +810,11 @@ pub fn updateTimeReportRunTest(
910	810	assert(offset == buf.len);
911	811
912	812	{
913		ws.time_report_mutex.lock(io) catch return;
914		defer ws.time_report_mutex.unlock(io);
915		assert(ws.time_report_msgs[step_idx].len == 0);
916		ws.time_report_msgs[step_idx] = buf;
917		ws.time_report_update_times[step_idx] = ws.now();
	813	configured.time_report_mutex.lock(io) catch return;
	814	defer configured.time_report_mutex.unlock(io);
	815	assert(configured.time_report_msgs[step_idx].len == 0);
	816	configured.time_report_msgs[step_idx] = buf;
	817	configured.time_report_update_times[step_idx] = ws.now();
918	818	}
919	819	ws.notifyUpdate();
920	820	}
...	...	@@ -923,7 +823,7 @@ const RunnerRequest = union(enum) {
923	823	rebuild,
924	824	};
925	825	pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest {
926		const io = ws.maker.graph.io;
	826	const io = ws.graph.io;
927	827	ws.runner_request_mutex.lock(io) catch return;
928	828	defer ws.runner_request_mutex.unlock(io);
929	829	if (ws.runner_request) \|req\| {
...	...	@@ -934,7 +834,7 @@ pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest {
934	834	return null;
935	835	}
936	836	pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest {
937		const io = ws.maker.graph.io;
	837	const io = ws.graph.io;
938	838	try ws.runner_request_mutex.lock(io);
939	839	defer ws.runner_request_mutex.unlock(io);
940	840	while (true) {

lib/compiler/configurer.zig+32-1

...	...	@@ -133,7 +133,7 @@ pub fn main(init: process.Init.Minimal) !void {
133	133	.off => .no_color,
134	134	};
135	135
136		try builder.runBuild(root);
	136	builder.runBuild(root);
137	137
138	138	if (builder.validateUserInputDidItFail()) {
139	139	fatal(" access the help menu with 'zig build -h'", .{});
...	...	@@ -632,6 +632,37 @@ fn serialize(b: std.Build, wc: Configuration.Wip, writer: *Io.Writer) !void {
632	632
633	633	var s: Serialize = .{ .wc = wc, .arena = arena };
634	634
	635	try wc.path_deps.ensureTotalCapacityPrecise(gpa, graph.configure_dependencies.items.len);
	636	for (
	637	graph.configure_dependencies.items,
	638	wc.path_deps.addManyAsSliceAssumeCapacity(graph.configure_dependencies.items.len),
	639	) \|src, *dest\| {
	640	dest.* = .{
	641	.flags = .{
	642	.base = switch (src.lazy_path) {
	643	.src_path, .dependency => .build_root,
	644	.generated => unreachable,
	645	.cwd_relative => .cwd,
	646	.relative => \|r\| r.base,
	647	},
	648	.mode = src.mode,
	649	},
	650	.sub = switch (src.lazy_path) {
	651	.src_path => \|sp\| try wc.addString(sp.sub_path),
	652	.generated => unreachable,
	653	.cwd_relative => \|sub_path\| try wc.addString(sub_path),
	654	.dependency => \|d\| try wc.addString(d.sub_path),
	655	.relative => \|r\| try wc.addString(r.sub_path),
	656	},
	657	.pkg = switch (src.lazy_path) {
	658	.src_path => \|sp\| .init(try s.builderToPackage(sp.owner)),
	659	.generated => unreachable,
	660	.cwd_relative, .relative => .none,
	661	.dependency => \|d\| .init(try s.builderToPackage(d.dependency.builder)),
	662	},
	663	};
	664	}
	665
635	666	// Starting from all top-level steps in `b`, traverse the entire step graph
636	667	// and add all step dependencies implied by module graphs.
637	668	const top_level_steps = b.top_level_steps.values();

lib/compiler/libc.zig deleted-140

...	...	@@ -1,140 +0,0 @@
1		const std = @import("std");
2		const Io = std.Io;
3		const mem = std.mem;
4		const LibCInstallation = std.zig.LibCInstallation;
5
6		const usage_libc =
7		\\Usage: zig libc
8		\\
9		\\ Detect the native libc installation and print the resulting
10		\\ paths to stdout. You can save this into a file and then edit
11		\\ the paths to create a cross compilation libc kit. Then you
12		\\ can pass `--libc [file]` for Zig to use it.
13		\\
14		\\Usage: zig libc [paths_file]
15		\\
16		\\ Parse a libc installation text file and validate it.
17		\\
18		\\Options:
19		\\ -h, --help Print this help and exit
20		\\ -target [name] <arch><sub>-<os>-<abi> see the targets command
21		\\ -includes Print the libc include directories for the target
22		\\
23		;
24
25		var stdout_buffer: [4096]u8 = undefined;
26
27		pub fn main(init: std.process.Init) !void {
28		const arena = init.arena.allocator();
29		const gpa = init.gpa;
30		const io = init.io;
31		const args = try init.minimal.args.toSlice(arena);
32		const environ_map = init.environ_map;
33
34		const zig_lib_directory = args[1];
35
36		var input_file: ?[]const u8 = null;
37		var target_arch_os_abi: []const u8 = "native";
38		var print_includes: bool = false;
39		var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
40		const stdout = &stdout_writer.interface;
41		{
42		var i: usize = 2;
43		while (i < args.len) : (i += 1) {
44		const arg = args[i];
45		if (mem.startsWith(u8, arg, "-")) {
46		if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
47		try stdout.writeAll(usage_libc);
48		try stdout.flush();
49		return std.process.cleanExit(io);
50		} else if (mem.eql(u8, arg, "-target")) {
51		if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
52		i += 1;
53		target_arch_os_abi = args[i];
54		} else if (mem.eql(u8, arg, "-includes")) {
55		print_includes = true;
56		} else {
57		fatal("unrecognized parameter: '{s}'", .{arg});
58		}
59		} else if (input_file != null) {
60		fatal("unexpected extra parameter: '{s}'", .{arg});
61		} else {
62		input_file = arg;
63		}
64		}
65		}
66
67		const target_query = std.zig.parseTargetQueryOrReportFatalError(gpa, .{
68		.arch_os_abi = target_arch_os_abi,
69		});
70		const target = std.zig.resolveTargetQueryOrFatal(io, target_query);
71
72		if (print_includes) {
73		const libc_installation: ?*LibCInstallation = libc: {
74		if (input_file) \|libc_file\| {
75		const libc = try arena.create(LibCInstallation);
76		libc.* = LibCInstallation.parse(arena, io, libc_file, &target) catch \|err\| {
77		fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err });
78		};
79		break :libc libc;
80		} else {
81		break :libc null;
82		}
83		};
84
85		const is_native_abi = target_query.isNativeAbi();
86
87		const libc_dirs = std.zig.LibCDirs.detect(
88		arena,
89		io,
90		zig_lib_directory,
91		&target,
92		is_native_abi,
93		true,
94		libc_installation,
95		environ_map,
96		) catch \|err\| {
97		const zig_target = try target.zigTriple(arena);
98		fatal("unable to detect libc for target {s}: {t}", .{ zig_target, err });
99		};
100
101		if (libc_dirs.libc_include_dir_list.len == 0) {
102		const zig_target = try target.zigTriple(arena);
103		fatal("no include dirs detected for target {s}", .{zig_target});
104		}
105
106		for (libc_dirs.libc_include_dir_list) \|include_dir\| {
107		try stdout.writeAll(include_dir);
108		try stdout.writeByte('\n');
109		}
110		try stdout.flush();
111		return std.process.cleanExit(io);
112		}
113
114		if (input_file) \|libc_file\| {
115		var libc = LibCInstallation.parse(gpa, io, libc_file, &target) catch \|err\| {
116		fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err });
117		};
118		defer libc.deinit(gpa);
119		} else {
120		if (!target_query.canDetectLibC()) {
121		fatal("unable to detect libc for non-native target", .{});
122		}
123		var libc = LibCInstallation.findNative(gpa, io, .{
124		.verbose = true,
125		.target = &target,
126		.environ_map = environ_map,
127		}) catch \|err\| {
128		fatal("unable to detect native libc: {t}", .{err});
129		};
130		defer libc.deinit(gpa);
131
132		try libc.render(stdout);
133		try stdout.flush();
134		}
135		}
136
137		fn fatal(comptime format: []const u8, args: anytype) noreturn {
138		std.log.err(format, args);
139		std.process.exit(1);
140		}

lib/compiler/resinator/main.zig+3-3

...	...	@@ -44,7 +44,7 @@ pub fn main(init: std.process.Init.Minimal) !void {
44	44	try renderErrorMessage(stderr.terminal(), .err, "expected zig lib dir as first argument", .{});
45	45	std.process.exit(1);
46	46	}
47		const zig_lib_dir = args[1];
	47	const zig_lib_dir = std.mem.cutPrefix(u8, args[1], "--zig-lib=") orelse @panic("bad --zig-lib= arg");
48	48	var cli_args = args[2..];
49	49
50	50	var zig_integration = false;
...	...	@@ -640,7 +640,7 @@ fn getIncludePaths(
640	640	};
641	641	const target = std.zig.resolveTargetQueryOrFatal(io, target_query);
642	642	const is_native_abi = target_query.isNativeAbi();
643		const detected_libc = std.zig.LibCDirs.detect(arena, io, zig_lib_dir, &target, is_native_abi, true, null, environ_map) catch {
	643	const detected_libc = std.zig.LibCDirs.detect(arena, io, .{ .root_dir = .cwd(), .sub_path = zig_lib_dir }, &target, is_native_abi, true, null, environ_map) catch {
644	644	if (includes == .any) {
645	645	// fall back to mingw
646	646	includes = .gnu;
...	...	@@ -669,7 +669,7 @@ fn getIncludePaths(
669	669	const detected_libc = std.zig.LibCDirs.detect(
670	670	arena,
671	671	io,
672		zig_lib_dir,
	672	.{ .root_dir = .cwd(), .sub_path = zig_lib_dir },
673	673	&target,
674	674	is_native_abi,
675	675	true,

lib/compiler/std-docs.zig+3-3

...	...	@@ -29,9 +29,9 @@ pub fn main(init: std.process.Init) !void {
29	29	var argv = try init.minimal.args.iterateAllocator(arena);
30	30	defer argv.deinit();
31	31	assert(argv.skip());
32		const zig_lib_directory = argv.next().?;
33		const zig_exe_path = argv.next().?;
34		const global_cache_path = argv.next().?;
	32	const zig_lib_directory = mem.cutPrefix(u8, argv.next().?, "--zig-lib=") orelse @panic("bad --zig-lib= arg");
	33	const zig_exe_path = mem.cutPrefix(u8, argv.next().?, "--zig=") orelse @panic("bad --zig= arg");
	34	const global_cache_path = mem.cutPrefix(u8, argv.next().?, "--global-cache=") orelse @panic("bad --global-cache= arg");
35	35
36	36	var lib_dir = try Io.Dir.cwd().openDir(io, zig_lib_directory, .{});
37	37	defer lib_dir.close(io);

lib/std/Build.zig+145-41

...	...	@@ -64,6 +64,11 @@ pkg_hash: []const u8,
64	64	/// A mapping from dependency names to package hashes.
65	65	available_deps: AvailableDeps,
66	66
	67	pub const ConfigureDependency = struct {
	68	lazy_path: LazyPath,
	69	mode: std.Build.Configuration.PathDep.Mode,
	70	};
	71
67	72	pub const ReleaseMode = enum {
68	73	off,
69	74	any,
...	...	@@ -102,6 +107,12 @@ pub const Graph = struct {
102	107	/// Observing this data causes cache poisoning. See `CachePoison`.
103	108	search_prefixes: std.ArrayList([]const u8) = .empty,
104	109
	110	/// Populated by calling one of:
	111	/// * `dependOnFileContents`
	112	/// * `dependOnFileMetadata`
	113	/// * `dependOnDirectory`
	114	configure_dependencies: ArrayList(ConfigureDependency) = .empty,
	115
105	116	/// If the cache is poisoned means that the configure logic had side
106	117	/// effects, or otherwise did something that could not be tracked by the
107	118	/// cache system.
...	...	@@ -165,7 +176,8 @@ pub const Graph = struct {
165	176
166	177	/// A path whose components and contents are known at some point during
167	178	/// `Step` resolution, relative to the provided base directory.
168		pub fn path(graph: *Graph, base: Configuration.Path.Base, sub_path: []const u8) LazyPath {
	179	pub fn path(graph: *Graph, base: Configuration.LazyPath.Relative.Base, sub_path: []const u8) LazyPath {
	180	assert(base != .build_root);
169	181	return .{ .relative = .{
170	182	.base = base,
171	183	.sub_path = @This().dupePath(graph, sub_path),
...	...	@@ -204,6 +216,9 @@ pub const Graph = struct {
204	216	/// did something that could not be tracked by the cache system.
205	217	///
206	218	/// See `CachePoison` documentation for more details.
	219	///
	220	/// As an alternative to calling this function, consider these APIs instead:
	221	/// * `dependOnFileContents`
207	222	pub fn poisonCache(graph: *Graph) void {
208	223	switch (graph.cache_poison) {
209	224	.pure => graph.cache_poison = .poisoned,
...	...	@@ -290,6 +305,7 @@ const UserValue = union(enum) {
290	305	lazy_path_list: std.array_list.Managed(LazyPath),
291	306	};
292	307
	308	/// Build system implementation detail.
293	309	pub fn create(
294	310	graph: *Graph,
295	311	root: Cache.Path,
...	...	@@ -681,9 +697,9 @@ fn hashUserInputOptionsMap(allocator: Allocator, user_input_options: UserInputOp
681	697
682	698	/// Create a set of key-value pairs that can be converted into a Zig source
683	699	/// file and then inserted into a Zig compilation's module table for importing.
684		/// In other words, this provides a way to expose build.zig values to Zig
685		/// source code with `@import`.
686		/// Related: `Module.addOptions`.
	700	///
	701	/// This provides a way to expose build.zig values to Zig source code with
	702	/// `@import`. Related: `Module.addOptions`.
687	703	pub fn addOptions(b: Build) Step.Options {
688	704	return Step.Options.create(b);
689	705	}
...	...	@@ -837,10 +853,7 @@ pub fn addModule(b: Build, name: []const u8, options: Module.CreateOptions) Mo
837	853	module,
838	854	) catch @panic("OOM");
839	855	if (gop.found_existing) {
840		panic(
841		"A module with the name '{s}' has already been added to the package. Consider creating a private module with std.Build.createModule",
842		.{name},
843		);
	856	panic("A module with the name {q} has already been added to the package. Consider creating a private module with std.Build.createModule", .{name});
844	857	}
845	858	return module;
846	859	}
...	...	@@ -970,6 +983,7 @@ pub fn addConfigHeader(
970	983	return config_header_step;
971	984	}
972	985
	986	/// Deprecated, call `Graph.dupeString` instead.
973	987	pub fn dupe(b: *Build, bytes: []const u8) []const u8 {
974	988	return b.graph.dupeString(bytes);
975	989	}
...	...	@@ -1000,7 +1014,7 @@ pub fn addNamedWriteFiles(b: Build, name: []const u8) Step.WriteFile {
1000	1014	) catch @panic("OOM");
1001	1015	if (gop.found_existing) {
1002	1016	panic(
1003		"A WriteFile step with the name '{s}' has already been added to the package. Consider creating a private WriteFile step with std.Build.addWriteFiles",
	1017	"A WriteFile step with the name {q} has already been added to the package. Consider creating a private WriteFile step with std.Build.addWriteFiles",
1004	1018	.{name},
1005	1019	);
1006	1020	}
...	...	@@ -1015,10 +1029,7 @@ pub fn addNamedLazyPath(b: *Build, name: []const u8, lp: LazyPath) void {
1015	1029	lp.dupe(graph),
1016	1030	) catch @panic("OOM");
1017	1031	if (gop.found_existing) {
1018		panic(
1019		"A LazyPath with the name '{s}' has already been added to the package.",
1020		.{name},
1021		);
	1032	panic("A LazyPath with the name {q} has already been added to the package.", .{name});
1022	1033	}
1023	1034	}
1024	1035
...	...	@@ -1121,7 +1132,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw
1121	1132	.enum_options = enum_options,
1122	1133	};
1123	1134	if ((b.available_options_map.fetchPut(arena, name, available_option) catch @panic("OOM")) != null) {
1124		panic("option '{s}' declared twice", .{name});
	1135	panic("option {q} declared twice", .{name});
1125	1136	}
1126	1137
1127	1138	const option_ptr = b.user_input_options.getPtr(name) orelse return null;
...	...	@@ -1292,6 +1303,8 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw
1292	1303	}
1293	1304	}
1294	1305
	1306	/// Creates a top-level build step, exposed to the CLI user and advertised in
	1307	/// the "--help" menu.
1295	1308	pub fn step(b: Build, name: []const u8, description: []const u8) Step {
1296	1309	const graph = b.graph;
1297	1310	const arena = graph.arena;
...	...	@@ -1371,7 +1384,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile
1371	1384	opts_copy.diagnostics = &diags;
1372	1385	return std.Target.Query.parse(opts_copy) catch \|err\| switch (err) {
1373	1386	error.UnknownCpuModel => {
1374		std.debug.print("unknown CPU: '{s}'\navailable CPUs for architecture '{t}':\n", .{
	1387	std.debug.print("unknown CPU: {q}\navailable CPUs for architecture {t}:\n", .{
1375	1388	diags.cpu_name.?, diags.arch.?,
1376	1389	});
1377	1390	for (diags.arch.?.allCpuModels()) \|cpu\| {
...	...	@@ -1381,7 +1394,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile
1381	1394	},
1382	1395	error.UnknownCpuFeature => {
1383	1396	std.debug.print(
1384		\\unknown CPU feature: '{s}'
	1397	\\unknown CPU feature: {q}
1385	1398	\\available CPU features for architecture '{t}':
1386	1399	\\
1387	1400	, .{
...	...	@@ -1394,7 +1407,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile
1394	1407	},
1395	1408	error.UnknownOperatingSystem => {
1396	1409	std.debug.print(
1397		\\unknown OS: '{s}'
	1410	\\unknown OS: {q}
1398	1411	\\available operating systems:
1399	1412	\\
1400	1413	, .{diags.os_name.?});
...	...	@@ -1404,9 +1417,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile
1404	1417	return error.ParseFailed;
1405	1418	},
1406	1419	else => \|e\| {
1407		std.debug.print("unable to parse target '{s}': {s}\n", .{
1408		options.arch_os_abi, @errorName(e),
1409		});
	1420	std.debug.print("unable to parse target {q}: {t}\n", .{ options.arch_os_abi, e });
1410	1421	return error.ParseFailed;
1411	1422	},
1412	1423	};
...	...	@@ -1469,13 +1480,14 @@ pub fn standardTargetOptionsQueryOnly(b: *Build, args: StandardTargetOptionsArgs
1469	1480	q.serializeCpuAlloc(arena) catch @panic("OOM"),
1470	1481	});
1471	1482	}
1472		log.err("chosen target '{s}' does not match one of the allowed targets", .{
	1483	log.err("chosen target {q} does not match one of the allowed targets", .{
1473	1484	selected_target.zigTriple(arena) catch @panic("OOM"),
1474	1485	});
1475	1486	b.markInvalidUserInput();
1476	1487	return args.default_target;
1477	1488	}
1478	1489
	1490	/// Build system implementation detail.
1479	1491	pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8) error{OutOfMemory}!bool {
1480	1492	const graph = b.graph;
1481	1493	const arena = graph.arena;
...	...	@@ -1523,7 +1535,7 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8
1523	1535	return true;
1524	1536	},
1525	1537	.lazy_path, .lazy_path_list => {
1526		log.warn("the lazy path value type isn't added from the CLI, but somehow '{s}' is a .{f}", .{
	1538	log.warn("the lazy path value type isn't added from the CLI, but somehow {q} is a .{f}", .{
1527	1539	name, std.zig.fmtId(@tagName(gop.value_ptr.value)),
1528	1540	});
1529	1541	return true;
...	...	@@ -1532,6 +1544,7 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8
1532	1544	return false;
1533	1545	}
1534	1546
	1547	/// Build system implementation detail.
1535	1548	pub fn addUserInputFlag(b: *Build, name_raw: []const u8) error{OutOfMemory}!bool {
1536	1549	const graph = b.graph;
1537	1550	const name = graph.dupeString(name_raw);
...	...	@@ -1592,6 +1605,7 @@ fn markInvalidUserInput(b: *Build) void {
1592	1605	b.invalid_user_input = true;
1593	1606	}
1594	1607
	1608	/// Build system implementation detail.
1595	1609	pub fn validateUserInputDidItFail(b: *Build) bool {
1596	1610	// Make sure all args are used.
1597	1611	var it = b.user_input_options.iterator();
...	...	@@ -1689,9 +1703,7 @@ pub fn addCheckFile(
1689	1703	/// References a file or directory relative to the source root.
1690	1704	pub fn path(b: *Build, sub_path: []const u8) LazyPath {
1691	1705	if (fs.path.isAbsolute(sub_path)) {
1692		panic("sub_path is expected to be relative to the build root, but was this absolute path: '{s}'. Absolute paths can cause problems but can be created via Graph.cwdRelativePath", .{
1693		sub_path,
1694		});
	1706	panic("sub_path is expected to be relative to the build root, but was this absolute path: {q}. Absolute paths can cause problems but can be created via Graph.cwdRelativePath", .{sub_path});
1695	1707	}
1696	1708	return .{ .src_path = .{
1697	1709	.owner = b,
...	...	@@ -2012,34 +2024,34 @@ pub const Dependency = struct {
2012	2024	for (d.builder.install_tls.step.dependencies.items) \|dep_step\| {
2013	2025	const inst = dep_step.cast(Step.InstallArtifact) orelse continue;
2014	2026	if (mem.eql(u8, inst.artifact.name, name)) {
2015		if (found != null) panic("artifact name '{s}' is ambiguous", .{name});
	2027	if (found != null) panic("artifact name {q} is ambiguous", .{name});
2016	2028	found = inst.artifact;
2017	2029	}
2018	2030	}
2019	2031	return found orelse {
2020	2032	for (d.builder.install_tls.step.dependencies.items) \|dep_step\| {
2021	2033	const inst = dep_step.cast(Step.InstallArtifact) orelse continue;
2022		log.info("available artifact: '{s}'", .{inst.artifact.name});
	2034	log.info("available artifact: {q}", .{inst.artifact.name});
2023	2035	}
2024		panic("unable to find artifact '{s}'", .{name});
	2036	panic("unable to find artifact {q}", .{name});
2025	2037	};
2026	2038	}
2027	2039
2028	2040	pub fn module(d: Dependency, name: []const u8) Module {
2029	2041	return d.builder.modules.get(name) orelse {
2030		panic("unable to find module '{s}'", .{name});
	2042	panic("unable to find module {q}", .{name});
2031	2043	};
2032	2044	}
2033	2045
2034	2046	pub fn namedWriteFiles(d: Dependency, name: []const u8) Step.WriteFile {
2035	2047	return d.builder.named_writefiles.get(name) orelse {
2036		panic("unable to find named writefiles '{s}'", .{name});
	2048	panic("unable to find named writefiles {q}", .{name});
2037	2049	};
2038	2050	}
2039	2051
2040	2052	pub fn namedLazyPath(d: *Dependency, name: []const u8) LazyPath {
2041	2053	return d.builder.named_lazy_paths.get(name) orelse {
2042		panic("unable to find named lazypath '{s}'", .{name});
	2054	panic("unable to find named lazypath {q}", .{name});
2043	2055	};
2044	2056	}
2045	2057
...	...	@@ -2178,6 +2190,7 @@ pub inline fn lazyImport(
2178	2190	comptime unreachable; // Bad @dependencies source
2179	2191	}
2180	2192
	2193	/// Build system implementation detail.
2181	2194	pub fn dependencyFromBuildZig(
2182	2195	b: *Build,
2183	2196	/// The build.zig struct of the dependency, normally obtained by `@import` of the dependency.
...	...	@@ -2310,14 +2323,13 @@ fn dependencyInner(
2310	2323	.root_dir = .{
2311	2324	.path = build_root_string,
2312	2325	.handle = Io.Dir.cwd().openDir(io, build_root_string, .{}) catch \|err\|
2313		process.fatal("unable to open {s}: {t}", .{ build_root_string, err }),
	2326	process.fatal("failed to open {q}: {t}", .{ build_root_string, err }),
2314	2327	},
2315	2328	};
2316	2329
2317		const sub_builder = b.createChild(name, dep_root, pkg_hash, pkg_deps, user_input_options) catch
2318		@panic("unhandled error");
	2330	const sub_builder = b.createChild(name, dep_root, pkg_hash, pkg_deps, user_input_options) catch @panic("OOM");
2319	2331	if (build_zig) \|bz\| {
2320		sub_builder.runBuild(bz) catch @panic("unhandled error");
	2332	sub_builder.runBuild(bz);
2321	2333
2322	2334	if (sub_builder.validateUserInputDidItFail()) {
2323	2335	std.debug.dumpCurrentStackTrace(.{ .first_address = @returnAddress() });
...	...	@@ -2334,11 +2346,11 @@ fn dependencyInner(
2334	2346	return dep;
2335	2347	}
2336	2348
2337		pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void {
	2349	/// Build system implementation detail.
	2350	pub fn runBuild(b: *Build, build_zig: anytype) void {
2338	2351	switch (@typeInfo(@typeInfo(@TypeOf(build_zig.build)).@"fn".return_type.?)) {
2339		.void => build_zig.build(b),
2340		.error_union => try build_zig.build(b),
2341		else => @compileError("expected return type of build to be 'void' or '!void'"),
	2352	.error_union => return build_zig.build(b) catch unreachable,
	2353	else => return build_zig.build(b),
2342	2354	}
2343	2355	}
2344	2356
...	...	@@ -2402,7 +2414,7 @@ pub const LazyPath = union(enum) {
2402	2414	},
2403	2415
2404	2416	relative: struct {
2405		base: Configuration.Path.Base,
	2417	base: Configuration.LazyPath.Relative.Base,
2406	2418	sub_path: []const u8 = "",
2407	2419
2408	2420	pub fn eql(a: @This(), b: @This()) bool {
...	...	@@ -2607,7 +2619,7 @@ fn dumpBadDirnameHelp(
2607	2619
2608	2620	if (asking_step) \|as\| {
2609	2621	stderr.setColor(.red) catch {};
2610		try w.print(" The step '{s}' that is missing a dependency on the above step was created by this stack trace:\n", .{as.name});
	2622	try w.print(" The step {q} that is missing a dependency on the above step was created by this stack trace:\n", .{as.name});
2611	2623	stderr.setColor(.reset) catch {};
2612	2624
2613	2625	as.dump(stderr);
...	...	@@ -2708,9 +2720,101 @@ pub fn systemIntegrationOption(
2708	2720	}
2709	2721	}
2710	2722
	2723	/// Indicates that the build.zig logic depends on a particular file's contents.
	2724	///
	2725	/// If the file is created, deleted, or has its contents changed, the configure
	2726	/// phase will be repeated. If the inode or mtime change, but the file contents
	2727	/// remain the same, it will not cause the configure logic to be repeated.
	2728	///
	2729	/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
	2730	/// of `zig build` into a cache miss.
	2731	///
	2732	/// Only a subset of `LazyPath` are supported:
	2733	/// - Relative to cwd
	2734	/// - Relative to any package root
	2735	/// - Relative to zig cache or zig installation
	2736	///
	2737	/// If the file would be inside one of the search prefixes, then the dependency
	2738	/// cannot be tracked; `Graph.poisonCache` must be used instead.
	2739	pub fn dependOnFileContents(b: *Build, lazy_path: LazyPath) void {
	2740	validateConfigureDependency(lazy_path);
	2741	const graph = b.graph;
	2742	graph.configure_dependencies.append(graph.arena, .{
	2743	.lazy_path = lazy_path.dupe(graph),
	2744	.mode = .contents,
	2745	}) catch @panic("OOM");
	2746	}
	2747
	2748	/// Indicates that the build.zig logic depends on a particular file's size,
	2749	/// inode, mtime, and contents.
	2750	///
	2751	/// If the file is created, deleted, has its contents changed, or the inode
	2752	/// changes, or the mtime changes, the configure phase will be repeated.
	2753	///
	2754	/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
	2755	/// of `zig build` into a cache miss.
	2756	///
	2757	/// Only a subset of `LazyPath` are supported:
	2758	/// - Relative to cwd
	2759	/// - Relative to any package root
	2760	/// - Relative to zig cache or zig installation
	2761	///
	2762	/// If the file would be inside one of the search prefixes, then the dependency
	2763	/// cannot be tracked; `Graph.poisonCache` must be used instead.
	2764	pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void {
	2765	validateConfigureDependency(lazy_path);
	2766	const graph = b.graph;
	2767	graph.configure_dependencies.append(graph.arena, .{
	2768	.lazy_path = lazy_path.dupe(graph),
	2769	.mode = .metadata,
	2770	}) catch @panic("OOM");
	2771	}
	2772
	2773	/// Indicates that the build.zig logic depends on a particular directory's entries.
	2774	///
	2775	/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
	2776	/// of `zig build` into a cache miss.
	2777	///
	2778	/// If any file is created, deleted, or renamed in this directory, the
	2779	/// configure phase will be repeated.
	2780	///
	2781	/// Only a subset of `LazyPath` are supported:
	2782	/// - Relative to cwd
	2783	/// - Relative to any package root
	2784	/// - Relative to zig cache or zig installation
	2785	///
	2786	/// If the directory would be inside one of the search prefixes, then the dependency
	2787	/// cannot be tracked; `Graph.poisonCache` must be used instead.
	2788	pub fn dependOnDirectory(b: *Build, lazy_path: LazyPath) void {
	2789	validateConfigureDependency(lazy_path);
	2790	const graph = b.graph;
	2791	graph.configure_dependencies.append(graph.arena, .{
	2792	.lazy_path = lazy_path.dupe(graph),
	2793	.mode = .directory,
	2794	}) catch @panic("OOM");
	2795	}
	2796
	2797	fn validateConfigureDependency(lazy_path: LazyPath) void {
	2798	switch (lazy_path) {
	2799	.src_path, .cwd_relative, .dependency => {}, // OK
	2800	.generated => @panic("configure phase cannot depend on files generated during make phase"),
	2801	.relative => \|relative\| switch (relative.base) {
	2802	.cwd, .build_root, .local_cache, .global_cache, .zig_exe, .zig_lib => {}, // OK
	2803	.install_prefix,
	2804	.install_lib,
	2805	.install_bin,
	2806	.install_include,
	2807	=> @panic("configure phase cannot depend on files installed during make phase"),
	2808	},
	2809	}
	2810	}
	2811
2711	2812	test {
2712	2813	_ = Cache;
	2814	_ = Configuration;
	2815	_ = Module;
2713	2816	_ = Step;
2714	2817	_ = Configuration;
2715	2818	_ = &findProgram;
	2819	_ = abi;
2716	2820	}

lib/std/Build/Cache.zig+42-24

...	...	@@ -28,7 +28,7 @@ mutex: Io.Mutex = .init,
28	28	/// are replaced with single-character indicators. This is not to save
29	29	/// space but to eliminate absolute file paths. This improves portability
30	30	/// and usefulness of the cache for advanced use cases.
31		prefixes_buffer: [4]Directory = undefined,
	31	prefixes_buffer: [5]Directory = undefined,
32	32	prefixes_len: usize = 0,
33	33	/// Used to identify prefixes. References external memory.
34	34	cwd: []const u8,
...	...	@@ -57,7 +57,7 @@ pub fn prefixes(cache: *const Cache) []const Directory {
57	57	return cache.prefixes_buffer[0..cache.prefixes_len];
58	58	}
59	59
60		const PrefixedPath = struct {
	60	pub const PrefixedPath = struct {
61	61	prefix: u8,
62	62	sub_path: []const u8,
63	63
...	...	@@ -70,6 +70,15 @@ const PrefixedPath = struct {
70	70	}
71	71	};
72	72
	73	fn findPrefixPath(cache: *const Cache, path: Path) !PrefixedPath {
	74	const gpa = cache.gpa;
	75	const resolved_path = try std.fs.path.resolve(gpa, &.{
	76	cache.cwd, path.root_dir.path orelse ".", path.subPathOrDot(),
	77	});
	78	errdefer gpa.free(resolved_path);
	79	return findPrefixResolved(cache, resolved_path);
	80	}
	81
73	82	fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
74	83	const gpa = cache.gpa;
75	84	const resolved_path = try std.fs.path.resolve(gpa, &.{file_path});
...	...	@@ -91,13 +100,13 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
91	100	};
92	101	// Free the resolved path since we're not going to return it
93	102	gpa.free(resolved_path);
94		return PrefixedPath{
	103	return .{
95	104	.prefix = i,
96	105	.sub_path = sub_path,
97	106	};
98	107	}
99	108
100		return PrefixedPath{
	109	return .{
101	110	.prefix = 0,
102	111	.sub_path = resolved_path,
103	112	};
...	...	@@ -998,20 +1007,34 @@ pub const Manifest = struct {
998	1007	/// This is useful for processes that don't know the all the files that are
999	1008	/// depended on ahead of time. For example, a source file that can import
1000	1009	/// other files will need to be recompiled if the imported file is changed.
1001		pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
1002		assert(self.manifest_file != null);
	1010	pub fn addFilePost(man: *Manifest, file_path: []const u8) !void {
	1011	assert(man.manifest_file != null);
	1012	const gpa = man.cache.gpa;
	1013	const prefixed_path = try man.cache.findPrefix(file_path);
	1014	var keep = false;
	1015	defer if (!keep) gpa.free(prefixed_path.sub_path);
	1016	keep = try addPrefixedPathPost(man, prefixed_path);
	1017	}
1003	1018
1004		const gpa = self.cache.gpa;
1005		const prefixed_path = try self.cache.findPrefix(file_path);
1006		errdefer gpa.free(prefixed_path.sub_path);
	1019	pub fn addPathPost(man: *Manifest, path: Path) !void {
	1020	assert(man.manifest_file != null);
	1021	const gpa = man.cache.gpa;
	1022	const prefixed_path: PrefixedPath = try man.cache.findPrefixPath(path);
	1023	var keep = false;
	1024	defer if (!keep) gpa.free(prefixed_path.sub_path);
	1025	keep = try addPrefixedPathPost(man, prefixed_path);
	1026	}
1007	1027
1008		const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
1009		errdefer _ = self.files.pop();
	1028	/// Low level function. `prefixed_path` references cloned memory. Returns
	1029	/// whether or not `prefixed_path.sub_path` should be kept.
	1030	pub fn addPrefixedPathPost(man: *Manifest, prefixed_path: PrefixedPath) !bool {
	1031	assert(man.manifest_file != null);
	1032	const gpa = man.cache.gpa;
1010	1033
1011		if (gop.found_existing) {
1012		gpa.free(prefixed_path.sub_path);
1013		return;
1014		}
	1034	const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
	1035	errdefer _ = man.files.pop();
	1036
	1037	if (gop.found_existing) return false;
1015	1038
1016	1039	gop.key_ptr.* = .{
1017	1040	.prefixed_path = prefixed_path,
...	...	@@ -1022,16 +1045,11 @@ pub const Manifest = struct {
1022	1045	.contents = null,
1023	1046	};
1024	1047
1025		self.files.lockPointers();
1026		defer self.files.unlockPointers();
	1048	man.files.lockPointers();
	1049	defer man.files.unlockPointers();
1027	1050
1028		try self.populateFileHash(gop.key_ptr);
1029		}
1030
1031		pub fn addPathPost(man: *Manifest, path: Path) !void {
1032		_ = man;
1033		_ = path;
1034		@panic("TODO");
	1051	try man.populateFileHash(gop.key_ptr);
	1052	return true;
1035	1053	}
1036	1054
1037	1055	/// Like `addFilePost` but when the file contents have already been loaded from disk.

lib/std/Build/Configuration.zig+51-32

...	...	@@ -10,8 +10,7 @@ const native_endian = builtin.target.cpu.arch.endian();
10	10
11	11	string_bytes: []u8,
12	12	steps: []Step,
13		path_deps_base: []Path.Base,
14		path_deps_sub: []String,
	13	path_deps: []PathDep,
15	14	unlazy_deps: []String,
16	15	system_integrations: []SystemIntegration,
17	16	available_options: []AvailableOption,
...	...	@@ -57,7 +56,7 @@ pub const Wip = struct {
57	56	system_integrations: std.ArrayList(SystemIntegration) = .empty,
58	57	available_options: std.ArrayList(AvailableOption) = .empty,
59	58	steps: std.ArrayList(Step) = .empty,
60		path_deps: std.MultiArrayList(Path) = .empty,
	59	path_deps: std.ArrayList(PathDep) = .empty,
61	60	search_prefixes: std.ArrayList(String) = .empty,
62	61	extra: std.ArrayList(u32) = .empty,
63	62	next_generated_file_index: u32 = 0,
...	...	@@ -154,7 +153,7 @@ pub const Wip = struct {
154	153	const header: Header = .{
155	154	.string_bytes_len = @intCast(wip.string_bytes.items.len),
156	155	.steps_len = @intCast(wip.steps.items.len),
157		.path_deps_len = @intCast(wip.path_deps.len),
	156	.path_deps_len = @intCast(wip.path_deps.items.len),
158	157	.unlazy_deps_len = @intCast(wip.unlazy_deps.items.len),
159	158	.system_integrations_len = @intCast(wip.system_integrations.items.len),
160	159	.available_options_len = @intCast(wip.available_options.items.len),
...	...	@@ -171,8 +170,7 @@ pub const Wip = struct {
171	170	@ptrCast(&header),
172	171	wip.string_bytes.items,
173	172	@ptrCast(wip.steps.items),
174		@ptrCast(wip.path_deps.items(.base)),
175		@ptrCast(wip.path_deps.items(.sub)),
	173	@ptrCast(wip.path_deps.items),
176	174	@ptrCast(wip.unlazy_deps.items),
177	175	@ptrCast(wip.system_integrations.items),
178	176	@ptrCast(wip.available_options.items),
...	...	@@ -1551,9 +1549,23 @@ pub const LazyPath = union(@This().Tag) {
1551	1549
1552	1550	pub const Flags = packed struct(u32) {
1553	1551	tag: Tag = .relative,
1554		base: Path.Base,
	1552	base: Base,
1555	1553	_: u16 = 0,
1556	1554	};
	1555
	1556	pub const Base = enum(u8) {
	1557	cwd,
	1558	local_cache,
	1559	global_cache,
	1560	/// Must not be used with Relative since package index is missing.
	1561	build_root,
	1562	zig_exe,
	1563	zig_lib,
	1564	install_prefix,
	1565	install_lib,
	1566	install_bin,
	1567	install_include,
	1568	};
1557	1569	};
1558	1570	};
1559	1571
...	...	@@ -1597,6 +1609,26 @@ pub const Package = struct {
1597	1609	return package.dep_prefix.slice(c);
1598	1610	}
1599	1611	};
	1612
	1613	pub const OptionalIndex = enum(u32) {
	1614	none = max_u32 - 1,
	1615	root = max_u32,
	1616	_,
	1617
	1618	pub fn init(i: Index) OptionalIndex {
	1619	const result: OptionalIndex = @enumFromInt(@intFromEnum(i));
	1620	assert(result != .none);
	1621	return result;
	1622	}
	1623
	1624	pub fn unwrap(this: @This()) ?Index {
	1625	return switch (this) {
	1626	.none => null,
	1627	.root => .root,
	1628	_ => @enumFromInt(@intFromEnum(this)),
	1629	};
	1630	}
	1631	};
1600	1632	};
1601	1633
1602	1634	pub const Module = struct {
...	...	@@ -1833,29 +1865,18 @@ pub const OptionalStringList = enum(u32) {
1833	1865	}
1834	1866	};
1835	1867
1836		pub const Path = extern struct {
1837		base: Base,
	1868	pub const PathDep = extern struct {
	1869	flags: Flags,
1838	1870	sub: String,
	1871	pkg: Package.OptionalIndex,
1839	1872
1840		pub const Base = enum(u8) {
1841		cwd,
1842		local_cache,
1843		global_cache,
1844		build_root,
1845		zig_exe,
1846		zig_lib,
1847		install_prefix,
1848		install_lib,
1849		install_bin,
1850		install_include,
1851		};
1852
1853		pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path {
1854		_ = c;
1855		_ = arena;
1856		_ = path;
1857		@panic("TODO");
1858		}
	1873	pub const Flags = packed struct(u32) {
	1874	mode: Mode,
	1875	base: LazyPath.Relative.Base,
	1876	_: u16 = 0,
	1877	};
	1878
	1879	pub const Mode = enum(u8) { directory, contents, metadata };
1859	1880	};
1860	1881
1861	1882	pub const InstallDestDir = enum(u32) {
...	...	@@ -3430,8 +3451,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration {
3430	3451	const result: Configuration = .{
3431	3452	.string_bytes = try arena.alloc(u8, header.string_bytes_len),
3432	3453	.steps = try arena.alloc(Step, header.steps_len),
3433		.path_deps_sub = try arena.alloc(String, header.path_deps_len),
3434		.path_deps_base = try arena.alloc(Path.Base, header.path_deps_len),
	3454	.path_deps = try arena.alloc(PathDep, header.path_deps_len),
3435	3455	.unlazy_deps = try arena.alloc(String, header.unlazy_deps_len),
3436	3456	.system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len),
3437	3457	.available_options = try arena.alloc(AvailableOption, header.available_options_len),
...	...	@@ -3444,8 +3464,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration {
3444	3464	var vecs = [_][]u8{
3445	3465	result.string_bytes,
3446	3466	@ptrCast(result.steps),
3447		@ptrCast(result.path_deps_base),
3448		@ptrCast(result.path_deps_sub),
	3467	@ptrCast(result.path_deps),
3449	3468	@ptrCast(result.unlazy_deps),
3450	3469	@ptrCast(result.system_integrations),
3451	3470	@ptrCast(result.available_options),

lib/std/Build/Step/ConfigHeader.zig+2-7

...	...	@@ -5,7 +5,6 @@ const Io = std.Io;
5	5	const Step = std.Build.Step;
6	6	const Allocator = std.mem.Allocator;
7	7	const Configuration = std.Build.Configuration;
8		const allocPrint = std.fmt.allocPrint;
9	8
10	9	step: Step,
11	10	values: std.array_hash_map.String(Value) = .empty,
...	...	@@ -84,13 +83,9 @@ pub fn create(owner: std.Build, options: Options) ConfigHeader {
84	83	};
85	84
86	85	const name = if (options.style.getPath()) \|s\|
87		allocPrint(arena, "configure {t} header {f} to {s}", .{
88		options.style, s, include_path,
89		}) catch @panic("OOM")
	86	arena.print("configure {t} header {f} to {s}", .{ options.style, s, include_path }) catch @panic("OOM")
90	87	else
91		allocPrint(arena, "configure {t} header to {s}", .{
92		options.style, include_path,
93		}) catch @panic("OOM");
	88	arena.print("configure {t} header to {s}", .{ options.style, include_path }) catch @panic("OOM");
94	89
95	90	config_header.* = .{
96	91	.step = .init(.{

lib/std/Build/Step/TranslateC.zig+1-2

...	...	@@ -3,7 +3,6 @@ const TranslateC = @This();
3	3	const std = @import("std");
4	4	const fs = std.fs;
5	5	const mem = std.mem;
6		const allocPrint = std.fmt.allocPrint;
7	6	const Step = std.Build.Step;
8	7	const LazyPath = std.Build.LazyPath;
9	8	const Configuration = std.Build.Configuration;
...	...	@@ -158,7 +157,7 @@ pub fn defineCMacro(translate_c: *TranslateC, name: []const u8, value: ?[]const
158	157	const graph = translate_c.step.owner.graph;
159	158	const arena = graph.arena;
160	159	const wc = &graph.wip_configuration;
161		const macro = allocPrint(arena, "{s}={s}", .{ name, value orelse "1" }) catch @panic("OOM");
	160	const macro = arena.print("{s}={s}", .{ name, value orelse "1" }) catch @panic("OOM");
162	161	const macro_string = wc.addString(macro) catch @panic("OOM");
163	162	translate_c.c_macros.append(arena, macro_string) catch @panic("OOM");
164	163	}

lib/std/Progress.zig+19-19

...	...	@@ -165,25 +165,6 @@ pub const TerminalMode = union(enum) {
165	165	};
166	166	};
167	167
168		pub const Options = struct {
169		/// User-provided buffer with static lifetime.
170		///
171		/// Used to store the entire write buffer sent to the terminal. Progress output will be truncated if it
172		/// cannot fit into this buffer which will look bad but not cause any malfunctions.
173		///
174		/// Must be at least 200 bytes.
175		draw_buffer: []u8 = &default_draw_buffer,
176		/// How many nanoseconds between writing updates to the terminal.
177		refresh_rate_ns: Io.Duration = .fromMilliseconds(80),
178		/// How many nanoseconds to keep the output hidden
179		initial_delay_ns: Io.Duration = .fromMilliseconds(200),
180		/// If provided, causes the progress item to have a denominator.
181		/// 0 means unknown.
182		estimated_total_items: usize = 0,
183		root_name: []const u8 = "",
184		disable_printing: bool = false,
185		};
186
187	168	/// Represents one unit of progress. Each node can have children nodes, or
188	169	/// one can use integers with `update`.
189	170	pub const Node = struct {
...	...	@@ -578,6 +559,25 @@ pub const ParentFileError = error{
578	559	UnrecognizedFormat,
579	560	};
580	561
	562	pub const Options = struct {
	563	/// User-provided buffer with static lifetime.
	564	///
	565	/// Used to store the entire write buffer sent to the terminal. Progress output will be truncated if it
	566	/// cannot fit into this buffer which will look bad but not cause any malfunctions.
	567	///
	568	/// Must be at least 200 bytes.
	569	draw_buffer: []u8 = &default_draw_buffer,
	570	/// How many nanoseconds between writing updates to the terminal.
	571	refresh_rate_ns: Io.Duration = .fromMilliseconds(80),
	572	/// How many nanoseconds to keep the output hidden
	573	initial_delay_ns: Io.Duration = .fromMilliseconds(200),
	574	/// If provided, causes the progress item to have a denominator.
	575	/// 0 means unknown.
	576	estimated_total_items: usize = 0,
	577	root_name: []const u8 = "",
	578	disable_printing: bool = false,
	579	};
	580
581	581	/// Initializes a global Progress instance.
582	582	///
583	583	/// Asserts there is only one global Progress instance.

lib/std/zig.zig+678-60

...	...	@@ -2,12 +2,18 @@
2	2	//! source lives here. These APIs are provided as-is and have absolutely no API
3	3	//! guarantees whatsoever.
4	4
	5	const builtin = @import("builtin");
	6
5	7	const std = @import("std.zig");
6	8	const assert = std.debug.assert;
7	9	const mem = std.mem;
	10	const log = std.log;
8	11	const Allocator = std.mem.Allocator;
9	12	const Io = std.Io;
10	13	const Writer = std.Io.Writer;
	14	const Cache = std.Build.Cache;
	15	const fatal = std.process.fatal;
	16	const Dir = std.Io.Dir;
11	17
12	18	const tokenizer = @import("zig/tokenizer.zig");
13	19
...	...	@@ -47,6 +53,9 @@ pub const c_translation = struct {
47	53	pub const helpers = @import("zig/c_translation/helpers.zig");
48	54	};
49	55
	56	pub const default_local_zig_cache_basename = ".zig-cache";
	57	pub const build_zig_basename = "build.zig";
	58
50	59	pub const SrcHasher = std.crypto.hash.Blake3;
51	60	pub const SrcHash = [16]u8;
52	61
...	...	@@ -70,7 +79,7 @@ pub const Color = enum {
70	79	/// CLICOLOR_FORCE environment variables. Color is always disabled on WASI per
71	80	/// https://github.com/WebAssembly/WASI/issues/162
72	81	pub fn settingFromEnvironment(environ_map: *const std.process.Environ.Map) Color {
73		return if (@import("builtin").os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map))
	82	return if (builtin.os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map))
74	83	.off
75	84	else if (EnvVar.CLICOLOR_FORCE.isSet(environ_map))
76	85	.on
...	...	@@ -163,8 +172,8 @@ pub const BinNameOptions = struct {
163	172	os_tag: std.Target.Os.Tag,
164	173	ofmt: std.Target.ObjectFormat,
165	174	abi: std.Target.Abi,
166		output_mode: std.builtin.OutputMode,
167		link_mode: ?std.builtin.LinkMode = null,
	175	output_mode: std.lang.OutputMode,
	176	link_mode: ?std.lang.LinkMode = null,
168	177	version: ?std.SemanticVersion = null,
169	178	};
170	179
...	...	@@ -512,7 +521,7 @@ pub const FormatId = struct {
512	521	pub fn format(ctx: FormatId, writer: *Writer) Writer.Error!void {
513	522	const bytes = ctx.bytes;
514	523	if (isValidId(bytes) and
515		(ctx.flags.allow_primitive or !std.zig.isPrimitive(bytes)) and
	524	(ctx.flags.allow_primitive or !isPrimitive(bytes)) and
516	525	(ctx.flags.allow_underscore or !isUnderscore(bytes)))
517	526	{
518	527	return writer.writeAll(bytes);
...	...	@@ -592,7 +601,7 @@ pub fn isValidId(bytes: []const u8) bool {
592	601	else => return false,
593	602	}
594	603	}
595		return std.zig.Token.getKeyword(bytes) == null;
	604	return Token.getKeyword(bytes) == null;
596	605	}
597	606
598	607	test isValidId {
...	...	@@ -658,7 +667,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *Io.File.Reader) ![
658	667	}
659	668
660	669	pub fn printAstErrorsToStderr(gpa: Allocator, io: Io, tree: Ast, path: []const u8, color: Color) !void {
661		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
	670	var wip_errors: ErrorBundle.Wip = undefined;
662	671	try wip_errors.init(gpa);
663	672	defer wip_errors.deinit();
664	673
...	...	@@ -673,7 +682,7 @@ pub fn putAstErrorsIntoBundle(
673	682	gpa: Allocator,
674	683	tree: Ast,
675	684	path: []const u8,
676		wip_errors: *std.zig.ErrorBundle.Wip,
	685	wip_errors: *ErrorBundle.Wip,
677	686	) Allocator.Error!void {
678	687	switch (tree.mode) {
679	688	.zig => {
...	...	@@ -692,7 +701,7 @@ pub fn putAstErrorsIntoBundle(
692	701	}
693	702
694	703	pub fn resolveTargetQueryOrFatal(io: Io, target_query: std.Target.Query) std.Target {
695		return std.zig.system.resolveTargetQuery(io, target_query) catch \|err\|
	704	return system.resolveTargetQuery(io, target_query) catch \|err\|
696	705	std.process.fatal("unable to resolve target: {t}", .{err});
697	706	}
698	707
...	...	@@ -713,7 +722,7 @@ pub fn parseTargetQueryOrReportFatalError(
713	722	for (diags.arch.?.allCpuModels()) \|cpu\| {
714	723	help_text.print(" {s}\n", .{cpu.name}) catch break :help;
715	724	}
716		std.log.info("available CPUs for architecture '{s}':\n{s}", .{
	725	log.info("available CPUs for architecture '{s}':\n{s}", .{
717	726	@tagName(diags.arch.?), help_text.items,
718	727	});
719	728	}
...	...	@@ -726,7 +735,7 @@ pub fn parseTargetQueryOrReportFatalError(
726	735	for (diags.arch.?.allFeaturesList()) \|feature\| {
727	736	help_text.print(" {s}: {s}\n", .{ feature.name, feature.description }) catch break :help;
728	737	}
729		std.log.info("available CPU features for architecture '{s}':\n{s}", .{
	738	log.info("available CPU features for architecture '{s}':\n{s}", .{
730	739	@tagName(diags.arch.?), help_text.items,
731	740	});
732	741	}
...	...	@@ -739,7 +748,7 @@ pub fn parseTargetQueryOrReportFatalError(
739	748	inline for (@typeInfo(std.Target.ObjectFormat).@"enum".field_names) \|field_name\| {
740	749	help_text.print(" {s}\n", .{field_name}) catch break :help;
741	750	}
742		std.log.info("available object formats:\n{s}", .{help_text.items});
	751	log.info("available object formats:\n{s}", .{help_text.items});
743	752	}
744	753	std.process.fatal("unknown object format: '{s}'", .{opts.object_format.?});
745	754	},
...	...	@@ -750,7 +759,7 @@ pub fn parseTargetQueryOrReportFatalError(
750	759	inline for (@typeInfo(std.Target.Cpu.Arch).@"enum".field_names) \|field_name\| {
751	760	help_text.print(" {s}\n", .{field_name}) catch break :help;
752	761	}
753		std.log.info("available architectures:\n{s} native\n", .{help_text.items});
	762	log.info("available architectures:\n{s} native\n", .{help_text.items});
754	763	}
755	764	std.process.fatal("unknown architecture: '{s}'", .{diags.unknown_architecture_name.?});
756	765	},
...	...	@@ -772,7 +781,9 @@ pub const EnvVar = enum {
772	781	ZIG_BUILD_MULTILINE_ERRORS,
773	782	ZIG_VERBOSE_LINK,
774	783	ZIG_VERBOSE_CC,
	784	ZIG_VERBOSE_CMD,
775	785	ZIG_DEBUG_CMD,
	786	ZIG_DEBUG_MAKER,
776	787	ZIG_IS_DETECTING_LIBC_PATHS,
777	788	ZIG_IS_AVOIDING_CALLING_ITSELF,
778	789
...	...	@@ -1174,72 +1185,679 @@ pub const ClangCliParam = struct {
1174	1185	}
1175	1186	};
1176	1187
	1188	/// Deprecated
1177	1189	pub const AllocPrintCmdOptions = struct {
1178	1190	cwd: ?[]const u8 = null,
1179	1191	parent_env: ?*const std.process.Environ.Map = null,
1180	1192	child_env: ?*const std.process.Environ.Map = null,
1181	1193	};
1182	1194
	1195	/// Deprecated
1183	1196	pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPrintCmdOptions) Allocator.Error![]u8 {
1184		const shell = struct {
1185		fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void {
1186		for (string) \|c\| {
1187		if (switch (c) {
1188		else => true,
1189		'%', '+'...':', '@'...'Z', '_', 'a'...'z' => false,
1190		'=' => is_argv0,
1191		}) break;
1192		} else return writer.writeAll(string);
1193
1194		try writer.writeByte('"');
1195		for (string) \|c\| {
1196		if (switch (c) {
1197		std.ascii.control_code.nul => break,
1198		'!', '"', '$', '\\', '`' => true,
1199		else => !std.ascii.isPrint(c),
1200		}) try writer.writeByte('\\');
1201		switch (c) {
1202		std.ascii.control_code.nul => unreachable,
1203		std.ascii.control_code.bel => try writer.writeByte('a'),
1204		std.ascii.control_code.bs => try writer.writeByte('b'),
1205		std.ascii.control_code.ht => try writer.writeByte('t'),
1206		std.ascii.control_code.lf => try writer.writeByte('n'),
1207		std.ascii.control_code.vt => try writer.writeByte('v'),
1208		std.ascii.control_code.ff => try writer.writeByte('f'),
1209		std.ascii.control_code.cr => try writer.writeByte('r'),
1210		std.ascii.control_code.esc => try writer.writeByte('E'),
1211		' '...'~' => try writer.writeByte(c),
1212		else => try writer.print("{o:0>3}", .{c}),
	1197	var aw: Io.Writer.Allocating = .init(gpa);
	1198	defer aw.deinit();
	1199	SubprocessCommand.format(.{
	1200	.argv = argv,
	1201	.cwd = options.cwd,
	1202	.parent_env = options.parent_env,
	1203	.child_env = options.child_env,
	1204	}, &aw.writer) catch return error.OutOfMemory;
	1205	return aw.toOwnedSlice();
	1206	}
	1207
	1208	fn shellEscape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void {
	1209	for (string) \|c\| {
	1210	if (switch (c) {
	1211	else => true,
	1212	'%', '+'...':', '@'...'Z', '_', 'a'...'z' => false,
	1213	'=' => is_argv0,
	1214	}) break;
	1215	} else return writer.writeAll(string);
	1216
	1217	try writer.writeByte('"');
	1218	for (string) \|c\| {
	1219	if (switch (c) {
	1220	std.ascii.control_code.nul => break,
	1221	'!', '"', '$', '\\', '`' => true,
	1222	else => !std.ascii.isPrint(c),
	1223	}) try writer.writeByte('\\');
	1224	switch (c) {
	1225	std.ascii.control_code.nul => unreachable,
	1226	std.ascii.control_code.bel => try writer.writeByte('a'),
	1227	std.ascii.control_code.bs => try writer.writeByte('b'),
	1228	std.ascii.control_code.ht => try writer.writeByte('t'),
	1229	std.ascii.control_code.lf => try writer.writeByte('n'),
	1230	std.ascii.control_code.vt => try writer.writeByte('v'),
	1231	std.ascii.control_code.ff => try writer.writeByte('f'),
	1232	std.ascii.control_code.cr => try writer.writeByte('r'),
	1233	std.ascii.control_code.esc => try writer.writeByte('E'),
	1234	' '...'~' => try writer.writeByte(c),
	1235	else => try writer.print("{o:0>3}", .{c}),
	1236	}
	1237	}
	1238	try writer.writeByte('"');
	1239	}
	1240
	1241	pub const SubprocessCommand = struct {
	1242	argv: []const []const u8,
	1243	cwd: ?[]const u8 = null,
	1244	parent_env: ?*const std.process.Environ.Map = null,
	1245	child_env: ?*const std.process.Environ.Map = null,
	1246
	1247	pub fn format(sc: SubprocessCommand, w: *Io.Writer) Io.Writer.Error!void {
	1248	if (sc.cwd) \|path\| {
	1249	try w.print("cd {s} && ", .{path});
	1250	}
	1251	if (sc.child_env) \|child_env\| {
	1252	for (child_env.keys(), child_env.values()) \|key, value\| {
	1253	if (sc.parent_env) \|parent_env\| {
	1254	if (parent_env.get(key)) \|process_value\| {
	1255	if (mem.eql(u8, value, process_value)) continue;
	1256	}
1213	1257	}
	1258	try w.print("{s}=", .{key});
	1259	try shellEscape(w, value, false);
	1260	try w.writeByte(' ');
1214	1261	}
1215		try writer.writeByte('"');
1216	1262	}
	1263	try shellEscape(w, sc.argv[0], true);
	1264	for (sc.argv[1..]) \|arg\| {
	1265	try w.writeByte(' ');
	1266	try shellEscape(w, arg, false);
	1267	}
	1268	}
	1269	};
	1270
	1271	/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
	1272	/// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
	1273	/// On WASI, "" is returned instead of ".".
	1274	pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
	1275	if (builtin.os.tag == .wasi) {
	1276	if (std.debug.runtime_safety) {
	1277	const cwd = try std.process.currentPathAlloc(io, gpa);
	1278	defer gpa.free(cwd);
	1279	assert(mem.eql(u8, cwd, "."));
	1280	}
	1281	return "";
	1282	}
	1283	const cwd = try std.process.currentPathAlloc(io, gpa);
	1284	defer gpa.free(cwd);
	1285	const resolved = try Dir.path.resolve(gpa, &.{cwd});
	1286	assert(Dir.path.isAbsolute(resolved));
	1287	return resolved;
	1288	}
	1289
	1290	pub const Directories = struct {
	1291	/// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path,
	1292	/// but on WASI is the empty string "" instead, because WASI does not have absolute paths.
	1293	cwd: []const u8,
	1294	/// The Zig 'lib' directory.
	1295	/// `zig_lib.path` is resolved (`resolvePath`) or `null` for cwd.
	1296	/// Guaranteed to be a different path from `global_cache` and `local_cache`.
	1297	zig_lib: Cache.Directory,
	1298	/// The global Zig cache directory.
	1299	/// `global_cache.path` is resolved (`resolvePath`) or `null` for cwd.
	1300	global_cache: Cache.Directory,
	1301	/// The local Zig cache directory.
	1302	/// `local_cache.path` is resolved (`resolvePath`) or `null` for cwd.
	1303	/// This may be the same as `global_cache`.
	1304	local_cache: Cache.Directory,
	1305
	1306	pub fn deinit(dirs: *Directories, io: Io) void {
	1307	// The local and global caches could be the same.
	1308	const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle;
	1309
	1310	dirs.global_cache.handle.close(io);
	1311	if (close_local) dirs.local_cache.handle.close(io);
	1312	dirs.zig_lib.handle.close(io);
	1313	}
	1314
	1315	/// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for
	1316	/// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it
	1317	/// shares handles with `dirs`.
	1318	pub fn withoutLocalCache(dirs: Directories) Directories {
	1319	return .{
	1320	.cwd = dirs.cwd,
	1321	.zig_lib = dirs.zig_lib,
	1322	.global_cache = dirs.global_cache,
	1323	.local_cache = dirs.global_cache,
	1324	};
	1325	}
	1326
	1327	const LocalCacheStrategy = union(enum) {
	1328	override: []const u8,
	1329	search,
	1330	global,
1217	1331	};
1218	1332
1219		var aw: Io.Writer.Allocating = .init(gpa);
1220		defer aw.deinit();
1221		const writer = &aw.writer;
1222		if (options.cwd) \|path\| {
1223		writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory;
	1333	/// Uses `std.process.fatal` on error conditions.
	1334	pub fn init(
	1335	arena: Allocator,
	1336	io: Io,
	1337	override_zig_lib: ?[]const u8,
	1338	override_global_cache: ?[]const u8,
	1339	local_cache_strat: LocalCacheStrategy,
	1340	preopens: std.process.Preopens,
	1341	self_exe_path: switch (builtin.target.os.tag) {
	1342	.wasi => void,
	1343	else => []const u8,
	1344	},
	1345	environ_map: *const std.process.Environ.Map,
	1346	cwd: []const u8,
	1347	) Directories {
	1348	const wasi = builtin.target.os.tag == .wasi;
	1349
	1350	const zig_lib: Cache.Directory = d: {
	1351	if (override_zig_lib) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"zig lib");
	1352	if (wasi) break :d getPreopen(preopens, "/lib");
	1353	break :d findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch \|err\| {
	1354	fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err });
	1355	};
	1356	};
	1357
	1358	const global_cache: Cache.Directory = d: {
	1359	if (override_global_cache) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"global cache");
	1360	if (wasi) break :d getPreopen(preopens, "/cache");
	1361	const path = resolveGlobalCacheDir(arena, environ_map) catch \|err\| {
	1362	fatal("unable to resolve zig cache directory: {t}", .{err});
	1363	};
	1364	break :d openUnresolved(arena, io, cwd, path, .@"global cache");
	1365	};
	1366
	1367	const local_cache = getLocalCacheDirectory(arena, io, cwd, global_cache, local_cache_strat);
	1368
	1369	if (mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) {
	1370	fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache });
	1371	}
	1372	if (mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) {
	1373	fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache });
	1374	}
	1375
	1376	return .{
	1377	.cwd = cwd,
	1378	.zig_lib = zig_lib,
	1379	.global_cache = global_cache,
	1380	.local_cache = local_cache,
	1381	};
	1382	}
	1383
	1384	fn getLocalCacheDirectory(
	1385	arena: Allocator,
	1386	io: Io,
	1387	cwd: []const u8,
	1388	global_cache: Cache.Directory,
	1389	local_cache_strat: LocalCacheStrategy,
	1390	) Cache.Directory {
	1391	return switch (local_cache_strat) {
	1392	.override => \|path\| openUnresolved(arena, io, cwd, path, .@"local cache"),
	1393	.search => d: {
	1394	const maybe_path = resolveSuitableLocalCacheDir(arena, io, cwd) catch \|err\|
	1395	fatal("unable to resolve zig cache directory: {t}", .{err});
	1396	const path = maybe_path orelse break :d global_cache;
	1397	break :d openUnresolved(arena, io, cwd, path, .@"local cache");
	1398	},
	1399	.global => global_cache,
	1400	};
	1401	}
	1402
	1403	fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory {
	1404	return .{
	1405	.path = if (mem.eql(u8, name, ".")) null else name,
	1406	.handle = switch (preopens.get(name) orelse fatal("preopen not found: {q}", .{name})) {
	1407	.file => fatal("preopen {q} is not a directory", .{name}),
	1408	.dir => \|d\| d,
	1409	},
	1410	};
1224	1411	}
1225		if (options.child_env) \|child_env\| {
1226		for (child_env.keys(), child_env.values()) \|key, value\| {
1227		if (options.parent_env) \|parent_env\| {
1228		if (parent_env.get(key)) \|process_value\| {
1229		if (std.mem.eql(u8, value, process_value)) continue;
	1412	pub fn openUnresolved(
	1413	arena: Allocator,
	1414	io: Io,
	1415	cwd: []const u8,
	1416	unresolved_path: []const u8,
	1417	thing: enum { @"zig lib", @"global cache", @"local cache" },
	1418	) Cache.Directory {
	1419	const path = resolvePath(arena, cwd, &.{unresolved_path}) catch \|err\| {
	1420	fatal("unable to resolve {t} directory: {t}", .{ thing, err });
	1421	};
	1422	const nonempty_path = if (path.len == 0) "." else path;
	1423	const handle_or_err = switch (thing) {
	1424	.@"zig lib" => Dir.cwd().openDir(io, nonempty_path, .{}),
	1425	.@"global cache", .@"local cache" => Dir.cwd().createDirPathOpen(io, nonempty_path, .{}),
	1426	};
	1427	return .{
	1428	.path = if (path.len == 0) null else path,
	1429	.handle = handle_or_err catch \|err\| {
	1430	const extra_str: []const u8 = e: {
	1431	if (thing == .@"global cache") switch (err) {
	1432	error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++
	1433	"If this location is not writable then consider specifying an alternative with " ++
	1434	"the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.",
	1435	else => {},
	1436	};
	1437	break :e "";
	1438	};
	1439	fatal("unable to open {t} directory {q}: {t}{s}", .{ thing, nonempty_path, err, extra_str });
	1440	},
	1441	};
	1442	}
	1443	};
	1444
	1445	/// Both the directory handle and the path are newly allocated resources which the caller now owns.
	1446	pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
	1447	const cwd_path = try getResolvedCwd(io, gpa);
	1448	defer gpa.free(cwd_path);
	1449	const self_exe_path = try std.process.executablePathAlloc(io, gpa);
	1450	defer gpa.free(self_exe_path);
	1451
	1452	return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path);
	1453	}
	1454
	1455	/// Both the directory handle and the path are newly allocated resources which the caller now owns.
	1456	pub fn findZigLibDirFromSelfExe(
	1457	allocator: Allocator,
	1458	io: Io,
	1459	/// The return value of `getResolvedCwd`.
	1460	/// Passed as an argument to avoid pointlessly repeating the call.
	1461	cwd_path: []const u8,
	1462	self_exe_path: []const u8,
	1463	) error{ OutOfMemory, FileNotFound }!Cache.Directory {
	1464	const cwd = Dir.cwd();
	1465	var cur_path: []const u8 = self_exe_path;
	1466	while (Dir.path.dirname(cur_path)) \|dirname\| : (cur_path = dirname) {
	1467	var base_dir = cwd.openDir(io, dirname, .{}) catch continue;
	1468	defer base_dir.close(io);
	1469
	1470	const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue;
	1471	const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? });
	1472	defer allocator.free(p);
	1473
	1474	const resolved = try resolvePath(allocator, cwd_path, &.{p});
	1475	return .{
	1476	.handle = sub_directory.handle,
	1477	.path = if (resolved.len == 0) null else resolved,
	1478	};
	1479	}
	1480	return error.FileNotFound;
	1481	}
	1482
	1483	/// Returns the sub_path that worked, or `null` if none did.
	1484	/// The path of the returned Directory is relative to `base`.
	1485	/// The handle of the returned Directory is open.
	1486	fn testZigInstallPrefix(io: Io, base_dir: Dir) ?Cache.Directory {
	1487	const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig";
	1488
	1489	zig_dir: {
	1490	// Try lib/zig/std/std.zig
	1491	const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig";
	1492	var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir;
	1493	const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
	1494	test_zig_dir.close(io);
	1495	break :zig_dir;
	1496	};
	1497	file.close(io);
	1498	return .{ .handle = test_zig_dir, .path = lib_zig };
	1499	}
	1500
	1501	// Try lib/std/std.zig
	1502	var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null;
	1503	const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
	1504	test_zig_dir.close(io);
	1505	return null;
	1506	};
	1507	file.close(io);
	1508	return .{ .handle = test_zig_dir, .path = "lib" };
	1509	}
	1510
	1511	pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 {
	1512	if (EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) \|value\| return value;
	1513
	1514	const app_name = "zig";
	1515
	1516	switch (builtin.os.tag) {
	1517	.wasi => @compileError("on WASI the global cache dir must be resolved with preopens"),
	1518	.windows => {
	1519	const local_app_data_dir = EnvVar.LOCALAPPDATA.get(environ_map) orelse
	1520	return error.AppDataDirUnavailable;
	1521	return Dir.path.join(arena, &.{ local_app_data_dir, app_name });
	1522	},
	1523	else => {
	1524	if (EnvVar.XDG_CACHE_HOME.get(environ_map)) \|cache_root\| {
	1525	if (cache_root.len > 0) {
	1526	return Dir.path.join(arena, &.{ cache_root, app_name });
	1527	}
	1528	}
	1529	if (EnvVar.HOME.get(environ_map)) \|home\| {
	1530	if (home.len > 0) {
	1531	return Dir.path.join(arena, &.{ home, ".cache", app_name });
1230	1532	}
1231	1533	}
1232		writer.print("{s}=", .{key}) catch return error.OutOfMemory;
1233		shell.escape(writer, value, false) catch return error.OutOfMemory;
1234		writer.writeByte(' ') catch return error.OutOfMemory;
	1534	return error.AppDataDirUnavailable;
	1535	},
	1536	}
	1537	}
	1538
	1539	/// Searches upwards from `cwd` for a directory containing a `build.zig` file.
	1540	/// If such a directory is found, returns the path to it joined to the `.zig_cache` name.
	1541	/// Otherwise, returns `null`, indicating no suitable local cache location.
	1542	pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 {
	1543	var cur_dir = cwd;
	1544	while (true) {
	1545	const joined = try Dir.path.join(arena, &.{ cur_dir, build_zig_basename });
	1546	if (Dir.cwd().access(io, joined, .{})) \|_\| {
	1547	return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename });
	1548	} else \|err\| switch (err) {
	1549	error.FileNotFound => {
	1550	cur_dir = Dir.path.dirname(cur_dir) orelse return null;
	1551	continue;
	1552	},
	1553	else => return null,
1235	1554	}
1236	1555	}
1237		shell.escape(writer, argv[0], true) catch return error.OutOfMemory;
1238		for (argv[1..]) \|arg\| {
1239		writer.writeByte(' ') catch return error.OutOfMemory;
1240		shell.escape(writer, arg, false) catch return error.OutOfMemory;
	1556	}
	1557
	1558	/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would
	1559	/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd
	1560	/// returns the empty string ("") instead of ".".
	1561	pub fn resolvePath(
	1562	gpa: Allocator,
	1563	/// The return value of `getResolvedCwd`.
	1564	/// Passed as an argument to avoid pointlessly repeating the call.
	1565	cwd_resolved: []const u8,
	1566	paths: []const []const u8,
	1567	) Allocator.Error![]u8 {
	1568	if (builtin.target.os.tag == .wasi) {
	1569	assert(mem.eql(u8, cwd_resolved, ""));
	1570	const res = try Dir.path.resolve(gpa, paths);
	1571	if (mem.eql(u8, res, ".")) {
	1572	gpa.free(res);
	1573	return "";
	1574	}
	1575	return res;
1241	1576	}
1242		return aw.toOwnedSlice();
	1577
	1578	// Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`.
	1579	for (paths) \|p\| {
	1580	if (Dir.path.isAbsolute(p)) break; // absolute path
	1581	if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir
	1582	} else {
	1583	// no absolute path, no "..".
	1584	const res = try Dir.path.resolve(gpa, paths);
	1585	if (mem.eql(u8, res, ".")) {
	1586	gpa.free(res);
	1587	return "";
	1588	}
	1589	assert(!Dir.path.isAbsolute(res));
	1590	assert(!isUpDir(res));
	1591	return res;
	1592	}
	1593
	1594	// The fast path failed; resolve the whole thing.
	1595	// Optimization: `paths` often has just one element.
	1596	const path_resolved = switch (paths.len) {
	1597	0 => unreachable,
	1598	1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }),
	1599	else => r: {
	1600	const all_paths = try gpa.alloc([]const u8, paths.len + 1);
	1601	defer gpa.free(all_paths);
	1602	all_paths[0] = cwd_resolved;
	1603	@memcpy(all_paths[1..], paths);
	1604	break :r try Dir.path.resolve(gpa, all_paths);
	1605	},
	1606	};
	1607	errdefer gpa.free(path_resolved);
	1608
	1609	assert(Dir.path.isAbsolute(path_resolved));
	1610	assert(Dir.path.isAbsolute(cwd_resolved));
	1611
	1612	if (!mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd
	1613	if (path_resolved.len == cwd_resolved.len) {
	1614	// equal to cwd
	1615	gpa.free(path_resolved);
	1616	return "";
	1617	}
	1618	if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs)
	1619
	1620	// in cwd; extract sub path
	1621	const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]);
	1622	gpa.free(path_resolved);
	1623	return sub_path;
	1624	}
	1625
	1626	pub fn isUpDir(p: []const u8) bool {
	1627	return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep);
	1628	}
	1629
	1630	pub const BuildExeSubprocessOptions = struct {
	1631	argv: []const []const u8,
	1632	cache_root: Cache.Directory,
	1633	root_name: []const u8,
	1634
	1635	environ_map: ?*std.process.Environ.Map = null,
	1636	cache_manifest: ?*Cache.Manifest = null,
	1637	arch_os_abi: ?[]const u8 = null,
	1638	cpu_features: ?[]const u8 = null,
	1639	progress_node: std.Progress.Node = .none,
	1640	skip_log_cmdline_on_compile_errors: bool = false,
	1641	};
	1642
	1643	pub const BuildExeSubprocessError = error{
	1644	/// Error message has been logged.
	1645	AlreadyReported,
	1646	/// Error message has been logged, and source files added to the `Cache.Manifest`.
	1647	FailedButCacheIntact,
	1648	} \|\| Io.Cancelable \|\| Allocator.Error;
	1649
	1650	pub const BuildExeSubprocessResult = struct {
	1651	received_fs_inputs: bool,
	1652	cache_hit: bool,
	1653	path: Cache.Path,
	1654	};
	1655
	1656	/// Assumes `argv` has `--listen=-` in it and the child process is `zig build-exe`.
	1657	///
	1658	/// Result path is allocated via gpa.
	1659	pub fn buildExeSubprocess(
	1660	gpa: Allocator,
	1661	io: Io,
	1662	options: BuildExeSubprocessOptions,
	1663	) BuildExeSubprocessError!BuildExeSubprocessResult {
	1664	const cmd: SubprocessCommand = .{ .argv = options.argv };
	1665
	1666	var child = std.process.spawn(io, .{
	1667	.argv = options.argv,
	1668	.environ_map = options.environ_map,
	1669	.stdin = .pipe,
	1670	.stdout = .pipe,
	1671	.stderr = .pipe,
	1672	.progress_node = options.progress_node,
	1673	}) catch \|err\| {
	1674	log.err("spawning command {t}: {f}", .{ err, cmd });
	1675	return error.AlreadyReported;
	1676	};
	1677	defer child.kill(io);
	1678
	1679	var stderr_task = io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }) catch
	1680	@panic("TODO use multireader instead");
	1681	defer if (stderr_task.cancel(io)) \|slice\| gpa.free(slice) else \|_\| {};
	1682
	1683	var stdout_buffer: [512]u8 = undefined;
	1684	var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer);
	1685	const stdout = &stdout_reader.interface;
	1686
	1687	{
	1688	var w = child.stdin.?.writer(io, &.{});
	1689	w.interface.writeStruct(Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch \|err\| switch (err) {
	1690	error.WriteFailed => {
	1691	log.err("{t} writing to command: {f}", .{ w.err.?, cmd });
	1692	return error.AlreadyReported;
	1693	},
	1694	};
	1695	w.interface.writeStruct(Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch \|err\| switch (err) {
	1696	error.WriteFailed => {
	1697	log.err("{t} writing to command: {f}", .{ w.err.?, cmd });
	1698	return error.AlreadyReported;
	1699	},
	1700	};
	1701	}
	1702
	1703	const Header = Server.Message.Header;
	1704
	1705	var result: ?Cache.Path = null;
	1706	defer if (result) \|r\| gpa.free(r.sub_path);
	1707
	1708	var result_error_bundle: ErrorBundle = .empty;
	1709	defer result_error_bundle.deinit(gpa);
	1710
	1711	var body_buffer: std.ArrayList(u8) = .empty;
	1712	defer body_buffer.deinit(gpa);
	1713
	1714	var received_fs_inputs = false;
	1715	var cache_hit = false;
	1716
	1717	while (true) {
	1718	const header = stdout.takeStruct(Header, .little) catch \|err\| switch (err) {
	1719	error.ReadFailed => {
	1720	log.err("{t} reading from command: {f}", .{ stdout_reader.err.?, cmd });
	1721	return error.AlreadyReported;
	1722	},
	1723	error.EndOfStream => break,
	1724	};
	1725	body_buffer.clearRetainingCapacity();
	1726	stdout.appendExact(gpa, &body_buffer, header.bytes_len) catch \|err\| switch (err) {
	1727	error.ReadFailed => {
	1728	log.err("{t} reading from command: {f}", .{ stdout_reader.err.?, cmd });
	1729	return error.AlreadyReported;
	1730	},
	1731	error.OutOfMemory => \|e\| return e,
	1732	error.EndOfStream => {
	1733	log.err("unexpected end of stream from command: {f}", .{cmd});
	1734	return error.AlreadyReported;
	1735	},
	1736	};
	1737	const body = body_buffer.items;
	1738
	1739	switch (header.tag) {
	1740	.zig_version => {
	1741	if (!mem.eql(u8, builtin.zig_version_string, body)) {
	1742	log.err("zig protocol version mismatch from command: {f}", .{cmd});
	1743	return error.AlreadyReported;
	1744	}
	1745	},
	1746	.error_bundle => {
	1747	result_error_bundle.deinit(gpa);
	1748	result_error_bundle = Server.allocErrorBundle(gpa, body) catch \|err\| switch (err) {
	1749	error.EndOfStream => break,
	1750	else => \|e\| return e,
	1751	};
	1752	},
	1753	.emit_digest => {
	1754	const EmitDigest = Server.Message.EmitDigest;
	1755	const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body);
	1756	cache_hit = ebp_hdr.flags.cache_hit;
	1757	const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
	1758	if (result) \|r\| gpa.free(r.sub_path);
	1759	result = .{
	1760	.root_dir = options.cache_root,
	1761	.sub_path = try Dir.path.join(gpa, &.{ "o", &Cache.binToHex(digest.*) }),
	1762	};
	1763	},
	1764	.file_system_inputs => if (options.cache_manifest) \|man\| {
	1765	received_fs_inputs = true;
	1766	var it = mem.splitScalar(u8, body, 0);
	1767	while (it.next()) \|prefixed_path\| {
	1768	const prefix: Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1);
	1769	const sub_path = try gpa.dupe(u8, prefixed_path[1..]);
	1770	var keep = false;
	1771	defer if (!keep) gpa.free(sub_path);
	1772	keep = man.addPrefixedPathPost(.{
	1773	.prefix = @intFromEnum(prefix),
	1774	.sub_path = sub_path,
	1775	}) catch \|err\| switch (err) {
	1776	error.Canceled, error.OutOfMemory => \|e\| return e,
	1777	else => \|e\| {
	1778	log.err("adding {t} {s} to cache failed: {t}", .{ prefix, sub_path, e });
	1779	return error.AlreadyReported;
	1780	},
	1781	};
	1782	}
	1783	},
	1784	else => {}, // ignore other messages
	1785	}
	1786	}
	1787
	1788	const stderr_contents = stderr_task.await(io) catch \|err\| switch (err) {
	1789	error.Canceled, error.OutOfMemory => \|e\| return e,
	1790	else => \|e\| c: {
	1791	log.warn("{t} reading stderr from command: {f}", .{ e, cmd });
	1792	break :c "";
	1793	},
	1794	};
	1795	if (stderr_contents.len > 0)
	1796	log.warn("unexpected stderr from {s} command:\n{s}", .{ options.argv[0], stderr_contents });
	1797
	1798	// Send EOF to stdin.
	1799	child.stdin.?.close(io);
	1800	child.stdin = null;
	1801
	1802	const term = child.wait(io) catch \|err\| switch (err) {
	1803	error.Canceled => \|e\| return e,
	1804	else => \|e\| {
	1805	log.err("{t} waiting for command: {f}", .{ e, cmd });
	1806	return error.AlreadyReported;
	1807	},
	1808	};
	1809
	1810	if (!term.success()) {
	1811	log.err("command {f}: {f}", .{ term, cmd });
	1812	if (received_fs_inputs) return error.FailedButCacheIntact;
	1813	return error.AlreadyReported;
	1814	}
	1815
	1816	if (result_error_bundle.errorMessageCount() > 0) {
	1817	result_error_bundle.renderToStderr(io, .{}, .auto) catch \|err\| switch (err) {
	1818	error.Canceled => \|e\| return e,
	1819	else => \|e\| {
	1820	log.err("failed rendering error bundle: {t}", .{e});
	1821	return error.AlreadyReported;
	1822	},
	1823	};
	1824	if (!options.skip_log_cmdline_on_compile_errors) log.err("command reported {d} compilation errors: {f}", .{
	1825	result_error_bundle.errorMessageCount(), cmd,
	1826	});
	1827	if (received_fs_inputs) return error.FailedButCacheIntact;
	1828	return error.AlreadyReported;
	1829	}
	1830
	1831	const base_path = result orelse {
	1832	log.err("command failed to report result: {f}", .{cmd});
	1833	return error.AlreadyReported;
	1834	};
	1835	const parsed_target = system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{
	1836	.arch_os_abi = options.arch_os_abi orelse "native",
	1837	.cpu_features = options.cpu_features,
	1838	}) catch unreachable) catch unreachable;
	1839	const bin_name = try binNameAlloc(gpa, .{
	1840	.root_name = options.root_name,
	1841	.cpu_arch = parsed_target.cpu.arch,
	1842	.os_tag = parsed_target.os.tag,
	1843	.ofmt = parsed_target.ofmt,
	1844	.abi = parsed_target.abi,
	1845	.output_mode = .Exe,
	1846	});
	1847	defer gpa.free(bin_name);
	1848	return .{
	1849	.received_fs_inputs = received_fs_inputs,
	1850	.cache_hit = cache_hit,
	1851	.path = try base_path.join(gpa, bin_name),
	1852	};
	1853	}
	1854
	1855	fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 {
	1856	var file_reader: Io.File.Reader = .initStreaming(file, io, &.{});
	1857	return file_reader.interface.allocRemaining(gpa, limit) catch \|err\| switch (err) {
	1858	error.ReadFailed => return file_reader.err.?,
	1859	else => \|e\| return e,
	1860	};
1243	1861	}
1244	1862
1245	1863	test {

lib/std/zig/LibCDirs.zig+16-31

...	...	@@ -5,6 +5,7 @@ const std = @import("../std.zig");
5	5	const Io = std.Io;
6	6	const LibCInstallation = std.zig.LibCInstallation;
7	7	const Allocator = std.mem.Allocator;
	8	const Path = std.Build.Cache.Path;
8	9
9	10	libc_include_dir_list: []const []const u8,
10	11	libc_installation: ?*const LibCInstallation,
...	...	@@ -23,7 +24,7 @@ pub const DarwinSdkLayout = enum {
23	24	pub fn detect(
24	25	arena: Allocator,
25	26	io: Io,
26		zig_lib_dir: []const u8,
	27	zig_lib_dir: Path,
27	28	target: *const std.Target,
28	29	is_native_abi: bool,
29	30	link_libc: bool,
...	...	@@ -166,20 +167,12 @@ fn detectFromInstallation(arena: Allocator, target: const std.Target, lci: con
166	167	};
167	168	}
168	169
169		pub fn detectFromBuilding(
170		arena: Allocator,
171		zig_lib_dir: []const u8,
172		target: *const std.Target,
173		) !LibCDirs {
	170	pub fn detectFromBuilding(arena: Allocator, zig_lib_dir: Path, target: *const std.Target) !LibCDirs {
174	171	const s = std.fs.path.sep_str;
175	172
176	173	if (target.os.tag.isDarwin()) {
177	174	const list = try arena.alloc([]const u8, 1);
178		list[0] = try std.fmt.allocPrint(
179		arena,
180		"{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-darwin-any",
181		.{zig_lib_dir},
182		);
	175	list[0] = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-darwin-any", .{zig_lib_dir});
183	176	return .{
184	177	.libc_include_dir_list = list,
185	178	.libc_installation = null,
...	...	@@ -212,27 +205,19 @@ pub fn detectFromBuilding(
212	205	std.zig.target.netbsdAbiNameHeaders(target.abi)
213	206	else
214	207	@tagName(target.abi);
215		const arch_include_dir = try std.fmt.allocPrint(
216		arena,
217		"{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}",
218		.{ zig_lib_dir, arch_name, os_name, abi_name },
219		);
220		const generic_include_dir = try std.fmt.allocPrint(
221		arena,
222		"{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}",
223		.{ zig_lib_dir, generic_name },
224		);
	208	const arch_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}", .{
	209	zig_lib_dir, arch_name, os_name, abi_name,
	210	});
	211	const generic_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}", .{
	212	zig_lib_dir, generic_name,
	213	});
225	214	const generic_arch_name = std.zig.target.osArchName(target);
226		const arch_os_include_dir = try std.fmt.allocPrint(
227		arena,
228		"{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any",
229		.{ zig_lib_dir, generic_arch_name, os_name },
230		);
231		const generic_os_include_dir = try std.fmt.allocPrint(
232		arena,
233		"{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any",
234		.{ zig_lib_dir, os_name },
235		);
	215	const arch_os_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any", .{
	216	zig_lib_dir, generic_arch_name, os_name,
	217	});
	218	const generic_os_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", .{
	219	zig_lib_dir, os_name,
	220	});
236	221
237	222	const list = try arena.alloc([]const u8, 4);
238	223	list[0] = arch_include_dir;

lib/std/zig/Server.zig+1-1

...	...	@@ -264,7 +264,7 @@ pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void {
264	264	try s.out.flush();
265	265	}
266	266
267		pub fn allocErrorBundle(gpa: std.mem.Allocator, body: []const u8) error{ OutOfMemory, EndOfStream }!std.zig.ErrorBundle {
	267	pub fn allocErrorBundle(gpa: Allocator, body: []const u8) error{ OutOfMemory, EndOfStream }!std.zig.ErrorBundle {
268	268	var r: Reader = .fixed(body);
269	269	const hdr = r.takeStruct(OutMessage.ErrorBundle, .little) catch \|err\| switch (err) {
270	270	error.EndOfStream => \|e\| return e,

src/Builtin.zig+1-1

...	...	@@ -370,7 +370,7 @@ const std = @import("std");
370	370	const Allocator = std.mem.Allocator;
371	371	const Cache = std.Build.Cache;
372	372	const build_options = @import("build_options");
373		const Module = @import("Package/Module.zig");
	373	const Module = @import("Module.zig");
374	374	const assert = std.debug.assert;
375	375	const AstGen = std.zig.AstGen;
376	376	const File = @import("Zcu.zig").File;

src/Compilation.zig+39-185

...	...	@@ -16,8 +16,6 @@ const fatal = std.process.fatal;
16	16	const Value = @import("Value.zig");
17	17	const Type = @import("Type.zig");
18	18	const target_util = @import("target.zig");
19		const Package = @import("Package.zig");
20		const introspect = @import("introspect.zig");
21	19	const link = @import("link.zig");
22	20	const tracy = @import("tracy.zig");
23	21	const trace = tracy.trace;
...	...	@@ -44,6 +42,7 @@ const Air = @import("Air.zig");
44	42	const Builtin = @import("Builtin.zig");
45	43	const LlvmObject = @import("codegen/llvm.zig").Object;
46	44	const dev = @import("dev.zig");
	45	const Module = @import("Module.zig");
47	46
48	47	pub const Config = @import("Compilation/Config.zig");
49	48
...	...	@@ -64,7 +63,7 @@ cache_use: CacheUse,
64	63	/// All compilations have a root module because this is where some important
65	64	/// settings are stored, such as target and optimization mode. This module
66	65	/// might not have any .zig code associated with it, however.
67		root_mod: *Package.Module,
	66	root_mod: *Module,
68	67
69	68	/// User-specified settings that have all the defaults resolved into concrete values.
70	69	config: Config,
...	...	@@ -190,7 +189,7 @@ parent_whole_cache: ?ParentWholeCache,
190	189	/// Path to own executable for invoking `zig clang`.
191	190	self_exe_path: ?[]const u8,
192	191	/// Owned by the caller of `Compilation.create`.
193		dirs: Directories,
	192	dirs: std.zig.Directories,
194	193	libc_include_dir_list: []const []const u8,
195	194	libc_framework_dir_list: []const []const u8,
196	195	rc_includes: std.zig.RcIncludes,
...	...	@@ -431,7 +430,7 @@ pub const Path = struct {
431	430	}
432	431
433	432	/// Given a `Path`, returns the directory handle and sub path to be used to open the path.
434		pub fn openInfo(p: Path, dirs: Directories) struct { Io.Dir, []const u8 } {
	433	pub fn openInfo(p: Path, dirs: std.zig.Directories) struct { Io.Dir, []const u8 } {
435	434	const dir = switch (p.root) {
436	435	.none => {
437	436	const cwd_sub_path = absToCwdRelative(p.sub_path, dirs.cwd);
...	...	@@ -491,8 +490,8 @@ pub const Path = struct {
491	490
492	491	/// From an unresolved path (which can be made of multiple not-yet-joined strings), construct a
493	492	/// canonical `Path`.
494		pub fn fromUnresolved(gpa: Allocator, dirs: Compilation.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path {
495		const resolved = try introspect.resolvePath(gpa, dirs.cwd, unresolved_parts);
	493	pub fn fromUnresolved(gpa: Allocator, dirs: std.zig.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path {
	494	const resolved = try std.zig.resolvePath(gpa, dirs.cwd, unresolved_parts);
496	495	errdefer gpa.free(resolved);
497	496
498	497	// If, for instance, `dirs.local_cache.path` is within the lib dir, it must take priority,
...	...	@@ -566,7 +565,7 @@ pub const Path = struct {
566	565	/// `.global_cache` could still end up returning a `Path` with `Path.root == .zig_lib`.
567	566	pub fn fromRoot(
568	567	gpa: Allocator,
569		dirs: Compilation.Directories,
	568	dirs: std.zig.Directories,
570	569	root: Path.Root,
571	570	sub_path: []const u8,
572	571	) Allocator.Error!Path {
...	...	@@ -589,7 +588,7 @@ pub const Path = struct {
589	588	pub fn join(
590	589	p: Path,
591	590	gpa: Allocator,
592		dirs: Compilation.Directories,
	591	dirs: std.zig.Directories,
593	592	sub_path: []const u8,
594	593	) Allocator.Error!Path {
595	594	// Currently, this just wraps `fromUnresolved` for simplicity. A more efficient impl is
...	...	@@ -610,7 +609,7 @@ pub const Path = struct {
610	609	pub fn upJoin(
611	610	p: Path,
612	611	gpa: Allocator,
613		dirs: Compilation.Directories,
	612	dirs: std.zig.Directories,
614	613	sub_path: []const u8,
615	614	) Allocator.Error!Path {
616	615	return .fromUnresolved(gpa, dirs, &.{
...	...	@@ -626,7 +625,7 @@ pub const Path = struct {
626	625	});
627	626	}
628	627
629		pub fn toCachePath(p: Path, dirs: Directories) Cache.Path {
	628	pub fn toCachePath(p: Path, dirs: std.zig.Directories) Cache.Path {
630	629	const root_dir: Cache.Directory = switch (p.root) {
631	630	.zig_lib => dirs.zig_lib,
632	631	.global_cache => dirs.global_cache,
...	...	@@ -649,7 +648,7 @@ pub const Path = struct {
649	648	/// This should not be used for most of the compiler pipeline, but is useful when emitting
650	649	/// paths from the compilation (e.g. in debug info), because they will not depend on the cwd.
651	650	/// The returned path is owned by the caller and allocated into `gpa`.
652		pub fn toAbsolute(p: Path, dirs: Directories, gpa: Allocator) Allocator.Error![]u8 {
	651	pub fn toAbsolute(p: Path, dirs: std.zig.Directories, gpa: Allocator) Allocator.Error![]u8 {
653	652	const root_path: []const u8 = switch (p.root) {
654	653	.zig_lib => dirs.zig_lib.path orelse "",
655	654	.global_cache => dirs.global_cache.path orelse "",
...	...	@@ -680,7 +679,7 @@ pub const Path = struct {
680	679	/// Returns whether this `Path` is illegal to have as a user-imported `Zcu.File` (including
681	680	/// as the root of a module). Such paths exist in directories which the Zig compiler treats
682	681	/// specially, like 'global_cache/b/', which stores 'builtin.zig' files.
683		pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: Directories) Allocator.Error!bool {
	682	pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: std.zig.Directories) Allocator.Error!bool {
684	683	const zig_builtin_dir: Path = try .fromRoot(gpa, dirs, .global_cache, "b");
685	684	defer zig_builtin_dir.deinit(gpa);
686	685	return switch (p.isNested(zig_builtin_dir)) {
...	...	@@ -690,149 +689,6 @@ pub const Path = struct {
690	689	}
691	690	};
692	691
693		pub const Directories = struct {
694		/// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path,
695		/// but on WASI is the empty string "" instead, because WASI does not have absolute paths.
696		cwd: []const u8,
697		/// The Zig 'lib' directory.
698		/// `zig_lib.path` is resolved (`introspect.resolvePath`) or `null` for cwd.
699		/// Guaranteed to be a different path from `global_cache` and `local_cache`.
700		zig_lib: Cache.Directory,
701		/// The global Zig cache directory.
702		/// `global_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd.
703		global_cache: Cache.Directory,
704		/// The local Zig cache directory.
705		/// `local_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd.
706		/// This may be the same as `global_cache`.
707		local_cache: Cache.Directory,
708
709		pub fn deinit(dirs: *Directories, io: Io) void {
710		// The local and global caches could be the same.
711		const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle;
712
713		dirs.global_cache.handle.close(io);
714		if (close_local) dirs.local_cache.handle.close(io);
715		dirs.zig_lib.handle.close(io);
716		}
717
718		/// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for
719		/// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it
720		/// shares handles with `dirs`.
721		pub fn withoutLocalCache(dirs: Directories) Directories {
722		return .{
723		.cwd = dirs.cwd,
724		.zig_lib = dirs.zig_lib,
725		.global_cache = dirs.global_cache,
726		.local_cache = dirs.global_cache,
727		};
728		}
729
730		/// Uses `std.process.fatal` on error conditions.
731		pub fn init(
732		arena: Allocator,
733		io: Io,
734		override_zig_lib: ?[]const u8,
735		override_global_cache: ?[]const u8,
736		local_cache_strat: union(enum) {
737		override: []const u8,
738		search,
739		global,
740		},
741		preopens: std.process.Preopens,
742		self_exe_path: switch (builtin.target.os.tag) {
743		.wasi => void,
744		else => []const u8,
745		},
746		environ_map: *const std.process.Environ.Map,
747		cwd: []const u8,
748		) Directories {
749		const wasi = builtin.target.os.tag == .wasi;
750
751		const zig_lib: Cache.Directory = d: {
752		if (override_zig_lib) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"zig lib");
753		if (wasi) break :d getPreopen(preopens, "/lib");
754		break :d introspect.findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch \|err\| {
755		fatal("unable to find zig installation directory '{s}': {t}", .{ self_exe_path, err });
756		};
757		};
758
759		const global_cache: Cache.Directory = d: {
760		if (override_global_cache) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"global cache");
761		if (wasi) break :d getPreopen(preopens, "/cache");
762		const path = introspect.resolveGlobalCacheDir(arena, environ_map) catch \|err\| {
763		fatal("unable to resolve zig cache directory: {t}", .{err});
764		};
765		break :d openUnresolved(arena, io, cwd, path, .@"global cache");
766		};
767
768		const local_cache: Cache.Directory = switch (local_cache_strat) {
769		.override => \|path\| openUnresolved(arena, io, cwd, path, .@"local cache"),
770		.search => d: {
771		const maybe_path = introspect.resolveSuitableLocalCacheDir(arena, io, cwd) catch \|err\| {
772		fatal("unable to resolve zig cache directory: {t}", .{err});
773		};
774		const path = maybe_path orelse break :d global_cache;
775		break :d openUnresolved(arena, io, cwd, path, .@"local cache");
776		},
777		.global => global_cache,
778		};
779
780		if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) {
781		fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache });
782		}
783		if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) {
784		fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache });
785		}
786
787		return .{
788		.cwd = cwd,
789		.zig_lib = zig_lib,
790		.global_cache = global_cache,
791		.local_cache = local_cache,
792		};
793		}
794		fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory {
795		return .{
796		.path = if (std.mem.eql(u8, name, ".")) null else name,
797		.handle = switch (preopens.get(name) orelse fatal("preopen not found: '{s}'", .{name})) {
798		.file => fatal("preopen {s} is not a directory", .{name}),
799		.dir => \|d\| d,
800		},
801		};
802		}
803		fn openUnresolved(
804		arena: Allocator,
805		io: Io,
806		cwd: []const u8,
807		unresolved_path: []const u8,
808		thing: enum { @"zig lib", @"global cache", @"local cache" },
809		) Cache.Directory {
810		const path = introspect.resolvePath(arena, cwd, &.{unresolved_path}) catch \|err\| {
811		fatal("unable to resolve {s} directory: {s}", .{ @tagName(thing), @errorName(err) });
812		};
813		const nonempty_path = if (path.len == 0) "." else path;
814		const handle_or_err = switch (thing) {
815		.@"zig lib" => Io.Dir.cwd().openDir(io, nonempty_path, .{}),
816		.@"global cache", .@"local cache" => Io.Dir.cwd().createDirPathOpen(io, nonempty_path, .{}),
817		};
818		return .{
819		.path = if (path.len == 0) null else path,
820		.handle = handle_or_err catch \|err\| {
821		const extra_str: []const u8 = e: {
822		if (thing == .@"global cache") switch (err) {
823		error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++
824		"If this location is not writable then consider specifying an alternative with " ++
825		"the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.",
826		else => {},
827		};
828		break :e "";
829		};
830		fatal("unable to open {s} directory '{s}': {s}{s}", .{ @tagName(thing), nonempty_path, @errorName(err), extra_str });
831		},
832		};
833		}
834		};
835
836	692	/// This small wrapper function just checks whether debug extensions are enabled before checking
837	693	/// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller,
838	694	/// preventing debugging features from making it into release builds of the compiler.
...	...	@@ -910,7 +766,7 @@ pub const CrtFile = struct {
910	766	/// For passing to a C compiler.
911	767	pub const CSourceFile = struct {
912	768	/// Many C compiler flags are determined by settings contained in the owning Module.
913		owner: *Package.Module,
	769	owner: *Module,
914	770	src_path: []const u8,
915	771	extra_flags: []const []const u8 = &.{},
916	772	/// Same as extra_flags except they are not added to the Cache hash.
...	...	@@ -922,7 +778,7 @@ pub const CSourceFile = struct {
922	778
923	779	/// For passing to resinator.
924	780	pub const RcSourceFile = struct {
925		owner: *Package.Module,
	781	owner: *Module,
926	782	src_path: []const u8,
927	783	extra_flags: []const []const u8 = &.{},
928	784	};
...	...	@@ -1360,7 +1216,7 @@ pub const MiscError = struct {
1360	1216	};
1361	1217
1362	1218	pub const cache_helpers = struct {
1363		pub fn addModule(hh: Cache.HashHelper, mod: const Package.Module) void {
	1219	pub fn addModule(hh: Cache.HashHelper, mod: const Module) void {
1364	1220	addResolvedTarget(hh, mod.resolved_target);
1365	1221	hh.add(mod.optimize_mode);
1366	1222	hh.add(mod.code_model);
...	...	@@ -1383,7 +1239,7 @@ pub const cache_helpers = struct {
1383	1239
1384	1240	pub fn addResolvedTarget(
1385	1241	hh: *Cache.HashHelper,
1386		resolved_target: Package.Module.ResolvedTarget,
	1242	resolved_target: Module.ResolvedTarget,
1387	1243	) void {
1388	1244	const target = &resolved_target.result;
1389	1245	hh.add(target.cpu.arch);
...	...	@@ -1549,23 +1405,23 @@ const CacheUse = union(CacheMode) {
1549	1405	};
1550	1406
1551	1407	pub const CreateOptions = struct {
1552		dirs: Directories,
	1408	dirs: std.zig.Directories,
1553	1409	thread_limit: usize,
1554	1410	self_exe_path: ?[]const u8 = null,
1555	1411
1556	1412	/// Options that have been resolved by calling `resolveDefaults`.
1557	1413	config: Compilation.Config,
1558	1414
1559		root_mod: *Package.Module,
	1415	root_mod: *Module,
1560	1416	/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig
1561	1417	/// test`, in which `root_mod` is the test runner, and `main_mod` is the
1562	1418	/// user's source file which has the tests.
1563		main_mod: ?*Package.Module = null,
	1419	main_mod: ?*Module = null,
1564	1420	/// This is provided so that the API user has a chance to tweak the
1565	1421	/// per-module settings of the standard library.
1566	1422	/// When this is null, a default configuration of the std lib is created
1567	1423	/// based on the settings of root_mod.
1568		std_mod: ?*Package.Module = null,
	1424	std_mod: ?*Module = null,
1569	1425	root_name: []const u8,
1570	1426	sysroot: ?[]const u8 = null,
1571	1427	cache_mode: CacheMode,
...	...	@@ -1873,7 +1729,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
1873	1729	const libc_dirs = std.zig.LibCDirs.detect(
1874	1730	arena,
1875	1731	io,
1876		options.dirs.zig_lib.path.?,
	1732	.{ .root_dir = options.dirs.zig_lib },
1877	1733	target,
1878	1734	options.root_mod.resolved_target.is_native_abi,
1879	1735	link_libc,
...	...	@@ -1907,7 +1763,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
1907	1763	if (compiler_rt_strat == .zcu) {
1908	1764	// For objects, this mechanism relies on essentially `_ = @import("compiler-rt");`
1909	1765	// injected into the object.
1910		const compiler_rt_mod = Package.Module.create(arena, .{
	1766	const compiler_rt_mod = Module.create(arena, .{
1911	1767	.paths = .{
1912	1768	.root = .zig_lib_root,
1913	1769	.root_src_path = "compiler_rt.zig",
...	...	@@ -1969,7 +1825,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
1969	1825	};
1970	1826
1971	1827	if (ubsan_rt_strat == .zcu) {
1972		const ubsan_rt_mod = Package.Module.create(arena, .{
	1828	const ubsan_rt_mod = Module.create(arena, .{
1973	1829	.paths = .{
1974	1830	.root = .zig_lib_root,
1975	1831	.root_src_path = "ubsan_rt.zig",
...	...	@@ -2010,7 +1866,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
2010	1866	};
2011	1867
2012	1868	if (zigc_strat == .zcu) {
2013		const zigc_mod = Package.Module.create(arena, .{
	1869	const zigc_mod = Module.create(arena, .{
2014	1870	.paths = .{
2015	1871	.root = .zig_lib_root,
2016	1872	.root_src_path = "c.zig",
...	...	@@ -2140,7 +1996,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
2140	1996	.path = try options.dirs.global_cache.join(arena, &.{zir_sub_dir}),
2141	1997	};
2142	1998
2143		const std_mod = options.std_mod orelse Package.Module.create(arena, .{
	1999	const std_mod = options.std_mod orelse Module.create(arena, .{
2144	2000	.paths = .{
2145	2001	.root = try .fromRoot(arena, options.dirs, .zig_lib, "std"),
2146	2002	.root_src_path = "std.zig",
...	...	@@ -2910,11 +2766,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
2910	2766	.file_open, .file_stat, .file_read, .file_hash => \|op\| {
2911	2767	const pp = man.files.keys()[op.file_index].prefixed_path;
2912	2768	const prefix = man.cache.prefixes()[pp.prefix];
2913		return comp.setMiscFailure(
2914		.check_whole_cache,
2915		"failed to check cache: '{f}{s}' {t} {t}",
2916		.{ prefix, pp.sub_path, man.diagnostic, op.err },
2917		);
	2769	return comp.setMiscFailure(.check_whole_cache, "failed to check cache: {f}{s} {t} {t}", .{
	2770	prefix, pp.sub_path, man.diagnostic, op.err,
	2771	});
2918	2772	},
2919	2773	},
2920	2774	error.OutOfMemory, error.Canceled => \|e\| return e,
...	...	@@ -4785,7 +4639,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4785	4639	var buffer: [1024]u8 = undefined;
4786	4640	var tar_file_writer = tar_file.writer(io, &buffer);
4787	4641
4788		var seen_table: std.array_hash_map.Auto(*Package.Module, []const u8) = .empty;
	4642	var seen_table: std.array_hash_map.Auto(*Module, []const u8) = .empty;
4789	4643	defer seen_table.deinit(comp.gpa);
4790	4644
4791	4645	try seen_table.put(comp.gpa, zcu.main_mod, comp.root_name);
...	...	@@ -4812,7 +4666,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4812	4666
4813	4667	fn docsCopyModule(
4814	4668	comp: *Compilation,
4815		module: *Package.Module,
	4669	module: *Module,
4816	4670	name: []const u8,
4817	4671	tar_file_writer: *Io.File.Writer,
4818	4672	) !void {
...	...	@@ -4892,7 +4746,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU
4892	4746
4893	4747	const optimize_mode = std.lang.OptimizeMode.ReleaseSmall;
4894	4748	const output_mode = std.lang.OutputMode.Exe;
4895		const resolved_target: Package.Module.ResolvedTarget = .{
	4749	const resolved_target: Module.ResolvedTarget = .{
4896	4750	.result = std.zig.system.resolveTargetQuery(io, .{
4897	4751	.cpu_arch = .wasm32,
4898	4752	.os_tag = .freestanding,
...	...	@@ -4932,7 +4786,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU
4932	4786
4933	4787	const dirs = comp.dirs.withoutLocalCache();
4934	4788
4935		const root_mod = Package.Module.create(arena, .{
	4789	const root_mod = Module.create(arena, .{
4936	4790	.paths = .{
4937	4791	.root = try .fromRoot(arena, dirs, .zig_lib, "docs/wasm"),
4938	4792	.root_src_path = src_basename,
...	...	@@ -4949,7 +4803,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU
4949	4803	comp.lockAndSetMiscFailure(.docs_wasm, "sub-compilation of docs_wasm failed: failed to create root module: {t}", .{err});
4950	4804	return error.AlreadyReported;
4951	4805	};
4952		const walk_mod = Package.Module.create(arena, .{
	4806	const walk_mod = Module.create(arena, .{
4953	4807	.paths = .{
4954	4808	.root = try .fromRoot(arena, dirs, .zig_lib, "docs/wasm"),
4955	4809	.root_src_path = "Walk.zig",
...	...	@@ -5034,7 +4888,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU
5034	4888
5035	4889	pub fn obtainCObjectCacheManifest(
5036	4890	comp: *const Compilation,
5037		owner_mod: *Package.Module,
	4891	owner_mod: *Module,
5038	4892	) Cache.Manifest {
5039	4893	var man = comp.cache_parent.obtain();
5040	4894
...	...	@@ -5082,7 +4936,7 @@ pub fn translateC(
5082	4936	ext: FileExt,
5083	4937	source_path: []const u8,
5084	4938	translated_basename: []const u8,
5085		owner_mod: *Package.Module,
	4939	owner_mod: *Module,
5086	4940	prog_node: std.Progress.Node,
5087	4941	environ_map: *const std.process.Environ.Map,
5088	4942	) !TranslateCResult {
...	...	@@ -6238,7 +6092,7 @@ fn addCommonCCArgs(
6238	6092	argv: *std.array_list.Managed([]const u8),
6239	6093	ext: FileExt,
6240	6094	out_dep_path: ?[]const u8,
6241		mod: *Package.Module,
	6095	mod: *Module,
6242	6096	c_frontend: Config.CFrontend,
6243	6097	) !void {
6244	6098	const target = &mod.resolved_target.result;
...	...	@@ -6592,7 +6446,7 @@ pub fn addCCArgs(
6592	6446	argv: *std.array_list.Managed([]const u8),
6593	6447	ext: FileExt,
6594	6448	out_dep_path: ?[]const u8,
6595		mod: *Package.Module,
	6449	mod: *Module,
6596	6450	) !void {
6597	6451	const target = &mod.resolved_target.result;
6598	6452
...	...	@@ -7383,7 +7237,7 @@ fn buildOutputFromZig(
7383	7237	return error.AlreadyReported;
7384	7238	};
7385	7239
7386		const root_mod = Package.Module.create(arena, .{
	7240	const root_mod = Module.create(arena, .{
7387	7241	.paths = .{
7388	7242	.root = .zig_lib_root,
7389	7243	.root_src_path = src_basename,
...	...	@@ -7530,7 +7384,7 @@ pub fn build_crt_file(
7530	7384	comp.lockAndSetMiscFailure(misc_task_tag, "sub-compilation of {t} failed: failed to resolve compilation config: {t}", .{ misc_task_tag, err });
7531	7385	return error.AlreadyReported;
7532	7386	};
7533		const root_mod = Package.Module.create(arena, .{
	7387	const root_mod = Module.create(arena, .{
7534	7388	.paths = .{
7535	7389	.root = .zig_lib_root,
7536	7390	.root_src_path = "",

src/Compilation/Config.zig+1-1

...	...	@@ -577,7 +577,7 @@ pub fn resolve(options: Options) ResolveError!Config {
577	577	}
578	578
579	579	const std = @import("std");
580		const Module = @import("../Package.zig").Module;
	580	const Module = @import("../Module.zig");
581	581	const Config = @This();
582	582	const target_util = @import("../target.zig");
583	583	const build_options = @import("build_options");

src/Module.zig created+523

...	...	@@ -0,0 +1,523 @@
	1	//! Corresponds to something that Zig source code can `@import`.
	2	const Module = @This();
	3
	4	const std = @import("std");
	5	const Allocator = std.mem.Allocator;
	6	const Cache = std.Build.Cache;
	7	const assert = std.debug.assert;
	8
	9	const target_util = @import("target.zig");
	10	const Builtin = @import("Builtin.zig");
	11	const Compilation = @import("Compilation.zig");
	12	const File = @import("Zcu.zig").File;
	13
	14	/// The root directory of the module. Only files inside this directory can be imported.
	15	root: Compilation.Path,
	16	/// Path to the root source file of this module. Relative to `root`. May contain path separators.
	17	root_src_path: []const u8,
	18	/// Name used in compile errors. Looks like "root.foo.bar".
	19	fully_qualified_name: []const u8,
	20	/// The dependency table of this module. The shared dependencies 'std' and
	21	/// 'root' are not specified in every module dependency table, but are stored
	22	/// separately in `Zcu`. 'builtin' is also not stored here, although it is
	23	/// not necessarily the same between all modules. Handling of `@import` in
	24	/// the rest of the compiler must detect these special names and use the
	25	/// correct module instead of consulting `deps`.
	26	deps: Deps = .{},
	27
	28	resolved_target: ResolvedTarget,
	29	optimize_mode: std.lang.OptimizeMode,
	30	code_model: std.lang.CodeModel,
	31	single_threaded: bool,
	32	error_tracing: bool,
	33	valgrind: bool,
	34	pic: bool,
	35	strip: bool,
	36	omit_frame_pointer: bool,
	37	stack_check: bool,
	38	stack_protector: u32,
	39	red_zone: bool,
	40	sanitize_c: std.zig.SanitizeC,
	41	sanitize_thread: bool,
	42	fuzz: bool,
	43	unwind_tables: std.lang.UnwindTables,
	44	cc_argv: []const []const u8,
	45	/// (SPIR-V) whether to generate a structured control flow graph or not
	46	structured_cfg: bool,
	47	no_builtin: bool,
	48
	49	pub const Deps = std.array_hash_map.String(*Module);
	50
	51	pub const CreateOptions = struct {
	52	paths: Paths,
	53	fully_qualified_name: []const u8,
	54
	55	cc_argv: []const []const u8,
	56	inherited: Inherited,
	57	global: Compilation.Config,
	58	/// If this is null then `resolved_target` must be non-null.
	59	parent: ?*Module,
	60
	61	pub const Paths = struct {
	62	root: Compilation.Path,
	63	/// Relative to `root`. May contain path separators.
	64	root_src_path: []const u8,
	65	};
	66
	67	pub const Inherited = struct {
	68	/// If this is null then `parent` must be non-null.
	69	resolved_target: ?ResolvedTarget = null,
	70	optimize_mode: ?std.lang.OptimizeMode = null,
	71	code_model: ?std.lang.CodeModel = null,
	72	single_threaded: ?bool = null,
	73	error_tracing: ?bool = null,
	74	valgrind: ?bool = null,
	75	pic: ?bool = null,
	76	strip: ?bool = null,
	77	omit_frame_pointer: ?bool = null,
	78	stack_check: ?bool = null,
	79	/// null means default.
	80	/// 0 means no stack protector.
	81	/// other number means stack protection with that buffer size.
	82	stack_protector: ?u32 = null,
	83	red_zone: ?bool = null,
	84	unwind_tables: ?std.lang.UnwindTables = null,
	85	sanitize_c: ?std.zig.SanitizeC = null,
	86	sanitize_thread: ?bool = null,
	87	fuzz: ?bool = null,
	88	structured_cfg: ?bool = null,
	89	no_builtin: ?bool = null,
	90	};
	91	};
	92
	93	pub const ResolvedTarget = struct {
	94	result: std.Target,
	95	is_native_os: bool,
	96	is_native_abi: bool,
	97	is_explicit_dynamic_linker: bool,
	98	llvm_cpu_features: ?[*:0]const u8 = null,
	99	};
	100
	101	pub const CreateError = error{
	102	OutOfMemory,
	103	ValgrindUnsupportedOnTarget,
	104	TargetRequiresSingleThreaded,
	105	BackendRequiresSingleThreaded,
	106	TargetRequiresPic,
	107	PieRequiresPic,
	108	DynamicLinkingRequiresPic,
	109	TargetHasNoRedZone,
	110	StackCheckUnsupportedByTarget,
	111	StackProtectorUnsupportedByTarget,
	112	StackProtectorUnavailableWithoutLibC,
	113	};
	114
	115	/// At least one of `parent` and `resolved_target` must be non-null.
	116	pub fn create(arena: Allocator, options: CreateOptions) !*Module {
	117	if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread);
	118	if (options.inherited.fuzz == true) assert(options.global.any_fuzz);
	119	if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded);
	120	if (options.inherited.unwind_tables) \|uwt\| if (uwt != .none) assert(options.global.any_unwind_tables);
	121	if (options.inherited.sanitize_c) \|sc\| if (sc != .off) assert(options.global.any_sanitize_c != .off);
	122	if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing);
	123
	124	const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target;
	125	const target = &resolved_target.result;
	126
	127	const optimize_mode = options.inherited.optimize_mode orelse
	128	if (options.parent) \|p\| p.optimize_mode else options.global.root_optimize_mode;
	129
	130	const strip = b: {
	131	if (options.inherited.strip) \|x\| break :b x;
	132	if (options.parent) \|p\| break :b p.strip;
	133	break :b options.global.root_strip;
	134	};
	135
	136	const zig_backend = target_util.zigBackend(target, options.global.use_llvm);
	137
	138	const valgrind = b: {
	139	if (!target_util.hasValgrindSupport(target, zig_backend)) {
	140	if (options.inherited.valgrind == true)
	141	return error.ValgrindUnsupportedOnTarget;
	142	break :b false;
	143	}
	144	if (options.inherited.valgrind) \|x\| break :b x;
	145	if (options.parent) \|p\| break :b p.valgrind;
	146	if (strip) break :b false;
	147	break :b optimize_mode == .Debug;
	148	};
	149
	150	const single_threaded = b: {
	151	if (target_util.alwaysSingleThreaded(target)) {
	152	if (options.inherited.single_threaded == false)
	153	return error.TargetRequiresSingleThreaded;
	154	break :b true;
	155	}
	156
	157	if (options.global.have_zcu) {
	158	if (!target_util.supportsThreads(target, zig_backend)) {
	159	if (options.inherited.single_threaded == false)
	160	return error.BackendRequiresSingleThreaded;
	161	break :b true;
	162	}
	163	}
	164
	165	if (options.inherited.single_threaded) \|x\| break :b x;
	166	if (options.parent) \|p\| break :b p.single_threaded;
	167	break :b target_util.defaultSingleThreaded(target);
	168	};
	169
	170	const error_tracing = b: {
	171	if (options.inherited.error_tracing) \|x\| break :b x;
	172	if (options.parent) \|p\| break :b p.error_tracing;
	173	break :b options.global.root_error_tracing;
	174	};
	175
	176	const pic = b: {
	177	if (target_util.requiresPic(target, options.global.link_libc)) {
	178	if (options.inherited.pic == false)
	179	return error.TargetRequiresPic;
	180	break :b true;
	181	}
	182	if (options.global.pie) {
	183	if (options.inherited.pic == false)
	184	return error.PieRequiresPic;
	185	break :b true;
	186	}
	187	if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) {
	188	if (options.inherited.pic == false)
	189	return error.DynamicLinkingRequiresPic;
	190	break :b true;
	191	}
	192	if (options.inherited.pic) \|x\| break :b x;
	193	if (options.parent) \|p\| break :b p.pic;
	194
	195	// Default to PIC on targets where we default to producing PIEs to make
	196	// the common case of linking objects and static libraries into an
	197	// executable work out of the box.
	198	break :b target_util.defaultPie(target);
	199	};
	200
	201	const red_zone = b: {
	202	if (!target_util.hasRedZone(target)) {
	203	if (options.inherited.red_zone == true)
	204	return error.TargetHasNoRedZone;
	205	break :b false;
	206	}
	207	if (options.inherited.red_zone) \|x\| break :b x;
	208	if (options.parent) \|p\| break :b p.red_zone;
	209	break :b true;
	210	};
	211
	212	const omit_frame_pointer = b: {
	213	if (options.inherited.omit_frame_pointer) \|x\| break :b x;
	214	if (options.parent) \|p\| break :b p.omit_frame_pointer;
	215	if (optimize_mode == .ReleaseSmall) {
	216	// On x86, in most cases, keeping the frame pointer usually results in smaller binary size.
	217	// This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer)
	218	// are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer).
	219	break :b !target.cpu.arch.isX86();
	220	}
	221	break :b false;
	222	};
	223
	224	const sanitize_thread = b: {
	225	if (options.inherited.sanitize_thread) \|x\| break :b x;
	226	if (options.parent) \|p\| break :b p.sanitize_thread;
	227	break :b false;
	228	};
	229
	230	const unwind_tables = b: {
	231	if (options.inherited.unwind_tables) \|x\| break :b x;
	232	if (options.parent) \|p\| break :b p.unwind_tables;
	233
	234	break :b target_util.defaultUnwindTables(
	235	target,
	236	options.global.link_libunwind,
	237	sanitize_thread or options.global.any_sanitize_thread,
	238	);
	239	};
	240
	241	const fuzz = b: {
	242	if (options.inherited.fuzz) \|x\| break :b x;
	243	if (options.parent) \|p\| break :b p.fuzz;
	244	break :b false;
	245	};
	246
	247	const code_model: std.lang.CodeModel = b: {
	248	if (options.inherited.code_model) \|x\| break :b x;
	249	if (options.parent) \|p\| break :b p.code_model;
	250	break :b .default;
	251	};
	252
	253	const is_safe_mode = switch (optimize_mode) {
	254	.Debug, .ReleaseSafe => true,
	255	.ReleaseFast, .ReleaseSmall => false,
	256	};
	257
	258	const sanitize_c: std.zig.SanitizeC = b: {
	259	if (options.inherited.sanitize_c) \|x\| break :b x;
	260	if (options.parent) \|p\| break :b p.sanitize_c;
	261	break :b switch (optimize_mode) {
	262	.Debug => .full,
	263	// It's recommended to use the minimal runtime in production
	264	// environments due to the security implications of the full runtime.
	265	// The minimal runtime doesn't provide much benefit over simply
	266	// trapping, however, so we do that instead.
	267	.ReleaseSafe => .trap,
	268	.ReleaseFast, .ReleaseSmall => .off,
	269	};
	270	};
	271
	272	const stack_check = b: {
	273	if (!target_util.supportsStackProbing(target, zig_backend)) {
	274	if (options.inherited.stack_check == true)
	275	return error.StackCheckUnsupportedByTarget;
	276	break :b false;
	277	}
	278	if (options.inherited.stack_check) \|x\| break :b x;
	279	if (options.parent) \|p\| break :b p.stack_check;
	280	break :b is_safe_mode;
	281	};
	282
	283	const stack_protector: u32 = sp: {
	284	const use_zig_backend = options.global.have_zcu or
	285	(options.global.any_c_source_files and options.global.c_frontend == .aro);
	286	if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) {
	287	if (options.inherited.stack_protector) \|x\| {
	288	if (x > 0) return error.StackProtectorUnsupportedByTarget;
	289	}
	290	break :sp 0;
	291	}
	292
	293	if (options.global.any_c_source_files and options.global.c_frontend == .clang and
	294	!target_util.clangSupportsStackProtector(target))
	295	{
	296	if (options.inherited.stack_protector) \|x\| {
	297	if (x > 0) return error.StackProtectorUnsupportedByTarget;
	298	}
	299	break :sp 0;
	300	}
	301
	302	// This logic is checking for linking libc because otherwise our start code
	303	// which is trying to set up TLS (i.e. the fs/gs registers) but the stack
	304	// protection code depends on fs/gs registers being already set up.
	305	// If we were able to annotate start code, or perhaps the entire std lib,
	306	// as being exempt from stack protection checks, we could change this logic
	307	// to supporting stack protection even when not linking libc.
	308	// TODO file issue about this
	309	if (!options.global.link_libc) {
	310	if (options.inherited.stack_protector) \|x\| {
	311	if (x > 0) return error.StackProtectorUnavailableWithoutLibC;
	312	}
	313	break :sp 0;
	314	}
	315
	316	if (options.inherited.stack_protector) \|x\| break :sp x;
	317	if (options.parent) \|p\| break :sp p.stack_protector;
	318	if (!is_safe_mode) break :sp 0;
	319
	320	break :sp target_util.default_stack_protector_buffer_size;
	321	};
	322
	323	const structured_cfg = b: {
	324	if (options.inherited.structured_cfg) \|x\| break :b x;
	325	if (options.parent) \|p\| break :b p.structured_cfg;
	326	// We always want a structured control flow in shaders. This option is
	327	// only relevant for OpenCL kernels.
	328	break :b switch (target.os.tag) {
	329	.opencl => false,
	330	else => true,
	331	};
	332	};
	333
	334	const no_builtin = b: {
	335	if (options.inherited.no_builtin) \|x\| break :b x;
	336	if (options.parent) \|p\| break :b p.no_builtin;
	337
	338	break :b target.cpu.arch.isBpf();
	339	};
	340
	341	const llvm_cpu_features: ?[*:0]const u8 = b: {
	342	if (resolved_target.llvm_cpu_features) \|x\| break :b x;
	343	if (!options.global.use_llvm) break :b null;
	344
	345	var buf = std.array_list.Managed(u8).init(arena);
	346	var disabled_features = std.array_list.Managed(u8).init(arena);
	347	defer disabled_features.deinit();
	348
	349	// Append disabled features after enabled ones, so that their effects aren't overwritten.
	350	for (target.cpu.arch.allFeaturesList()) \|feature\| {
	351	if (feature.llvm_name) \|llvm_name\| {
	352	// Ignore these until we figure out how to handle the concept of omitting features.
	353	// See https://github.com/ziglang/zig/issues/23539
	354	if (target_util.isDynamicAMDGCNFeature(target, feature)) continue;
	355
	356	if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue;
	357	if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue;
	358
	359	var is_enabled = target.cpu.features.isEnabled(feature.index);
	360	if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) {
	361	is_enabled = !omit_frame_pointer;
	362	}
	363
	364	if (is_enabled) {
	365	try buf.ensureUnusedCapacity(2 + llvm_name.len);
	366	buf.appendAssumeCapacity('+');
	367	buf.appendSliceAssumeCapacity(llvm_name);
	368	buf.appendAssumeCapacity(',');
	369	} else {
	370	try disabled_features.ensureUnusedCapacity(2 + llvm_name.len);
	371	disabled_features.appendAssumeCapacity('-');
	372	disabled_features.appendSliceAssumeCapacity(llvm_name);
	373	disabled_features.appendAssumeCapacity(',');
	374	}
	375	}
	376	}
	377
	378	try buf.appendSlice(disabled_features.items);
	379	if (buf.items.len == 0) break :b "";
	380	assert(std.mem.endsWith(u8, buf.items, ","));
	381	buf.items[buf.items.len - 1] = 0;
	382	buf.shrinkAndFree(buf.items.len);
	383	break :b buf.items[0 .. buf.items.len - 1 :0].ptr;
	384	};
	385
	386	const mod = try arena.create(Module);
	387	mod.* = .{
	388	.root = options.paths.root,
	389	.root_src_path = options.paths.root_src_path,
	390	.fully_qualified_name = options.fully_qualified_name,
	391	.resolved_target = .{
	392	.result = target.*,
	393	.is_native_os = resolved_target.is_native_os,
	394	.is_native_abi = resolved_target.is_native_abi,
	395	.is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker,
	396	.llvm_cpu_features = llvm_cpu_features,
	397	},
	398	.optimize_mode = optimize_mode,
	399	.single_threaded = single_threaded,
	400	.error_tracing = error_tracing,
	401	.valgrind = valgrind,
	402	.pic = pic,
	403	.strip = strip,
	404	.omit_frame_pointer = omit_frame_pointer,
	405	.stack_check = stack_check,
	406	.stack_protector = stack_protector,
	407	.code_model = code_model,
	408	.red_zone = red_zone,
	409	.sanitize_c = sanitize_c,
	410	.sanitize_thread = sanitize_thread,
	411	.fuzz = fuzz,
	412	.unwind_tables = unwind_tables,
	413	.cc_argv = options.cc_argv,
	414	.structured_cfg = structured_cfg,
	415	.no_builtin = no_builtin,
	416	};
	417	return mod;
	418	}
	419
	420	/// All fields correspond to `CreateOptions`.
	421	pub const LimitedOptions = struct {
	422	root: Compilation.Path,
	423	root_src_path: []const u8,
	424	fully_qualified_name: []const u8,
	425	};
	426
	427	/// This one can only be used if the Module will only be used for AstGen and earlier in
	428	/// the pipeline. Illegal behavior occurs if a limited module touches Sema.
	429	pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Module {
	430	const mod = try gpa.create(Module);
	431	mod.* = .{
	432	.root = options.root,
	433	.root_src_path = options.root_src_path,
	434	.fully_qualified_name = options.fully_qualified_name,
	435
	436	.resolved_target = undefined,
	437	.optimize_mode = undefined,
	438	.code_model = undefined,
	439	.single_threaded = undefined,
	440	.error_tracing = undefined,
	441	.valgrind = undefined,
	442	.pic = undefined,
	443	.strip = undefined,
	444	.omit_frame_pointer = undefined,
	445	.stack_check = undefined,
	446	.stack_protector = undefined,
	447	.red_zone = undefined,
	448	.sanitize_c = undefined,
	449	.sanitize_thread = undefined,
	450	.fuzz = undefined,
	451	.unwind_tables = undefined,
	452	.cc_argv = undefined,
	453	.structured_cfg = undefined,
	454	.no_builtin = undefined,
	455	};
	456	return mod;
	457	}
	458
	459	/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task.
	460	pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: std.zig.Directories) Allocator.Error!*Module {
	461	const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash());
	462	const new = try arena.create(Module);
	463	new.* = .{
	464	.root = try .fromRoot(arena, dirs, .global_cache, sub_path),
	465	.root_src_path = "builtin.zig",
	466	.fully_qualified_name = "builtin",
	467	.resolved_target = .{
	468	.result = opts.target,
	469	// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
	470	.is_native_os = false,
	471	.is_native_abi = false,
	472	.is_explicit_dynamic_linker = false,
	473	.llvm_cpu_features = null,
	474	},
	475	.optimize_mode = opts.optimize_mode,
	476	.single_threaded = opts.single_threaded,
	477	.error_tracing = opts.error_tracing,
	478	.valgrind = opts.valgrind,
	479	.pic = opts.pic,
	480	.strip = opts.strip,
	481	.omit_frame_pointer = opts.omit_frame_pointer,
	482	.code_model = opts.code_model,
	483	.sanitize_thread = opts.sanitize_thread,
	484	.fuzz = opts.fuzz,
	485	.unwind_tables = opts.unwind_tables,
	486	.cc_argv = &.{},
	487	// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
	488	.stack_check = false,
	489	.stack_protector = 0,
	490	.red_zone = false,
	491	.sanitize_c = .off,
	492	.structured_cfg = false,
	493	.no_builtin = false,
	494	};
	495	return new;
	496	}
	497
	498	/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module.
	499	pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin {
	500	assert(global.have_zcu);
	501	return .{
	502	.target = m.resolved_target.result,
	503	.zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm),
	504	.output_mode = global.output_mode,
	505	.link_mode = global.link_mode,
	506	.unwind_tables = m.unwind_tables,
	507	.is_test = global.is_test,
	508	.single_threaded = m.single_threaded,
	509	.link_libc = global.link_libc,
	510	.link_libcpp = global.link_libcpp,
	511	.optimize_mode = m.optimize_mode,
	512	.error_tracing = m.error_tracing,
	513	.valgrind = m.valgrind,
	514	.sanitize_thread = m.sanitize_thread,
	515	.fuzz = m.fuzz,
	516	.pic = m.pic,
	517	.pie = global.pie,
	518	.strip = m.strip,
	519	.code_model = m.code_model,
	520	.omit_frame_pointer = m.omit_frame_pointer,
	521	.wasi_exec_model = global.wasi_exec_model,
	522	};
	523	}

src/Package.zig deleted-209

...	...	@@ -1,209 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3
4		pub const Module = @import("Package/Module.zig");
5		pub const Fetch = @import("Package/Fetch.zig");
6		pub const build_zig_basename = "build.zig";
7		pub const Manifest = @import("Package/Manifest.zig");
8
9		pub const Fingerprint = packed struct(u64) {
10		id: u32,
11		checksum: u32,
12
13		pub fn generate(rng: std.Random, name: []const u8) Fingerprint {
14		return .{
15		.id = rng.intRangeLessThan(u32, 1, 0xffffffff),
16		.checksum = std.hash.Crc32.hash(name),
17		};
18		}
19
20		pub fn validate(n: Fingerprint, name: []const u8) bool {
21		switch (n.id) {
22		0x00000000, 0xffffffff => return false,
23		else => return std.hash.Crc32.hash(name) == n.checksum,
24		}
25		}
26
27		pub fn int(n: Fingerprint) u64 {
28		return @bitCast(n);
29		}
30		};
31
32		/// A user-readable, file system safe hash that identifies an exact package
33		/// snapshot, including file contents.
34		///
35		/// The hash is not only to prevent collisions but must resist attacks where
36		/// the adversary fully controls the contents being hashed. Thus, it contains
37		/// a full SHA-256 digest.
38		///
39		/// This data structure can be used to store the legacy hash format too. Legacy
40		/// hash format is scheduled to be removed after 0.14.0 is tagged.
41		///
42		/// There's also a third way this structure is used. When using path rather than
43		/// hash, a unique hash is still needed, so one is computed based on the path.
44		pub const Hash = struct {
45		/// Maximum size of a package hash. Unused bytes at the end are
46		/// filled with zeroes.
47		///
48		/// Assumed to be already validated.
49		bytes: [max_len]u8,
50
51		pub const Algo = std.crypto.hash.sha2.Sha256;
52		pub const Digest = [Algo.digest_length]u8;
53
54		/// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
55		pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6;
56
57		/// Asserts `s` is valid.
58		pub fn fromSlice(s: []const u8) Hash {
59		assert(validate(s) == .ok);
60		var result: Hash = undefined;
61		@memcpy(result.bytes[0..s.len], s);
62		@memset(result.bytes[s.len..], 0);
63		return result;
64		}
65
66		pub const Validation = enum { ok, short, long, incomplete };
67
68		pub fn validate(s: []const u8) Validation {
69		if (s.len > max_len) return .long;
70		if (s.len < 44) return .short;
71		const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-');
72		if (n_dashes < 2) return .incomplete;
73		return .ok;
74		}
75
76		test validate {
77		try std.testing.expectEqual(.short, validate(""));
78		}
79
80		pub fn toSlice(ph: *const Hash) []const u8 {
81		var end: usize = ph.bytes.len;
82		while (true) {
83		end -= 1;
84		if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
85		}
86		}
87
88		pub fn eql(a: const Hash, b: const Hash) bool {
89		return std.mem.eql(u8, &a.bytes, &b.bytes);
90		}
91
92		/// Produces "$name-$semver-$hashplus".
93		/// * name is the name field from build.zig.zon, asserted to be at most 32
94		/// bytes and assumed be a valid zig identifier
95		/// * semver is the version field from build.zig.zon, asserted to be at
96		/// most 32 bytes
97		/// * hashplus is the following 33-byte array, base64 encoded using -_ to make
98		/// it filesystem safe:
99		/// - (4 bytes) LE u32 Package ID
100		/// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
101		/// - (25 bytes) truncated SHA-256 digest of hashed files of the package
102		pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash {
103		assert(name.len <= 32);
104		assert(ver.len <= 32);
105		var result: Hash = undefined;
106		var buf: std.ArrayList(u8) = .initBuffer(&result.bytes);
107		buf.appendSliceAssumeCapacity(name);
108		buf.appendAssumeCapacity('-');
109		buf.appendSliceAssumeCapacity(ver);
110		buf.appendAssumeCapacity('-');
111		var hashplus: [33]u8 = undefined;
112		std.mem.writeInt(u32, hashplus[0..4], id, .little);
113		std.mem.writeInt(u32, hashplus[4..8], size, .little);
114		hashplus[8..].* = digest[0..25].*;
115		_ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus);
116		@memset(buf.unusedCapacitySlice(), 0);
117		return result;
118		}
119
120		/// Produces a unique hash based on the path provided. The result should
121		/// not be user-visible.
122		pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
123		var result: Hash = .{ .bytes = @splat(0) };
124		var i: usize = 0;
125		if (is_global) {
126		result.bytes[0] = '/';
127		i += 1;
128		}
129		if (i + sub_path.len <= result.bytes.len) {
130		@memcpy(result.bytes[i..][0..sub_path.len], sub_path);
131		return result;
132		}
133		var bin_digest: [Algo.digest_length]u8 = undefined;
134		Algo.hash(sub_path, &bin_digest, .{});
135		_ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable;
136		return result;
137		}
138
139		pub fn projectId(hash: *const Hash) ProjectId {
140		const bytes = hash.toSlice();
141		const name = std.mem.sliceTo(bytes, '-');
142		const encoded_hashplus = bytes[bytes.len - 44 ..];
143		var hashplus: [33]u8 = undefined;
144		std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable;
145		const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little);
146		return .init(name, fingerprint_id);
147		}
148
149		test projectId {
150		const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw");
151		const project_id = hash.projectId();
152
153		var expected_name: [32]u8 = @splat(0);
154		expected_name[0.."pulseaudio".len].* = "pulseaudio".*;
155		try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
156
157		try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id);
158		}
159
160		test "projectId with dashes in the base64" {
161		const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC");
162		const project_id = hash.projectId();
163
164		var expected_name: [32]u8 = @splat(0);
165		expected_name[0.."dvui".len].* = "dvui".*;
166		try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
167
168		try std.testing.expectEqual(0x99490101, project_id.fingerprint_id);
169		}
170		};
171
172		/// Minimum information required to identify whether a package is an artifact
173		/// of a given project.
174		pub const ProjectId = struct {
175		/// Bytes after name.len are set to zero.
176		padded_name: [32]u8,
177		fingerprint_id: u32,
178
179		pub fn init(name: []const u8, fingerprint_id: u32) ProjectId {
180		var padded_name: [32]u8 = @splat(0);
181		@memcpy(padded_name[0..name.len], name);
182		return .{
183		.padded_name = padded_name,
184		.fingerprint_id = fingerprint_id,
185		};
186		}
187
188		pub fn eql(a: const ProjectId, b: const ProjectId) bool {
189		return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name);
190		}
191
192		pub fn hash(a: *const ProjectId) u64 {
193		const x: u64 = @bitCast(a.padded_name[0..8].*);
194		return std.hash.int(x \| a.fingerprint_id);
195		}
196		};
197
198		test Hash {
199		const example_digest: Hash.Digest = .{
200		0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
201		0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
202		};
203		const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024);
204		try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice());
205		}
206
207		test {
208		_ = Fetch;
209		}

src/Package/Fetch.zig deleted-2283

...	...	@@ -1,2283 +0,0 @@
1		//! Represents one independent job whose responsibility is to:
2		//!
3		//! 1. Check the local zig package directory to see if the hash already exists.
4		//! If so, load, parse, and validate the build.zig.zon file therein, and
5		//! goto step 9. Likewise if the location is a relative path, treat this
6		//! the same as a cache hit. Otherwise, proceed.
7		//! 2. Check the global package cache for a compressed tarball matching the
8		//! hash. If it is found, unpack the contents into a temporary directory inside
9		//! project local zig cache. Rename this directory into the local zig package
10		//! directory and goto step 9, skipping step 10.
11		//! 3. Fetch and unpack a URL into a temporary directory.
12		//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
13		//! for the file to be missing, in which case this fetched package is considered
14		//! to be a "naked" package.
15		//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
16		//! deleting excluded files. If any files had errors for files that were
17		//! ultimately excluded, those errors should be ignored, such as failure to
18		//! create symlinks that weren't supposed to be included anyway.
19		//! 6. Compute the package hash based on the remaining files in the temporary
20		//! directory.
21		//! 7. Rename the temporary directory into the local zig package directory. If
22		//! the hash already exists, delete the temporary directory and leave the zig
23		//! package directory untouched as it may be in use. This is done even if
24		//! the hash is invalid, in case the package with the different hash is used
25		//! in the future.
26		//! 8. Validate the computed hash against the expected hash. If invalid,
27		//! this job is done.
28		//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
29		//! a mutex and a hash map so that redundant jobs do not get queued up.
30		//! 10.Compress the package directory and store it into the global package
31		//! cache.
32		//!
33		//! All of this must be done with only referring to the state inside this struct
34		//! because this work will be done in a dedicated thread.
35		const Fetch = @This();
36
37		const builtin = @import("builtin");
38		const native_os = builtin.os.tag;
39
40		const std = @import("std");
41		const Io = std.Io;
42		const fs = std.fs;
43		const log = std.log.scoped(.fetch);
44		const assert = std.debug.assert;
45		const ascii = std.ascii;
46		const Allocator = std.mem.Allocator;
47		const Cache = std.Build.Cache;
48		const git = @import("Fetch/git.zig");
49		const Package = @import("../Package.zig");
50		const Manifest = Package.Manifest;
51		const ErrorBundle = std.zig.ErrorBundle;
52
53		arena: std.heap.ArenaAllocator,
54		location: Location,
55		location_tok: std.zig.Ast.TokenIndex,
56		hash_tok: std.zig.Ast.OptionalTokenIndex,
57		name_tok: std.zig.Ast.TokenIndex,
58		lazy_status: LazyStatus,
59		/// Same as `parent_packge_root` except it is unchanged when recursing into
60		/// relative file paths (as opposed to URL).
61		remote_package_root: Cache.Path,
62		parent_package_root: Cache.Path,
63		parent_manifest_ast: ?*const std.zig.Ast,
64		prog_node: std.Progress.Node,
65		job_queue: *JobQueue,
66		/// If true, don't add an error for a missing hash. This flag is not passed
67		/// down to recursive dependencies. It's intended to be used only be the CLI.
68		omit_missing_hash_error: bool,
69		/// If true, don't fail when a manifest file is missing the `paths` field,
70		/// which specifies inclusion rules. This is intended to be true for the first
71		/// fetch task and false for the recursive dependencies.
72		allow_missing_paths_field: bool,
73		/// If true and URL points to a Git repository, will use the latest commit.
74		use_latest_commit: bool,
75
76		// Above this are fields provided as inputs to `run`.
77		// Below this are fields populated by `run`.
78
79		/// Relative to the build root of the root package.
80		package_root: Cache.Path,
81		error_bundle: ErrorBundle.Wip,
82		manifest: Manifest,
83		manifest_ast: std.zig.Ast,
84		have_manifest: bool,
85		computed_hash: ComputedHash,
86		/// Fetch logic notices whether a package has a build.zig file and sets this flag.
87		has_build_zig: bool,
88		/// Indicates whether the task aborted due to an out-of-memory condition.
89		oom_flag: bool,
90		/// If `use_latest_commit` was true, this will be set to the commit that was used.
91		/// If the resource pointed to by the location is not a Git-repository, this
92		/// will be left unchanged.
93		latest_commit: ?git.Oid,
94
95		// This field is used by the CLI only, untouched by this file.
96
97		/// The module for this `Fetch` tasks's package, which exposes `build.zig` as
98		/// the root source file.
99		module: ?*Package.Module,
100
101		pub const LazyStatus = enum {
102		/// Not lazy.
103		eager,
104		/// Lazy, found.
105		available,
106		/// Lazy, not found.
107		unavailable,
108		};
109
110		pub const LocalStorage = struct {
111		cache_root: Cache.Path,
112		/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
113		pkg_root: Cache.Path,
114		};
115
116		/// Contains shared state among all `Fetch` tasks.
117		pub const JobQueue = struct {
118		io: Io,
119		mutex: Io.Mutex = .init,
120		/// It's an array hash map so that it can be sorted before rendering the
121		/// dependencies.zig source file.
122		/// Protected by `mutex`.
123		table: Table = .{},
124		/// `table` may be missing some tasks such as ones that failed, so this
125		/// field contains references to all of them.
126		/// Protected by `mutex`.
127		all_fetches: std.ArrayList(*Fetch) = .empty,
128		prog_node: std.Progress.Node,
129
130		http_client: *std.http.Client,
131		/// This tracks `Fetch` tasks as well as recompression tasks.
132		group: Io.Group = .init,
133		global_cache: Cache.Directory,
134		/// If `null`, indicates fetch globally only.
135		local_storage: ?*const LocalStorage,
136		/// If true then, no fetching occurs, and:
137		/// * The `global_cache` directory is assumed to be the direct parent
138		/// directory of on-disk packages rather than having the "p/" directory
139		/// prefix inside of it.
140		/// * An error occurs if any non-lazy packages are not already present in
141		/// the package cache directory.
142		/// * Missing hash field causes an error, and no fetching occurs so it does
143		/// not print the correct hash like usual.
144		read_only: bool,
145		recursive: bool,
146		/// Dumps hash information to stdout which can be used to troubleshoot why
147		/// two hashes of the same package do not match.
148		/// If this is true, `recursive` must be false.
149		debug_hash: bool,
150		mode: Mode,
151		/// Set of hashes that will be additionally fetched even if they are marked
152		/// as lazy.
153		unlazy_set: UnlazySet = .{},
154		/// Identifies paths that override all packages in the tree with matching
155		/// project ids.
156		fork_set: ForkSet = .{},
157
158		pub const Mode = enum {
159		/// Non-lazy dependencies are always fetched.
160		/// Lazy dependencies are fetched only when needed.
161		needed,
162		/// Both non-lazy and lazy dependencies are always fetched.
163		all,
164		};
165		pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch);
166		pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void);
167		pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false);
168
169		pub const Fork = struct {
170		path: Cache.Path,
171		manifest_ast: std.zig.Ast,
172		manifest: Package.Manifest,
173		uses: usize,
174
175		pub const Context = struct {
176		pub fn hash(_: @This(), a: Fork) u32 {
177		const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
178		return @truncate(project_id.hash());
179		}
180
181		pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool {
182		const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
183		const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
184		return a_project_id.eql(&b_project_id);
185		}
186		};
187
188		pub const Adapter = struct {
189		pub fn hash(_: @This(), a: Package.ProjectId) u32 {
190		return @truncate(a.hash());
191		}
192
193		pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool {
194		const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
195		return a_project_id.eql(&b_project_id);
196		}
197		};
198		};
199
200		pub fn deinit(jq: *JobQueue) void {
201		const io = jq.io;
202		jq.group.cancel(io);
203		if (jq.all_fetches.items.len == 0) return;
204		const gpa = jq.all_fetches.items[0].arena.child_allocator;
205		jq.table.deinit(gpa);
206		// These must be deinitialized in reverse order because subsequent
207		// `Fetch` instances are allocated in prior ones' arenas.
208		// Sorry, I know it's a bit weird, but it slightly simplifies the
209		// critical section.
210		while (jq.all_fetches.pop()) \|f\| f.deinit();
211		jq.all_fetches.deinit(gpa);
212		jq.* = undefined;
213		}
214
215		/// Dumps all subsequent error bundles into the first one.
216		pub fn consolidateErrors(jq: *JobQueue) !void {
217		const root = &jq.all_fetches.items[0].error_bundle;
218		const gpa = root.gpa;
219		for (jq.all_fetches.items[1..]) \|fetch\| {
220		if (fetch.error_bundle.root_list.items.len > 0) {
221		var bundle = try fetch.error_bundle.toOwnedBundle("");
222		defer bundle.deinit(gpa);
223		try root.addBundleAsRoots(bundle);
224		}
225		}
226		}
227
228		/// Creates the dependencies.zig source code for the build runner to obtain
229		/// via `@import("@dependencies")`.
230		pub fn createDependenciesSource(jq: JobQueue, buf: std.array_list.Managed(u8)) Allocator.Error!void {
231		const keys = jq.table.keys();
232
233		assert(keys.len != 0); // caller should have added the first one
234		if (keys.len == 1) {
235		// This is the first one. It must have no dependencies.
236		return createEmptyDependenciesSource(buf);
237		}
238
239		try buf.appendSlice("pub const packages = struct {\n");
240
241		// Ensure the generated .zig file is deterministic.
242		jq.table.sortUnstable(@as(struct {
243		keys: []const Package.Hash,
244		pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
245		return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
246		}
247		}, .{ .keys = keys }));
248
249		for (keys, jq.table.values()) \|*hash, fetch\| {
250		if (fetch == jq.all_fetches.items[0]) {
251		// The first one is a dummy package for the current project.
252		continue;
253		}
254
255		const hash_slice = hash.toSlice();
256
257		try buf.print(
258		\\ pub const {f} = struct {{
259		\\
260		, .{std.zig.fmtId(hash_slice)});
261
262		lazy: {
263		switch (fetch.lazy_status) {
264		.eager => break :lazy,
265		.available => {
266		try buf.appendSlice(
267		\\ pub const available = true;
268		\\
269		);
270		break :lazy;
271		},
272		.unavailable => {
273		try buf.appendSlice(
274		\\ pub const available = false;
275		\\ };
276		\\
277		);
278		continue;
279		},
280		}
281		}
282
283		try buf.print(
284		\\ pub const build_root = "{f}";
285		\\
286		, .{std.fmt.alt(fetch.package_root, .formatEscapeString)});
287
288		if (fetch.has_build_zig) {
289		try buf.print(
290		\\ pub const build_zig = @import("{f}");
291		\\
292		, .{std.zig.fmtString(hash_slice)});
293		}
294
295		if (fetch.have_manifest) {
296		const manifest = &fetch.manifest;
297		try buf.appendSlice(
298		\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
299		\\
300		);
301		for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
302		const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
303		try buf.print(
304		" .{{ \"{f}\", \"{f}\" }},\n",
305		.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
306		);
307		}
308
309		try buf.appendSlice(
310		\\ };
311		\\ };
312		\\
313		);
314		} else {
315		try buf.appendSlice(
316		\\ pub const deps: []const struct { []const u8, []const u8 } = &.{};
317		\\ };
318		\\
319		);
320		}
321		}
322
323		try buf.appendSlice(
324		\\};
325		\\
326		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
327		\\
328		);
329
330		const root_fetch = jq.all_fetches.items[0];
331		assert(root_fetch.have_manifest);
332		const root_manifest = &root_fetch.manifest;
333
334		for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) \|name, dep\| {
335		const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
336		try buf.print(
337		" .{{ \"{f}\", \"{f}\" }},\n",
338		.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
339		);
340		}
341		try buf.appendSlice("};\n");
342		}
343
344		pub fn createEmptyDependenciesSource(buf: *std.array_list.Managed(u8)) Allocator.Error!void {
345		try buf.appendSlice(
346		\\pub const packages = struct {};
347		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
348		\\
349		);
350		}
351
352		fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void {
353		const pkg_hash_slice = package_hash.toSlice();
354
355		const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
356		defer prog_node.end();
357
358		var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
359		const dest_path: Cache.Path = .{
360		.root_dir = jq.global_cache,
361		.sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
362		};
363
364		const gpa = jq.http_client.allocator;
365
366		var arena_instance = std.heap.ArenaAllocator.init(gpa);
367		defer arena_instance.deinit();
368		const arena = arena_instance.allocator();
369
370		recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch \|err\| switch (err) {
371		error.Canceled => \|e\| return e,
372		error.ReadFailed => comptime unreachable,
373		error.WriteFailed => comptime unreachable,
374		else => \|e\| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
375		};
376		}
377
378		fn recompressFallible(
379		jq: *JobQueue,
380		arena: Allocator,
381		dest_path: Cache.Path,
382		pkg_hash_slice: []const u8,
383		package_root: Cache.Path,
384		prog_node: std.Progress.Node,
385		) !void {
386		const gpa = jq.http_client.allocator;
387		const io = jq.io;
388
389		// We have to walk the file system up front in order to sort the file
390		// list for determinism purposes. The hash of the recompressed file is
391		// not critical because the true hash is based on the content alone.
392		// However, if we want Zig users to be able to share cached package
393		// data with each other via peer-to-peer protocols, we benefit greatly
394		// from the data being identical on everyone's computers.
395		var scanned_files: std.ArrayList(ScannedFile) = .empty;
396		defer scanned_files.deinit(gpa);
397
398		var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
399		defer pkg_dir.close(io);
400
401		{
402		var walker = try pkg_dir.walk(gpa);
403		defer walker.deinit();
404
405		while (try walker.next(io)) \|entry\| {
406		const symlink = switch (entry.kind) {
407		.directory => continue,
408		.file => false,
409		.sym_link => true,
410		else => return error.IllegalFileType,
411		};
412		const entry_path = try arena.dupe(u8, entry.path);
413		// If necessary, normalize path separators to POSIX-style since the tar format requires that.
414		if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) {
415		std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix);
416		}
417		try scanned_files.append(gpa, .{
418		.ptr = entry_path.ptr,
419		.len = @intCast(entry_path.len),
420		.symlink = symlink,
421		});
422		}
423
424		std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp);
425		}
426
427		prog_node.setEstimatedTotalItems(scanned_files.items.len);
428
429		var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
430		.make_path = true,
431		.replace = true,
432		});
433		defer atomic_file.deinit(io);
434
435		var file_write_buffer: [4096]u8 = undefined;
436		var file_writer = atomic_file.file.writer(io, &file_write_buffer);
437
438		var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
439		var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch \|err\| switch (err) {
440		error.WriteFailed => return file_writer.err.?,
441		};
442
443		var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
444		archiver.prefix = pkg_hash_slice;
445
446		var file_read_buffer: [4096]u8 = undefined;
447		var link_buf: [fs.max_path_bytes]u8 = undefined;
448
449		for (scanned_files.items) \|scanned_file\| {
450		const entry_path = scanned_file.ptr[0..scanned_file.len];
451		if (scanned_file.symlink) {
452		const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)];
453		archiver.writeLink(entry_path, link_name, .{}) catch \|err\| switch (err) {
454		error.WriteFailed => return file_writer.err.?,
455		else => \|e\| return e,
456		};
457		} else {
458		var file = try pkg_dir.openFile(io, entry_path, .{});
459		defer file.close(io);
460		var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
461		archiver.writeFile(entry_path, &file_reader, 0) catch \|err\| switch (err) {
462		error.ReadFailed => return file_reader.err.?,
463		error.WriteFailed => return file_writer.err.?,
464		else => \|e\| return e,
465		};
466		}
467		prog_node.completeOne();
468		}
469
470		// intentionally omitting the pointless trailer
471		//try archiver.finish();
472		compress.finish() catch \|err\| switch (err) {
473		error.WriteFailed => return file_writer.err.?,
474		};
475		try file_writer.flush();
476		try atomic_file.replace(io);
477		}
478		};
479
480		const ScannedFile = struct {
481		ptr: [*]const u8,
482		len: u32,
483		symlink: bool,
484		};
485
486		fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool {
487		return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]);
488		}
489
490		pub const Location = union(enum) {
491		remote: Remote,
492		/// A directory found inside the parent package.
493		relative_path: Cache.Path,
494		/// Recursive Fetch tasks will never use this Location, but it may be
495		/// passed in by the CLI. Indicates the file contents here should be copied
496		/// into the global package cache. It may be a file relative to the cwd or
497		/// absolute, in which case it should be treated exactly like a `file://`
498		/// URL, or a directory, in which case it should be treated as an
499		/// already-unpacked directory (but still needs to be copied into the
500		/// global package cache and have inclusion rules applied).
501		path_or_url: []const u8,
502
503		pub const Remote = struct {
504		url: []const u8,
505		/// If this is null it means the user omitted the hash field from a dependency.
506		/// It will be an error but the logic should still fetch and print the discovered hash.
507		hash: ?Package.Hash,
508		};
509		};
510
511		pub const RunError = error{
512		OutOfMemory,
513		Canceled,
514		/// This error code is intended to be handled by inspecting the
515		/// `error_bundle` field.
516		FetchFailed,
517		};
518
519		pub fn run(f: *Fetch) RunError!void {
520		const job_queue = f.job_queue;
521		const io = job_queue.io;
522		const eb = &f.error_bundle;
523		const arena = f.arena.allocator();
524		const gpa = f.arena.child_allocator;
525
526		try eb.init(gpa);
527
528		// Check the global zig package cache to see if the hash already exists. If
529		// so, load, parse, and validate the build.zig.zon file therein, and skip
530		// ahead to queuing up jobs for dependencies. Likewise if the location is a
531		// relative path, treat this the same as a cache hit. Otherwise, proceed.
532
533		const remote = switch (f.location) {
534		.relative_path => \|pkg_root\| {
535		if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
536		f.location_tok,
537		try eb.addString("expected path relative to build root; found absolute path"),
538		);
539		if (f.hash_tok.unwrap()) \|hash_tok\| return f.fail(
540		hash_tok,
541		try eb.addString("path-based dependencies are not hashed"),
542		);
543		// Packages fetched by URL may not use relative paths to escape outside the
544		// fetched package directory from within the package cache.
545
546		// This code path is only reachable recursively and the sub_path
547		// will already have been resolved to no longer have extra ".." or
548		// "." components.
549		assert(job_queue.local_storage != null);
550		log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{
551		pkg_root.sub_path, f.remote_package_root.sub_path,
552		});
553		assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir));
554		if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail(
555		f.location_tok,
556		try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
557		);
558		f.package_root = pkg_root;
559		try loadManifest(f, pkg_root);
560		if (!f.has_build_zig) try checkBuildFileExistence(f);
561		if (!job_queue.recursive) return;
562		return queueJobsForDeps(f);
563		},
564		.remote => \|remote\| remote,
565		.path_or_url => \|path_or_url\| {
566		if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) \|dir\| {
567		var resource: Resource = .{ .dir = dir };
568		return f.runResource(path_or_url, &resource, null, false);
569		} else \|dir_err\| {
570		var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
571
572		const file_err = if (dir_err == error.NotDir) e: {
573		if (Io.Dir.cwd().openFile(io, path_or_url, .{})) \|file\| {
574		var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };
575		return f.runResource(path_or_url, &resource, null, false);
576		} else \|err\| break :e err;
577		} else dir_err;
578
579		const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
580		return f.fail(0, try eb.printString(
581		"'{s}' could not be recognized as a file path ({t}) or an URL ({t})",
582		.{ path_or_url, file_err, uri_err },
583		));
584		};
585		var resource: Resource = undefined;
586		try f.initResource(uri, &resource, &server_header_buffer);
587		return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false);
588		}
589		},
590		};
591
592		var resource_buffer: [init_resource_buffer_size]u8 = undefined;
593
594		if (remote.hash) \|expected_hash\| {
595		const expected_project_id: Package.ProjectId = expected_hash.projectId();
596		if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) \|fork\| {
597		log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
598		fork.uses += 1;
599		f.package_root = fork.path;
600		f.remote_package_root = f.package_root;
601		f.manifest_ast = fork.manifest_ast;
602		f.manifest = fork.manifest;
603		f.have_manifest = true;
604		try checkBuildFileExistence(f);
605		if (!job_queue.recursive) return;
606		return queueJobsForDeps(f);
607		}
608
609		if (job_queue.local_storage) \|ls\| {
610		const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
611		if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) \|_\| {
612		assert(f.lazy_status != .unavailable);
613		f.package_root = package_root;
614		f.remote_package_root = f.package_root;
615		try loadManifest(f, f.package_root);
616		try checkBuildFileExistence(f);
617		if (!job_queue.recursive) return;
618		return queueJobsForDeps(f);
619		} else \|err\| switch (err) {
620		error.FileNotFound => {
621		log.debug("FileNotFound: {f}", .{package_root});
622		if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
623		f.name_tok,
624		try eb.printString("package not found at '{f}'", .{package_root}),
625		);
626		},
627		error.Canceled => \|e\| return e,
628		else => \|e\| {
629		try eb.addRootErrorMessage(.{
630		.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
631		package_root, e,
632		}),
633		});
634		return error.FetchFailed;
635		},
636		}
637		}
638
639		// Check global cache before remote fetch.
640		const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()});
641		const cached_tarball_path: Cache.Path = .{
642		.root_dir = job_queue.global_cache,
643		.sub_path = cached_tarball_sub_path,
644		};
645		if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) \|file\| {
646		log.debug("found global cached tarball {f}", .{cached_tarball_path});
647		var resource: Resource = .{ .file = file.reader(io, &resource_buffer) };
648		return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true);
649		} else \|err\| switch (err) {
650		error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}),
651		error.Canceled => \|e\| return e,
652		else => \|e\| {
653		try eb.addRootErrorMessage(.{
654		.msg = try eb.printString("unable to open globally cached package {f}: {t}", .{
655		cached_tarball_path, e,
656		}),
657		});
658		return error.FetchFailed;
659		},
660		}
661
662		switch (f.lazy_status) {
663		.eager => {},
664		.available => if (!job_queue.unlazy_set.contains(expected_hash)) {
665		f.lazy_status = .unavailable;
666		return;
667		},
668		.unavailable => unreachable,
669		}
670		} else if (job_queue.read_only) {
671		try eb.addRootErrorMessage(.{
672		.msg = try eb.addString("dependency is missing hash field"),
673		.src_loc = try f.srcLoc(f.location_tok),
674		});
675		return error.FetchFailed;
676		}
677
678		// Fetch and unpack the remote into a temporary directory.
679		const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
680		f.location_tok,
681		try eb.printString("invalid URI: {t}", .{err}),
682		);
683		var resource: Resource = undefined;
684		try f.initResource(uri, &resource, &resource_buffer);
685		return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false);
686		}
687
688		pub fn deinit(f: *Fetch) void {
689		f.error_bundle.deinit();
690		f.arena.deinit();
691		}
692
693		/// Consumes `resource`, even if an error is returned.
694		fn runResource(
695		f: *Fetch,
696		uri_path: []const u8,
697		resource: *Resource,
698		remote_hash: ?Package.Hash,
699		disable_recompress: bool,
700		) RunError!void {
701		const job_queue = f.job_queue;
702		assert(!job_queue.read_only);
703
704		const io = job_queue.io;
705		defer resource.deinit(io);
706
707		const arena = f.arena.allocator();
708		const eb = &f.error_bundle;
709		const rand_int = r: {
710		var x: u64 = undefined;
711		io.random(@ptrCast(&x));
712		break :r x;
713		};
714		const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
715		const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
716		const tmp_directory_path: Cache.Path = if (job_queue.local_storage) \|ls\|
717		try ls.pkg_root.join(arena, tmp_dir_sub_path)
718		else
719		.{
720		.root_dir = job_queue.global_cache,
721		.sub_path = tmp_tmp_dir_sub_path,
722		};
723
724		const package_sub_path = blk: {
725		var tmp_directory: Cache.Directory = .{
726		.path = tmp_directory_path.sub_path,
727		.handle = handle: {
728		const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
729		.open_options = .{ .iterate = true },
730		}) catch \|err\| {
731		try eb.addRootErrorMessage(.{
732		.msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
733		tmp_directory_path, err,
734		}),
735		});
736		return error.FetchFailed;
737		};
738		break :handle dir;
739		},
740		};
741		defer tmp_directory.handle.close(io);
742
743		// Fetch and unpack a resource into a temporary directory.
744		var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
745
746		const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
747
748		// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
749		// for the file to be missing, in which case this fetched package is
750		// considered to be a "naked" package.
751		try loadManifest(f, pkg_path);
752
753		const filter: Filter = .{
754		.include_paths = if (f.have_manifest) f.manifest.paths else .{},
755		};
756
757		// Ignore errors that were excluded by manifest, such as failure to
758		// create symlinks that weren't supposed to be included anyway.
759		try unpack_result.validate(f, filter);
760
761		// Apply the manifest's inclusion rules to the temporary directory by
762		// deleting excluded files.
763		// Empty directories have already been omitted by `unpackResource`.
764		// Compute the package hash based on the remaining files in the temporary
765		// directory.
766		f.computed_hash = try computeHash(f, pkg_path, filter);
767
768		if (unpack_result.root_dir.len > 0)
769		break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
770
771		break :blk tmp_directory_path;
772		};
773
774		const computed_package_hash = computedPackageHash(f);
775
776		// Rename the temporary directory into the local zig package directory. If
777		// the hash already exists, delete the temporary directory and leave the
778		// zig package directory untouched as it may be in use. This is done even
779		// if the hash is invalid, in case the package with the different hash is
780		// used in the future.
781		if (job_queue.local_storage) \|ls\| {
782		f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
783		renameTmpIntoCache(io, package_sub_path, f.package_root) catch \|err\| {
784		try eb.addRootErrorMessage(.{ .msg = try eb.printString(
785		"failed renaming temporary directory {f} into package cache directory {f}: {t}",
786		.{ package_sub_path, f.package_root, err },
787		) });
788		return error.FetchFailed;
789		};
790		} else {
791		f.package_root = tmp_directory_path;
792		}
793		f.remote_package_root = f.package_root;
794
795		if (!disable_recompress) {
796		// Spin off a task to recompress the tarball, with filtered files deleted, into
797		// the global cache.
798		job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
799		}
800
801		// Remove temporary directory root if not already renamed to global cache.
802		if (!package_sub_path.eql(tmp_directory_path)) {
803		tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch \|err\| switch (err) {
804		error.Canceled => \|e\| return e,
805		else => \|e\| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }),
806		};
807		}
808
809		// Validate the computed hash against the expected hash. If invalid, this
810		// job is done.
811
812		if (remote_hash) \|declared_hash\| {
813		const hash_tok = f.hash_tok.unwrap().?;
814		if (!computed_package_hash.eql(&declared_hash)) {
815		return f.fail(hash_tok, try eb.printString(
816		"hash mismatch: manifest declares {s} but the fetched package has {s}",
817		.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
818		));
819		}
820		} else if (!f.omit_missing_hash_error) {
821		const notes_len = 1;
822		try eb.addRootErrorMessage(.{
823		.msg = try eb.addString("dependency is missing hash field"),
824		.src_loc = try f.srcLoc(f.location_tok),
825		.notes_len = notes_len,
826		});
827		const notes_start = try eb.reserveNotes(notes_len);
828		eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
829		.msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
830		}));
831		return error.FetchFailed;
832		}
833
834		// Spawn a new fetch job for each dependency in the manifest file. Use
835		// a mutex and a hash map so that redundant jobs do not get queued up.
836		if (!job_queue.recursive) return;
837		return queueJobsForDeps(f);
838		}
839
840		pub fn computedPackageHash(f: *const Fetch) Package.Hash {
841		const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
842		if (f.have_manifest) {
843		const man = &f.manifest;
844		var version_buffer: [32]u8 = undefined;
845		const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer;
846		return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
847		}
848		// In the future build.zig.zon fields will be added to allow overriding these values
849		// for naked tarballs.
850		return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
851		}
852
853		/// `computeHash` gets a free check for the existence of `build.zig`, but when
854		/// not computing a hash, we need to do a syscall to check for it.
855		fn checkBuildFileExistence(f: *Fetch) RunError!void {
856		const io = f.job_queue.io;
857		const eb = &f.error_bundle;
858		if (f.package_root.access(io, Package.build_zig_basename, .{})) \|_\| {
859		f.has_build_zig = true;
860		} else \|err\| switch (err) {
861		error.FileNotFound => {},
862		else => \|e\| {
863		try eb.addRootErrorMessage(.{
864		.msg = try eb.printString("unable to access '{f}{s}': {t}", .{
865		f.package_root, Package.build_zig_basename, e,
866		}),
867		});
868		return error.FetchFailed;
869		},
870		}
871		}
872
873		/// This function populates `f.manifest` or leaves it `null`.
874		fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
875		const io = f.job_queue.io;
876		const eb = &f.error_bundle;
877		const arena = f.arena.allocator();
878		const manifest_path = try pkg_root.join(arena, Manifest.basename);
879
880		Manifest.load(
881		io,
882		arena,
883		manifest_path,
884		&f.manifest_ast,
885		eb,
886		&f.manifest,
887		f.allow_missing_paths_field,
888		) catch \|err\| switch (err) {
889		error.FileNotFound => return,
890		error.Canceled => \|e\| return e,
891		error.ErrorsBundled => return error.FetchFailed,
892		else => \|e\| {
893		try eb.addRootErrorMessage(.{
894		.msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
895		});
896		return error.FetchFailed;
897		},
898		};
899		f.have_manifest = true;
900		}
901
902		fn queueJobsForDeps(f: *Fetch) RunError!void {
903		const io = f.job_queue.io;
904
905		assert(f.job_queue.recursive);
906
907		// If the package does not have a build.zig.zon file then there are no dependencies.
908		if (!f.have_manifest) return;
909		const manifest = &f.manifest;
910
911		const new_fetches, const prog_names = nf: {
912		const parent_arena = f.arena.allocator();
913		const gpa = f.arena.child_allocator;
914		const cache_root = f.job_queue.global_cache;
915		const dep_names = manifest.dependencies.keys();
916		const deps = manifest.dependencies.values();
917		// Grab the new tasks into a temporary buffer so we can unlock that mutex
918		// as fast as possible.
919		// This overallocates any fetches that get skipped by the `continue` in the
920		// loop below.
921		const new_fetches = try parent_arena.alloc(Fetch, deps.len);
922		const prog_names = try parent_arena.alloc([]const u8, deps.len);
923		var new_fetch_index: usize = 0;
924
925		try f.job_queue.mutex.lock(io);
926		defer f.job_queue.mutex.unlock(io);
927
928		try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
929		try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
930
931		// There are four cases here:
932		// * Correct hash is provided by manifest.
933		// - Hash map already has the entry, no need to add it again.
934		// * Incorrect hash is provided by manifest.
935		// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
936		// * Hash is not provided by manifest.
937		// - Hash missing error emitted; `queueJobsForDeps` is not called.
938		// * path-based location is used without a hash.
939		// - Hash is added to the table based on the path alone before
940		// calling run(); no need to add it again.
941		//
942		// If we add a dep as lazy and then later try to add the same dep as eager,
943		// eagerness takes precedence and the existing entry is updated and re-scheduled
944		// for fetching.
945
946		for (dep_names, deps) \|dep_name, dep\| {
947		var promoted_existing_to_eager = false;
948		const new_fetch = &new_fetches[new_fetch_index];
949		const location: Location = switch (dep.location) {
950		.url => \|url\| .{
951		.remote = .{
952		.url = url,
953		.hash = h: {
954		const h = dep.hash orelse break :h null;
955		const pkg_hash: Package.Hash = .fromSlice(h);
956		if (h.len == 0) break :h pkg_hash;
957		const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
958		if (gop.found_existing) {
959		if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
960		gop.value_ptr.*.lazy_status = .eager;
961		promoted_existing_to_eager = true;
962		} else {
963		continue;
964		}
965		}
966		gop.value_ptr.* = new_fetch;
967		break :h pkg_hash;
968		},
969		},
970		},
971		.path => \|rel_path\| l: {
972		// This might produce an invalid path, which is checked for
973		// at the beginning of run().
974		const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
975		const pkg_hash = relativePathDigest(new_root, cache_root);
976		const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
977		if (gop.found_existing) {
978		if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
979		gop.value_ptr.*.lazy_status = .eager;
980		promoted_existing_to_eager = true;
981		} else {
982		continue;
983		}
984		}
985		gop.value_ptr.* = new_fetch;
986		break :l .{ .relative_path = new_root };
987		},
988		};
989		prog_names[new_fetch_index] = dep_name;
990		new_fetch_index += 1;
991		if (!promoted_existing_to_eager) {
992		f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
993		}
994		new_fetch.* = .{
995		.arena = std.heap.ArenaAllocator.init(gpa),
996		.location = location,
997		.location_tok = dep.location_tok,
998		.hash_tok = dep.hash_tok,
999		.name_tok = dep.name_tok,
1000		.lazy_status = switch (f.job_queue.mode) {
1001		.needed => if (dep.lazy) .available else .eager,
1002		.all => .eager,
1003		},
1004		.parent_package_root = f.package_root,
1005		.remote_package_root = f.remote_package_root,
1006		.parent_manifest_ast = &f.manifest_ast,
1007		.prog_node = f.prog_node,
1008		.job_queue = f.job_queue,
1009		.omit_missing_hash_error = false,
1010		.allow_missing_paths_field = true,
1011		.use_latest_commit = false,
1012
1013		.package_root = undefined,
1014		.error_bundle = undefined,
1015		.manifest = undefined,
1016		.manifest_ast = undefined,
1017		.have_manifest = false,
1018		.computed_hash = undefined,
1019		.has_build_zig = false,
1020		.oom_flag = false,
1021		.latest_commit = null,
1022
1023		.module = null,
1024		};
1025		}
1026
1027		f.prog_node.increaseEstimatedTotalItems(new_fetch_index);
1028
1029		break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
1030		};
1031
1032		// Now it's time to dispatch tasks.
1033		for (new_fetches, prog_names) \|*new_fetch, prog_name\| {
1034		f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name });
1035		}
1036		}
1037
1038		pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
1039		return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
1040		}
1041
1042		pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void {
1043		const prog_node = f.prog_node.start(prog_name, 0);
1044		defer prog_node.end();
1045
1046		run(f) catch \|err\| switch (err) {
1047		error.OutOfMemory => f.oom_flag = true,
1048		error.Canceled => \|e\| return e,
1049		error.FetchFailed => {
1050		// Nothing to do because the errors are already reported in `error_bundle`,
1051		// and a reference is kept to the `Fetch` task inside `all_fetches`.
1052		},
1053		};
1054		}
1055
1056		fn srcLoc(
1057		f: *Fetch,
1058		tok: std.zig.Ast.TokenIndex,
1059		) Allocator.Error!ErrorBundle.SourceLocationIndex {
1060		const ast = f.parent_manifest_ast orelse return .none;
1061		const eb = &f.error_bundle;
1062		const start_loc = ast.tokenLocation(0, tok);
1063		const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root});
1064		const msg_off = 0;
1065		return eb.addSourceLocation(.{
1066		.src_path = src_path,
1067		.span_start = ast.tokenStart(tok),
1068		.span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len),
1069		.span_main = ast.tokenStart(tok) + msg_off,
1070		.line = @intCast(start_loc.line),
1071		.column = @intCast(start_loc.column),
1072		.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
1073		});
1074		}
1075
1076		fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
1077		const eb = &f.error_bundle;
1078		try eb.addRootErrorMessage(.{
1079		.msg = msg_str,
1080		.src_loc = try f.srcLoc(msg_tok),
1081		});
1082		return error.FetchFailed;
1083		}
1084
1085		const Resource = union(enum) {
1086		file: Io.File.Reader,
1087		http_request: HttpRequest,
1088		git: Git,
1089		dir: Io.Dir,
1090
1091		const Git = struct {
1092		session: git.Session,
1093		fetch_stream: git.Session.FetchStream,
1094		want_oid: git.Oid,
1095		};
1096
1097		const HttpRequest = struct {
1098		request: std.http.Client.Request,
1099		response: std.http.Client.Response,
1100		transfer_buffer: []u8,
1101		decompress: std.http.Decompress,
1102		decompress_buffer: []u8,
1103		};
1104
1105		fn deinit(resource: *Resource, io: Io) void {
1106		switch (resource.*) {
1107		.file => \|*file_reader\| file_reader.file.close(io),
1108		.http_request => \|*http_request\| http_request.request.deinit(),
1109		.git => \|*git_resource\| {
1110		git_resource.fetch_stream.deinit();
1111		},
1112		.dir => \|*dir\| dir.close(io),
1113		}
1114		resource.* = undefined;
1115		}
1116
1117		fn reader(resource: Resource) Io.Reader {
1118		return switch (resource.*) {
1119		.file => \|*file_reader\| return &file_reader.interface,
1120		.http_request => \|*http_request\| return http_request.response.readerDecompressing(
1121		http_request.transfer_buffer,
1122		&http_request.decompress,
1123		http_request.decompress_buffer,
1124		),
1125		.git => \|*g\| return &g.fetch_stream.reader,
1126		.dir => unreachable,
1127		};
1128		}
1129		};
1130
1131		const FileType = enum {
1132		tar,
1133		@"tar.gz",
1134		@"tar.xz",
1135		@"tar.zst",
1136		git_pack,
1137		zip,
1138
1139		fn fromPath(file_path: []const u8) ?FileType {
1140		if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
1141		if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz";
1142		if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
1143		if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz";
1144		if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
1145		if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";
1146		if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";
1147		if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip;
1148		if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip;
1149		return null;
1150		}
1151
1152		/// Parameter is a content-disposition header value.
1153		fn fromContentDisposition(cd_header: []const u8) ?FileType {
1154		const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse
1155		return null;
1156
1157		var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse
1158		return null;
1159		value_start += "filename".len;
1160		if (cd_header[value_start] == '*') {
1161		value_start += 1;
1162		}
1163		if (cd_header[value_start] != '=') return null;
1164		value_start += 1;
1165
1166		var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len;
1167		if (cd_header[value_end - 1] == '\"') {
1168		value_end -= 1;
1169		}
1170		return fromPath(cd_header[value_start..value_end]);
1171		}
1172
1173		test fromContentDisposition {
1174		try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
1175		try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\""));
1176		try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
1177		try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
1178		try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
1179		try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
1180
1181		try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
1182		try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
1183		try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
1184		try std.testing.expect(fromContentDisposition("inline; size=42") == null);
1185		try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null);
1186		try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null);
1187		}
1188		};
1189
1190		const init_resource_buffer_size = git.Packet.max_data_length;
1191
1192		fn initResource(f: Fetch, uri: std.Uri, resource: Resource, reader_buffer: []u8) RunError!void {
1193		const io = f.job_queue.io;
1194		const arena = f.arena.allocator();
1195		const eb = &f.error_bundle;
1196
1197		if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
1198		const path = try uri.path.toRawMaybeAlloc(arena);
1199		const file = f.parent_package_root.openFile(io, path, .{}) catch \|err\| {
1200		return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{
1201		f.parent_package_root, path, err,
1202		}));
1203		};
1204		resource.* = .{ .file = file.reader(io, reader_buffer) };
1205		return;
1206		}
1207
1208		const http_client = f.job_queue.http_client;
1209
1210		if (ascii.eqlIgnoreCase(uri.scheme, "http") or
1211		ascii.eqlIgnoreCase(uri.scheme, "https"))
1212		{
1213		resource.* = .{ .http_request = .{
1214		.request = http_client.request(.GET, uri, .{}) catch \|err\|
1215		return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})),
1216		.response = undefined,
1217		.transfer_buffer = reader_buffer,
1218		.decompress_buffer = &.{},
1219		.decompress = undefined,
1220		} };
1221		const request = &resource.http_request.request;
1222		errdefer request.deinit();
1223
1224		request.sendBodiless() catch \|err\|
1225		return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
1226
1227		var redirect_buffer: [8000]u8 = undefined;
1228		const response = &resource.http_request.response;
1229		response.* = request.receiveHead(&redirect_buffer) catch \|err\| switch (err) {
1230		error.ReadFailed => {
1231		return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
1232		request.connection.?.getReadError().?,
1233		}));
1234		},
1235		else => \|e\| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
1236		};
1237
1238		if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
1239		"bad HTTP response code: '{d} {s}'",
1240		.{ response.head.status, response.head.status.phrase() orelse "" },
1241		));
1242
1243		resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
1244		return;
1245		}
1246
1247		if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
1248		ascii.eqlIgnoreCase(uri.scheme, "git+https"))
1249		{
1250		var transport_uri = uri;
1251		transport_uri.scheme = uri.scheme["git+".len..];
1252		var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch \|err\| {
1253		return f.fail(
1254		f.location_tok,
1255		try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
1256		);
1257		};
1258
1259		const want_oid = want_oid: {
1260		const want_ref =
1261		if (uri.fragment) \|fragment\| try fragment.toRawMaybeAlloc(arena) else "HEAD";
1262		if (git.Oid.parseAny(want_ref)) \|oid\| break :want_oid oid else \|_\| {}
1263
1264		const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
1265		const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
1266
1267		var ref_iterator: git.Session.RefIterator = undefined;
1268		session.listRefs(&ref_iterator, .{
1269		.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
1270		.include_peeled = true,
1271		.buffer = reader_buffer,
1272		}) catch \|err\| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err}));
1273		defer ref_iterator.deinit();
1274		while (ref_iterator.next() catch \|err\| {
1275		return f.fail(f.location_tok, try eb.printString(
1276		"unable to iterate refs: {s}",
1277		.{@errorName(err)},
1278		));
1279		}) \|ref\| {
1280		if (std.mem.eql(u8, ref.name, want_ref) or
1281		std.mem.eql(u8, ref.name, want_ref_head) or
1282		std.mem.eql(u8, ref.name, want_ref_tag))
1283		{
1284		break :want_oid ref.peeled orelse ref.oid;
1285		}
1286		}
1287		return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
1288		};
1289		if (f.use_latest_commit) {
1290		f.latest_commit = want_oid;
1291		} else if (uri.fragment == null) {
1292		const notes_len = 1;
1293		try eb.addRootErrorMessage(.{
1294		.msg = try eb.addString("url field is missing an explicit ref"),
1295		.src_loc = try f.srcLoc(f.location_tok),
1296		.notes_len = notes_len,
1297		});
1298		const notes_start = try eb.reserveNotes(notes_len);
1299		eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1300		.msg = try eb.printString("try .url = \"{f}#{f}\",", .{
1301		uri.fmt(.{ .scheme = true, .authority = true, .path = true }),
1302		want_oid,
1303		}),
1304		}));
1305		return error.FetchFailed;
1306		}
1307
1308		var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;
1309		_ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable;
1310		resource.* = .{ .git = .{
1311		.session = session,
1312		.fetch_stream = undefined,
1313		.want_oid = want_oid,
1314		} };
1315		const fetch_stream = &resource.git.fetch_stream;
1316		session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch \|err\| {
1317		return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
1318		};
1319		errdefer fetch_stream.deinit(fetch_stream);
1320
1321		return;
1322		}
1323
1324		return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme}));
1325		}
1326
1327		fn unpackResource(
1328		f: *Fetch,
1329		resource: *Resource,
1330		uri_path: []const u8,
1331		tmp_directory: Cache.Directory,
1332		) RunError!UnpackResult {
1333		const eb = &f.error_bundle;
1334		const file_type = switch (resource.*) {
1335		.file => FileType.fromPath(uri_path) orelse
1336		return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
1337
1338		.http_request => \|*http_request\| ft: {
1339		const head = &http_request.response.head;
1340
1341		// Content-Type takes first precedence.
1342		const content_type = head.content_type orelse
1343		return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
1344
1345		// Extract the MIME type, ignoring charset and boundary directives
1346		const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len;
1347		const mime_type = content_type[0..mime_type_end];
1348
1349		if (ascii.eqlIgnoreCase(mime_type, "application/x-tar"))
1350		break :ft .tar;
1351
1352		if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or
1353		ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or
1354		ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or
1355		ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or
1356		ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed"))
1357		{
1358		break :ft .@"tar.gz";
1359		}
1360
1361		if (ascii.eqlIgnoreCase(mime_type, "application/x-xz"))
1362		break :ft .@"tar.xz";
1363
1364		if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))
1365		break :ft .@"tar.zst";
1366
1367		if (ascii.eqlIgnoreCase(mime_type, "application/zip") or
1368		ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or
1369		ascii.eqlIgnoreCase(mime_type, "application/java-archive"))
1370		{
1371		break :ft .zip;
1372		}
1373
1374		if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and
1375		!ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))
1376		{
1377		return f.fail(f.location_tok, try eb.printString(
1378		"unrecognized 'Content-Type' header: '{s}'",
1379		.{content_type},
1380		));
1381		}
1382
1383		// Next, the filename from 'content-disposition: attachment' takes precedence.
1384		if (head.content_disposition) \|cd_header\| {
1385		break :ft FileType.fromContentDisposition(cd_header) orelse {
1386		return f.fail(f.location_tok, try eb.printString(
1387		"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
1388		.{cd_header},
1389		));
1390		};
1391		}
1392
1393		// Finally, the path from the URI is used.
1394		break :ft FileType.fromPath(uri_path) orelse {
1395		return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path}));
1396		};
1397		},
1398
1399		.git => .git_pack,
1400
1401		.dir => \|dir\| {
1402		f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
1403		return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{
1404		uri_path, err,
1405		}));
1406		};
1407		return .{};
1408		},
1409		};
1410
1411		switch (file_type) {
1412		.tar => {
1413		return unpackTarball(f, tmp_directory.handle, resource.reader());
1414		},
1415		.@"tar.gz" => {
1416		var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1417		var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer);
1418		return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1419		},
1420		.@"tar.xz" => {
1421		const gpa = f.arena.child_allocator;
1422		var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch \|err\|
1423		return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
1424		defer decompress.deinit();
1425		return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1426		},
1427		.@"tar.zst" => {
1428		const window_len = std.compress.zstd.default_window_len;
1429		const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max);
1430		var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
1431		.verify_checksum = false,
1432		.window_len = window_len,
1433		});
1434		return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1435		},
1436		.git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch \|err\| switch (err) {
1437		error.FetchFailed, error.OutOfMemory => \|e\| return e,
1438		else => \|e\| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})),
1439		},
1440		.zip => return unzip(f, tmp_directory.handle, resource.reader()) catch \|err\| switch (err) {
1441		error.ReadFailed => return f.fail(f.location_tok, try eb.printString(
1442		"failed reading resource: {t}",
1443		.{err},
1444		)),
1445		else => \|e\| return e,
1446		},
1447		}
1448		}
1449
1450		fn unpackTarball(f: Fetch, out_dir: Io.Dir, reader: Io.Reader) RunError!UnpackResult {
1451		const eb = &f.error_bundle;
1452		const arena = f.arena.allocator();
1453		const io = f.job_queue.io;
1454
1455		var diagnostics: std.tar.Diagnostics = .{ .allocator = arena };
1456
1457		std.tar.pipeToFileSystem(io, out_dir, reader, .{
1458		.diagnostics = &diagnostics,
1459		.strip_components = 0,
1460		.mode_mode = .ignore,
1461		.exclude_empty_directories = true,
1462		}) catch \|err\| return f.fail(
1463		f.location_tok,
1464		try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
1465		);
1466
1467		var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
1468		if (diagnostics.errors.items.len > 0) {
1469		try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball");
1470		for (diagnostics.errors.items) \|item\| {
1471		switch (item) {
1472		.unable_to_create_file => \|i\| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code),
1473		.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code),
1474		.unsupported_file_type => \|i\| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)),
1475		.components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0
1476		}
1477		}
1478		}
1479		return res;
1480		}
1481
1482		fn unzip(
1483		f: *Fetch,
1484		out_dir: Io.Dir,
1485		reader: *Io.Reader,
1486		) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult {
1487		// We write the entire contents to a file first because zip files
1488		// must be processed back to front and they could be too large to
1489		// load into memory.
1490
1491		const io = f.job_queue.io;
1492		const cache_root = f.job_queue.global_cache;
1493		const prefix = "tmp/";
1494		const suffix = ".zip";
1495		const eb = &f.error_bundle;
1496		const random_len = @sizeOf(u64) * 2;
1497
1498		var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined;
1499		zip_path[0..prefix.len].* = prefix.*;
1500		zip_path[prefix.len + random_len ..].* = suffix.*;
1501
1502		var zip_file = while (true) {
1503		const random_integer = r: {
1504		var x: u64 = undefined;
1505		io.random(@ptrCast(&x));
1506		break :r x;
1507		};
1508		zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer);
1509
1510		break cache_root.handle.createFile(io, &zip_path, .{
1511		.exclusive = true,
1512		.read = true,
1513		}) catch \|err\| switch (err) {
1514		error.PathAlreadyExists => continue,
1515		error.FileNotFound => {
1516		cache_root.handle.createDir(io, prefix, .default_dir) catch \|dir_err\| switch (dir_err) {
1517		error.Canceled => \|e\| return e,
1518		// error.PathAlreadyExists is considered a failure here because
1519		// it implies that the prefix is not a directory.
1520		else => \|e\| return f.fail(
1521		f.location_tok,
1522		try eb.printString("failed to create temporary directory: {t}", .{e}),
1523		),
1524		};
1525		continue;
1526		},
1527		error.Canceled => \|e\| return e,
1528		else => \|e\| return f.fail(
1529		f.location_tok,
1530		try eb.printString("failed to create temporary zip file: {t}", .{e}),
1531		),
1532		};
1533		};
1534		defer zip_file.close(io);
1535		var zip_file_buffer: [4096]u8 = undefined;
1536		var zip_file_reader = b: {
1537		var zip_file_writer = zip_file.writer(io, &zip_file_buffer);
1538
1539		_ = reader.streamRemaining(&zip_file_writer.interface) catch \|err\| switch (err) {
1540		error.ReadFailed => \|e\| return e,
1541		error.WriteFailed => return f.fail(
1542		f.location_tok,
1543		try eb.printString("failed writing temporary zip file: {t}", .{err}),
1544		),
1545		};
1546		zip_file_writer.interface.flush() catch \|err\| return f.fail(
1547		f.location_tok,
1548		try eb.printString("failed writing temporary zip file: {t}", .{err}),
1549		);
1550		break :b zip_file_writer.moveToReader();
1551		};
1552
1553		var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
1554		// no need to deinit since we are using an arena allocator
1555
1556		zip_file_reader.seekTo(0) catch \|err\|
1557		return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err}));
1558		std.zip.extract(out_dir, &zip_file_reader, .{
1559		.allow_backslashes = true,
1560		.diagnostics = &diagnostics,
1561		}) catch \|err\| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err}));
1562
1563		cache_root.handle.deleteFile(io, &zip_path) catch \|err\|
1564		return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err}));
1565
1566		return .{ .root_dir = diagnostics.root_dir };
1567		}
1568
1569		fn unpackGitPack(f: Fetch, out_dir: Io.Dir, resource: Resource.Git) anyerror!UnpackResult {
1570		const io = f.job_queue.io;
1571		const arena = f.arena.allocator();
1572		// TODO don't try to get a gpa from an arena. expose this dependency higher up
1573		// because the backing of arena could be page allocator
1574		const gpa = f.arena.child_allocator;
1575		const object_format: git.Oid.Format = resource.want_oid;
1576
1577		var res: UnpackResult = .{};
1578		// The .git directory is used to store the packfile and associated index, but
1579		// we do not attempt to replicate the exact structure of a real .git
1580		// directory, since that isn't relevant for fetching a package.
1581		{
1582		var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{});
1583		defer pack_dir.close(io);
1584		var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true });
1585		defer pack_file.close(io);
1586		var pack_file_buffer: [4096]u8 = undefined;
1587		var pack_file_reader = b: {
1588		var pack_file_writer = pack_file.writer(io, &pack_file_buffer);
1589		const fetch_reader = &resource.fetch_stream.reader;
1590		_ = try fetch_reader.streamRemaining(&pack_file_writer.interface);
1591		try pack_file_writer.interface.flush();
1592		break :b pack_file_writer.moveToReader();
1593		};
1594
1595		var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true });
1596		defer index_file.close(io);
1597		var index_file_buffer: [2000]u8 = undefined;
1598		var index_file_writer = index_file.writer(io, &index_file_buffer);
1599		{
1600		const index_prog_node = f.prog_node.start("Index pack", 0);
1601		defer index_prog_node.end();
1602		try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
1603		}
1604
1605		{
1606		var index_file_reader = index_file.reader(io, &index_file_buffer);
1607		const checkout_prog_node = f.prog_node.start("Checkout", 0);
1608		defer checkout_prog_node.end();
1609		var repository: git.Repository = undefined;
1610		try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
1611		defer repository.deinit();
1612		var diagnostics: git.Diagnostics = .{ .allocator = arena };
1613		try repository.checkout(io, out_dir, resource.want_oid, &diagnostics);
1614
1615		if (diagnostics.errors.items.len > 0) {
1616		try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");
1617		for (diagnostics.errors.items) \|item\| {
1618		switch (item) {
1619		.unable_to_create_file => \|i\| res.unableToCreateFile(i.file_name, i.code),
1620		.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(i.file_name, i.link_name, i.code),
1621		}
1622		}
1623		}
1624		}
1625		}
1626
1627		try out_dir.deleteTree(io, ".git");
1628		return res;
1629		}
1630
1631		fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void {
1632		const gpa = f.arena.child_allocator;
1633		const io = f.job_queue.io;
1634		// Recursive directory copy.
1635		var it = try dir.walk(gpa);
1636		defer it.deinit();
1637		while (try it.next(io)) \|entry\| {
1638		switch (entry.kind) {
1639		.directory => {}, // omit empty directories
1640		.file => {
1641		dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch \|err\| switch (err) {
1642		error.FileNotFound => {
1643		if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
1644		try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{});
1645		},
1646		else => \|e\| return e,
1647		};
1648		},
1649		.sym_link => {
1650		var buf: [fs.max_path_bytes]u8 = undefined;
1651		const link_name = buf[0..try dir.readLink(io, entry.path, &buf)];
1652		// TODO: if this would create a symlink to outside
1653		// the destination directory, fail with an error instead.
1654		tmp_dir.symLink(io, link_name, entry.path, .{}) catch \|err\| switch (err) {
1655		error.FileNotFound => {
1656		if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
1657		try tmp_dir.symLink(io, link_name, entry.path, .{});
1658		},
1659		else => \|e\| return e,
1660		};
1661		},
1662		else => return error.IllegalFileTypeInPackage,
1663		}
1664		}
1665		}
1666
1667		pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void {
1668		var handled_missing_dir = false;
1669		while (true) {
1670		Io.Dir.rename(
1671		tmp_path.root_dir.handle,
1672		tmp_path.sub_path,
1673		dest_path.root_dir.handle,
1674		dest_path.sub_path,
1675		io,
1676		) catch \|err\| switch (err) {
1677		error.FileNotFound => {
1678		if (handled_missing_dir) return err;
1679		const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
1680		dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch \|er\| switch (er) {
1681		error.PathAlreadyExists => handled_missing_dir = true,
1682		else => \|e\| return e,
1683		};
1684		continue;
1685		},
1686		error.DirNotEmpty, error.AccessDenied => {
1687		// Package has been already downloaded and may already be in use on the system.
1688		tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch \|er\| switch (er) {
1689		error.Canceled => \|e\| return e,
1690		// Garbage files leftover in zig-cache/tmp/ is, as they say
1691		// on Star Trek, "operating within normal parameters".
1692		else => \|e\| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
1693		};
1694		},
1695		else => \|e\| return e,
1696		};
1697		break;
1698		}
1699		}
1700
1701		const ComputedHash = struct {
1702		digest: Package.Hash.Digest,
1703		total_size: u64,
1704		};
1705
1706		/// Assumes that files not included in the package have already been filtered
1707		/// prior to calling this function. This ensures that files not protected by
1708		/// the hash are not present on the file system. Empty directories are *not
1709		/// hashed* and must not be present on the file system when calling this
1710		/// function.
1711		fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1712		const io = f.job_queue.io;
1713		// All the path name strings need to be in memory for sorting.
1714		const arena = f.arena.allocator();
1715		const gpa = f.arena.child_allocator;
1716		const eb = &f.error_bundle;
1717		const root_dir = pkg_path.root_dir.handle;
1718
1719		// Collect all files, recursively, then sort.
1720		var all_files = std.array_list.Managed(*HashedFile).init(gpa);
1721		defer all_files.deinit();
1722
1723		var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa);
1724		defer deleted_files.deinit();
1725
1726		// Track directories which had any files deleted from them so that empty directories
1727		// can be deleted.
1728		var sus_dirs: std.array_hash_map.String(void) = .empty;
1729		defer sus_dirs.deinit(gpa);
1730
1731		var walker = try root_dir.walk(gpa);
1732		defer walker.deinit();
1733
1734		// Total number of bytes of file contents included in the package.
1735		var total_size: u64 = 0;
1736
1737		{
1738		// The final hash will be a hash of each file hashed independently. This
1739		// allows hashing in parallel.
1740		var group: Io.Group = .init;
1741		defer group.cancel(io);
1742
1743		while (walker.next(io) catch \|err\| {
1744		try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1745		"unable to walk temporary directory '{f}': {t}",
1746		.{ pkg_path, err },
1747		) });
1748		return error.FetchFailed;
1749		}) \|entry\| {
1750		if (entry.kind == .directory) continue;
1751
1752		const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
1753		if (!filter.includePath(entry_pkg_path)) {
1754		// Delete instead of including in hash calculation.
1755		const fs_path = try arena.dupe(u8, entry.path);
1756
1757		// Also track the parent directory in case it becomes empty.
1758		if (fs.path.dirname(fs_path)) \|parent\|
1759		try sus_dirs.put(gpa, parent, {});
1760
1761		const deleted_file = try arena.create(DeletedFile);
1762		deleted_file.* = .{
1763		.fs_path = fs_path,
1764		.failure = undefined, // to be populated by the worker
1765		};
1766		group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file });
1767		try deleted_files.append(deleted_file);
1768		continue;
1769		}
1770
1771		const kind: HashedFile.Kind = switch (entry.kind) {
1772		.directory => unreachable,
1773		.file => .file,
1774		.sym_link => .link,
1775		else => return f.fail(f.location_tok, try eb.printString(
1776		"package contains '{s}' which has illegal file type '{t}'",
1777		.{ entry.path, entry.kind },
1778		)),
1779		};
1780
1781		if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename))
1782		f.has_build_zig = true;
1783
1784		const fs_path = try arena.dupe(u8, entry.path);
1785		const hashed_file = try arena.create(HashedFile);
1786		hashed_file.* = .{
1787		.fs_path = fs_path,
1788		.normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
1789		.kind = kind,
1790		.hash = undefined, // to be populated by the worker
1791		.failure = undefined, // to be populated by the worker
1792		.size = undefined, // to be populated by the worker
1793		};
1794		group.async(io, workerHashFile, .{ io, root_dir, hashed_file });
1795		try all_files.append(hashed_file);
1796		}
1797
1798		try group.await(io);
1799		}
1800
1801		{
1802		// Sort by length, descending, so that child directories get removed first.
1803		sus_dirs.sortUnstable(@as(struct {
1804		keys: []const []const u8,
1805		pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
1806		return ctx.keys[b_index].len < ctx.keys[a_index].len;
1807		}
1808		}, .{ .keys = sus_dirs.keys() }));
1809
1810		// During this loop, more entries will be added, so we must loop by index.
1811		var i: usize = 0;
1812		while (i < sus_dirs.count()) : (i += 1) {
1813		const sus_dir = sus_dirs.keys()[i];
1814		root_dir.deleteDir(io, sus_dir) catch \|err\| switch (err) {
1815		error.DirNotEmpty => continue,
1816		error.FileNotFound => continue,
1817		else => \|e\| {
1818		try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1819		"unable to delete empty directory '{s}': {s}",
1820		.{ sus_dir, @errorName(e) },
1821		) });
1822		return error.FetchFailed;
1823		},
1824		};
1825		if (fs.path.dirname(sus_dir)) \|parent\| {
1826		try sus_dirs.put(gpa, parent, {});
1827		}
1828		}
1829		}
1830
1831		std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1832
1833		var hasher = Package.Hash.Algo.init(.{});
1834		var any_failures = false;
1835		for (all_files.items) \|hashed_file\| {
1836		hashed_file.failure catch \|err\| {
1837		any_failures = true;
1838		try eb.addRootErrorMessage(.{
1839		.msg = try eb.printString("unable to hash '{s}': {s}", .{
1840		hashed_file.fs_path, @errorName(err),
1841		}),
1842		});
1843		};
1844		hasher.update(&hashed_file.hash);
1845		total_size += hashed_file.size;
1846		}
1847		for (deleted_files.items) \|deleted_file\| {
1848		deleted_file.failure catch \|err\| {
1849		any_failures = true;
1850		try eb.addRootErrorMessage(.{
1851		.msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
1852		deleted_file.fs_path, @errorName(err),
1853		}),
1854		});
1855		};
1856		}
1857
1858		if (any_failures) return error.FetchFailed;
1859
1860		if (f.job_queue.debug_hash) {
1861		assert(!f.job_queue.recursive);
1862		// Print something to stdout that can be text diffed to figure out why
1863		// the package hash is different.
1864		dumpHashInfo(io, all_files.items) catch \|err\|
1865		std.process.fatal("unable to write to stdout: {t}", .{err});
1866		}
1867
1868		return .{
1869		.digest = hasher.finalResult(),
1870		.total_size = total_size,
1871		};
1872		}
1873
1874		fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void {
1875		var stdout_buffer: [1024]u8 = undefined;
1876		var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer);
1877		dumpHashInfoWriter(&stdout_writer.interface, all_files) catch \|err\| switch (err) {
1878		error.WriteFailed => return stdout_writer.err.?,
1879		};
1880		try stdout_writer.flush();
1881		}
1882
1883		fn dumpHashInfoWriter(w: Io.Writer, all_files: []const const HashedFile) Io.Writer.Error!void {
1884		for (all_files) \|hashed_file\| {
1885		try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path });
1886		}
1887		}
1888
1889		fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void {
1890		hashed_file.failure = hashFileFallible(io, dir, hashed_file);
1891		}
1892
1893		fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void {
1894		deleted_file.failure = deleteFileFallible(io, dir, deleted_file);
1895		}
1896
1897		fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1898		var buf: [8000]u8 = undefined;
1899		var hasher = Package.Hash.Algo.init(.{});
1900		hasher.update(hashed_file.normalized_path);
1901		var file_size: u64 = 0;
1902
1903		switch (hashed_file.kind) {
1904		.file => {
1905		var file = try dir.openFile(io, hashed_file.fs_path, .{});
1906		defer file.close(io);
1907		// Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
1908		hasher.update(&.{ 0, 0 });
1909		var file_header: FileHeader = .{};
1910		while (true) {
1911		const bytes_read = try file.readPositional(io, &.{&buf}, file_size);
1912		if (bytes_read == 0) break;
1913		file_size += bytes_read;
1914		hasher.update(buf[0..bytes_read]);
1915		file_header.update(buf[0..bytes_read]);
1916		}
1917		if (file_header.isExecutable()) {
1918		try setExecutable(io, file);
1919		}
1920		},
1921		.link => {
1922		const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)];
1923		if (fs.path.sep != canonical_sep) {
1924		// Package hashes are intended to be consistent across
1925		// platforms which means we must normalize path separators
1926		// inside symlinks.
1927		normalizePath(link_name);
1928		}
1929		hasher.update(link_name);
1930		},
1931		}
1932		hasher.final(&hashed_file.hash);
1933		hashed_file.size = file_size;
1934		}
1935
1936		fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
1937		try dir.deleteFile(io, deleted_file.fs_path);
1938		}
1939
1940		fn setExecutable(io: Io, file: Io.File) !void {
1941		if (!Io.File.Permissions.has_executable_bit) return;
1942		try file.setPermissions(io, .executable_file);
1943		}
1944
1945		const DeletedFile = struct {
1946		fs_path: []const u8,
1947		failure: Error!void,
1948
1949		const Error =
1950		Io.Dir.DeleteFileError \|\|
1951		Io.Dir.DeleteDirError;
1952		};
1953
1954		const HashedFile = struct {
1955		fs_path: []const u8,
1956		normalized_path: []const u8,
1957		hash: Package.Hash.Digest,
1958		failure: Error!void,
1959		kind: Kind,
1960		size: u64,
1961
1962		const Error =
1963		Io.File.OpenError \|\|
1964		Io.File.ReadPositionalError \|\|
1965		Io.File.StatError \|\|
1966		Io.File.SetPermissionsError \|\|
1967		Io.Dir.ReadLinkError;
1968
1969		const Kind = enum { file, link };
1970
1971		fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
1972		_ = context;
1973		return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
1974		}
1975		};
1976
1977		/// Strips root directory name from file system path.
1978		fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
1979		if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
1980
1981		if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) {
1982		return fs_path[root_dir.len + 1 ..];
1983		}
1984
1985		return fs_path;
1986		}
1987
1988		/// Make a file system path identical independently of operating system path inconsistencies.
1989		/// This converts backslashes into forward slashes.
1990		fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
1991		const normalized = try arena.dupe(u8, pkg_path);
1992		if (fs.path.sep == canonical_sep) return normalized;
1993		normalizePath(normalized);
1994		return normalized;
1995		}
1996
1997		const canonical_sep = fs.path.sep_posix;
1998
1999		fn normalizePath(bytes: []u8) void {
2000		assert(fs.path.sep != canonical_sep);
2001		std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
2002		}
2003
2004		const Filter = struct {
2005		include_paths: std.array_hash_map.String(void) = .empty,
2006
2007		/// sub_path is relative to the package root.
2008		pub fn includePath(self: *const Filter, sub_path: []const u8) bool {
2009		if (self.include_paths.count() == 0) return true;
2010		if (self.include_paths.contains("")) return true;
2011		if (self.include_paths.contains(".")) return true;
2012		if (self.include_paths.contains(sub_path)) return true;
2013
2014		// Check if any included paths are parent directories of sub_path.
2015		var dirname = sub_path;
2016		while (std.fs.path.dirname(dirname)) \|next_dirname\| {
2017		if (self.include_paths.contains(next_dirname)) return true;
2018		dirname = next_dirname;
2019		}
2020
2021		return false;
2022		}
2023
2024		test includePath {
2025		const gpa = std.testing.allocator;
2026		var filter: Filter = .{};
2027		defer filter.include_paths.deinit(gpa);
2028
2029		try filter.include_paths.put(gpa, "src", {});
2030		try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c"));
2031		try std.testing.expect(!filter.includePath(".gitignore"));
2032		}
2033		};
2034
2035		pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
2036		if (dep.hash) \|h\| return .fromSlice(h);
2037
2038		switch (dep.location) {
2039		.url => return null,
2040		.path => \|rel_path\| {
2041		var buf: [fs.max_path_bytes]u8 = undefined;
2042		var fba = std.heap.FixedBufferAllocator.init(&buf);
2043		const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch
2044		return null;
2045		return relativePathDigest(new_root, cache_root);
2046		},
2047		}
2048		}
2049
2050		// Detects executable header: ELF or Macho-O magic header or shebang line.
2051		const FileHeader = struct {
2052		header: [4]u8 = undefined,
2053		bytes_read: usize = 0,
2054
2055		pub fn update(self: *FileHeader, buf: []const u8) void {
2056		if (self.bytes_read >= self.header.len) return;
2057		const n = @min(self.header.len - self.bytes_read, buf.len);
2058		@memcpy(self.header[self.bytes_read..][0..n], buf[0..n]);
2059		self.bytes_read += n;
2060		}
2061
2062		fn isScript(self: *FileHeader) bool {
2063		const shebang = "#!";
2064		return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang);
2065		}
2066
2067		fn isElf(self: *FileHeader) bool {
2068		const elf_magic = std.elf.MAGIC;
2069		return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic);
2070		}
2071
2072		fn isMachO(self: *FileHeader) bool {
2073		if (self.bytes_read < 4) return false;
2074		const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian());
2075		return magic_number == std.macho.MH_MAGIC or
2076		magic_number == std.macho.MH_MAGIC_64 or
2077		magic_number == std.macho.FAT_MAGIC or
2078		magic_number == std.macho.FAT_MAGIC_64 or
2079		magic_number == std.macho.MH_CIGAM or
2080		magic_number == std.macho.MH_CIGAM_64 or
2081		magic_number == std.macho.FAT_CIGAM or
2082		magic_number == std.macho.FAT_CIGAM_64;
2083		}
2084
2085		pub fn isExecutable(self: *FileHeader) bool {
2086		return self.isScript() or self.isElf() or self.isMachO();
2087		}
2088		};
2089
2090		test FileHeader {
2091		var h: FileHeader = .{};
2092		try std.testing.expect(!h.isExecutable());
2093
2094		const elf_magic = std.elf.MAGIC;
2095		h.update(elf_magic[0..2]);
2096		try std.testing.expect(!h.isExecutable());
2097		h.update(elf_magic[2..4]);
2098		try std.testing.expect(h.isExecutable());
2099
2100		h.update(elf_magic[2..4]);
2101		try std.testing.expect(h.isExecutable());
2102
2103		const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE };
2104		h.bytes_read = 0;
2105		h.update(&macho64_magic_bytes);
2106		try std.testing.expect(h.isExecutable());
2107
2108		const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF };
2109		h.bytes_read = 0;
2110		h.update(&macho64_cigam_bytes);
2111		try std.testing.expect(h.isExecutable());
2112		}
2113
2114		// Result of the `unpackResource` operation. Enables collecting errors from
2115		// tar/git diagnostic, filtering that errors by manifest inclusion rules and
2116		// emitting remaining errors to an `ErrorBundle`.
2117		const UnpackResult = struct {
2118		errors: []Error = undefined,
2119		errors_count: usize = 0,
2120		root_error_message: []const u8 = "",
2121
2122		// A non empty value means that the package contents are inside a
2123		// sub-directory indicated by the named path.
2124		root_dir: []const u8 = "",
2125
2126		const Error = union(enum) {
2127		unable_to_create_sym_link: struct {
2128		code: anyerror,
2129		file_name: []const u8,
2130		link_name: []const u8,
2131		},
2132		unable_to_create_file: struct {
2133		code: anyerror,
2134		file_name: []const u8,
2135		},
2136		unsupported_file_type: struct {
2137		file_name: []const u8,
2138		file_type: u8,
2139		},
2140
2141		fn excluded(self: Error, filter: Filter) bool {
2142		const file_name = switch (self) {
2143		.unable_to_create_file => \|info\| info.file_name,
2144		.unable_to_create_sym_link => \|info\| info.file_name,
2145		.unsupported_file_type => \|info\| info.file_name,
2146		};
2147		return !filter.includePath(file_name);
2148		}
2149		};
2150
2151		fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void {
2152		self.root_error_message = try arena.dupe(u8, root_error_message);
2153		self.errors = try arena.alloc(UnpackResult.Error, n);
2154		}
2155
2156		fn hasErrors(self: *UnpackResult) bool {
2157		return self.errors_count > 0;
2158		}
2159
2160		fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void {
2161		self.errors[self.errors_count] = .{ .unable_to_create_file = .{
2162		.code = err,
2163		.file_name = file_name,
2164		} };
2165		self.errors_count += 1;
2166		}
2167
2168		fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void {
2169		self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{
2170		.code = err,
2171		.file_name = file_name,
2172		.link_name = link_name,
2173		} };
2174		self.errors_count += 1;
2175		}
2176
2177		fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void {
2178		self.errors[self.errors_count] = .{ .unsupported_file_type = .{
2179		.file_name = file_name,
2180		.file_type = file_type,
2181		} };
2182		self.errors_count += 1;
2183		}
2184
2185		fn validate(self: UnpackResult, f: Fetch, filter: Filter) !void {
2186		if (self.errors_count == 0) return;
2187
2188		var unfiltered_errors: u32 = 0;
2189		for (self.errors) \|item\| {
2190		if (item.excluded(filter)) continue;
2191		unfiltered_errors += 1;
2192		}
2193		if (unfiltered_errors == 0) return;
2194
2195		// Emmit errors to an `ErrorBundle`.
2196		const eb = &f.error_bundle;
2197		try eb.addRootErrorMessage(.{
2198		.msg = try eb.addString(self.root_error_message),
2199		.src_loc = try f.srcLoc(f.location_tok),
2200		.notes_len = unfiltered_errors,
2201		});
2202		var note_i: u32 = try eb.reserveNotes(unfiltered_errors);
2203		for (self.errors) \|item\| {
2204		if (item.excluded(filter)) continue;
2205		switch (item) {
2206		.unable_to_create_sym_link => \|info\| {
2207		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
2208		.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
2209		info.file_name, info.link_name, @errorName(info.code),
2210		}),
2211		}));
2212		},
2213		.unable_to_create_file => \|info\| {
2214		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
2215		.msg = try eb.printString("unable to create file '{s}': {s}", .{
2216		info.file_name, @errorName(info.code),
2217		}),
2218		}));
2219		},
2220		.unsupported_file_type => \|info\| {
2221		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
2222		.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
2223		info.file_name, info.file_type,
2224		}),
2225		}));
2226		},
2227		}
2228		note_i += 1;
2229		}
2230
2231		return error.FetchFailed;
2232		}
2233
2234		test validate {
2235		const gpa = std.testing.allocator;
2236		var arena_instance = std.heap.ArenaAllocator.init(gpa);
2237		defer arena_instance.deinit();
2238		const arena = arena_instance.allocator();
2239
2240		// fill UnpackResult with errors
2241		var res: UnpackResult = .{};
2242		try res.allocErrors(arena, 4, "unable to unpack");
2243		try std.testing.expectEqual(0, res.errors_count);
2244		res.unableToCreateFile("dir1/file1", error.File1);
2245		res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError);
2246		res.unableToCreateFile("dir1/file3", error.File3);
2247		res.unsupportedFileType("dir2/file4", 'x');
2248		try std.testing.expectEqual(4, res.errors_count);
2249
2250		// create filter, includes dir2, excludes dir1
2251		var filter: Filter = .{};
2252		try filter.include_paths.put(arena, "dir2", {});
2253
2254		// init Fetch
2255		var fetch: Fetch = undefined;
2256		fetch.parent_manifest_ast = null;
2257		fetch.location_tok = 0;
2258		try fetch.error_bundle.init(gpa);
2259		defer fetch.error_bundle.deinit();
2260
2261		// validate errors with filter
2262		try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter));
2263
2264		// output errors to string
2265		var errors = try fetch.error_bundle.toOwnedBundle("");
2266		defer errors.deinit(gpa);
2267		var aw: Io.Writer.Allocating = .init(gpa);
2268		defer aw.deinit();
2269		try errors.renderToWriter(.{}, &aw.writer);
2270		try std.testing.expectEqualStrings(
2271		\\error: unable to unpack
2272		\\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError
2273		\\ note: file 'dir2/file4' has unsupported type 'x'
2274		\\
2275		, aw.written());
2276		}
2277		};
2278
2279		test {
2280		_ = Filter;
2281		_ = FileType;
2282		_ = UnpackResult;
2283		}

src/Package/Fetch/git.zig deleted-1750

...	...	@@ -1,1750 +0,0 @@
1		//! Git support for package fetching.
2		//!
3		//! This is not intended to support all features of Git: it is limited to the
4		//! basic functionality needed to clone a repository for the purpose of fetching
5		//! a package.
6
7		const std = @import("std");
8		const Io = std.Io;
9		const mem = std.mem;
10		const testing = std.testing;
11		const Allocator = mem.Allocator;
12		const Sha1 = std.crypto.hash.Sha1;
13		const Sha256 = std.crypto.hash.sha2.Sha256;
14		const assert = std.debug.assert;
15
16		/// The ID of a Git object.
17		pub const Oid = union(Format) {
18		sha1: [Sha1.digest_length]u8,
19		sha256: [Sha256.digest_length]u8,
20
21		pub const max_formatted_length = len: {
22		var max: usize = 0;
23		for (std.enums.values(Format)) \|f\| {
24		max = @max(max, f.formattedLength());
25		}
26		break :len max;
27		};
28
29		pub const Format = enum {
30		sha1,
31		sha256,
32
33		pub fn byteLength(f: Format) usize {
34		return switch (f) {
35		.sha1 => Sha1.digest_length,
36		.sha256 => Sha256.digest_length,
37		};
38		}
39
40		pub fn formattedLength(f: Format) usize {
41		return 2 * f.byteLength();
42		}
43		};
44
45		const Hasher = union(Format) {
46		sha1: Sha1,
47		sha256: Sha256,
48
49		fn init(oid_format: Format) Hasher {
50		return switch (oid_format) {
51		.sha1 => .{ .sha1 = Sha1.init(.{}) },
52		.sha256 => .{ .sha256 = Sha256.init(.{}) },
53		};
54		}
55
56		// Must be public for use from HashedReader and HashedWriter.
57		pub fn update(hasher: *Hasher, b: []const u8) void {
58		switch (hasher.*) {
59		inline else => \|*inner\| inner.update(b),
60		}
61		}
62
63		fn finalResult(hasher: *Hasher) Oid {
64		return switch (hasher.*) {
65		inline else => \|*inner, tag\| @unionInit(Oid, @tagName(tag), inner.finalResult()),
66		};
67		}
68		};
69
70		const Hashing = union(Format) {
71		sha1: Io.Writer.Hashing(Sha1),
72		sha256: Io.Writer.Hashing(Sha256),
73
74		fn init(oid_format: Format, buffer: []u8) Hashing {
75		return switch (oid_format) {
76		.sha1 => .{ .sha1 = .init(buffer) },
77		.sha256 => .{ .sha256 = .init(buffer) },
78		};
79		}
80
81		fn writer(h: @This()) Io.Writer {
82		return switch (h.*) {
83		inline else => \|*inner\| &inner.writer,
84		};
85		}
86
87		fn final(h: *@This()) Oid {
88		switch (h.*) {
89		inline else => \|*inner, tag\| {
90		inner.writer.flush() catch unreachable; // hashers cannot fail
91		return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult());
92		},
93		}
94		}
95		};
96
97		pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
98		assert(bytes.len == oid_format.byteLength());
99		return switch (oid_format) {
100		inline else => \|tag\| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*),
101		};
102		}
103
104		pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid {
105		return switch (oid_format) {
106		inline else => \|tag\| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*),
107		};
108		}
109
110		pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid {
111		switch (oid_format) {
112		inline else => \|tag\| {
113		if (s.len != tag.formattedLength()) return error.InvalidOid;
114		var bytes: [tag.byteLength()]u8 = undefined;
115		for (&bytes, 0..) \|*b, i\| {
116		b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
117		}
118		return @unionInit(Oid, @tagName(tag), bytes);
119		},
120		}
121		}
122
123		test parse {
124		try testing.expectEqualSlices(
125		u8,
126		&.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
127		&(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1,
128		);
129		try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588"));
130		try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"));
131		try testing.expectEqualSlices(
132		u8,
133		&.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A },
134		&(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256,
135		);
136		try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf"));
137		try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf"));
138		try testing.expectError(error.InvalidOid, parse(.sha1, "master"));
139		try testing.expectError(error.InvalidOid, parse(.sha256, "master"));
140		try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD"));
141		try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD"));
142		}
143
144		pub fn parseAny(s: []const u8) error{InvalidOid}!Oid {
145		return for (std.enums.values(Format)) \|f\| {
146		if (s.len == f.formattedLength()) break parse(f, s);
147		} else error.InvalidOid;
148		}
149
150		pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void {
151		try writer.print("{x}", .{oid.slice()});
152		}
153
154		pub fn slice(oid: *const Oid) []const u8 {
155		return switch (oid.*) {
156		inline else => \|*bytes\| bytes,
157		};
158		}
159		};
160
161		pub const Diagnostics = struct {
162		allocator: Allocator,
163		errors: std.ArrayList(Error) = .empty,
164
165		pub const Error = union(enum) {
166		unable_to_create_sym_link: struct {
167		code: anyerror,
168		file_name: []const u8,
169		link_name: []const u8,
170		},
171		unable_to_create_file: struct {
172		code: anyerror,
173		file_name: []const u8,
174		},
175		};
176
177		pub fn deinit(d: *Diagnostics) void {
178		for (d.errors.items) \|item\| {
179		switch (item) {
180		.unable_to_create_sym_link => \|info\| {
181		d.allocator.free(info.file_name);
182		d.allocator.free(info.link_name);
183		},
184		.unable_to_create_file => \|info\| {
185		d.allocator.free(info.file_name);
186		},
187		}
188		}
189		d.errors.deinit(d.allocator);
190		d.* = undefined;
191		}
192		};
193
194		pub const Repository = struct {
195		odb: Odb,
196
197		pub fn init(
198		repo: *Repository,
199		allocator: Allocator,
200		format: Oid.Format,
201		pack_file: *Io.File.Reader,
202		index_file: *Io.File.Reader,
203		) !void {
204		repo.* = .{ .odb = undefined };
205		try repo.odb.init(allocator, format, pack_file, index_file);
206		}
207
208		pub fn deinit(repository: *Repository) void {
209		repository.odb.deinit();
210		repository.* = undefined;
211		}
212
213		/// Checks out the repository at `commit_oid` to `worktree`.
214		pub fn checkout(
215		repository: *Repository,
216		io: Io,
217		worktree: Io.Dir,
218		commit_oid: Oid,
219		diagnostics: *Diagnostics,
220		) !void {
221		try repository.odb.seekOid(commit_oid);
222		const tree_oid = tree_oid: {
223		const commit_object = try repository.odb.readObject();
224		if (commit_object.type != .commit) return error.NotACommit;
225		break :tree_oid try getCommitTree(repository.odb.format, commit_object.data);
226		};
227		try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics);
228		}
229
230		/// Checks out the tree at `tree_oid` to `worktree`.
231		fn checkoutTree(
232		repository: *Repository,
233		io: Io,
234		dir: Io.Dir,
235		tree_oid: Oid,
236		current_path: []const u8,
237		diagnostics: *Diagnostics,
238		) !void {
239		try repository.odb.seekOid(tree_oid);
240		const tree_object = try repository.odb.readObject();
241		if (tree_object.type != .tree) return error.NotATree;
242		// The tree object may be evicted from the object cache while we're
243		// iterating over it, so we can make a defensive copy here to make sure
244		// it remains valid until we're done with it
245		const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
246		defer repository.odb.allocator.free(tree_data);
247
248		var tree_iter: TreeIterator = .{
249		.format = repository.odb.format,
250		.data = tree_data,
251		.pos = 0,
252		};
253		while (try tree_iter.next()) \|entry\| {
254		switch (entry.type) {
255		.directory => {
256		try dir.createDir(io, entry.name, .default_dir);
257		var subdir = try dir.openDir(io, entry.name, .{});
258		defer subdir.close(io);
259		const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
260		defer repository.odb.allocator.free(sub_path);
261		try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics);
262		},
263		.file => {
264		try repository.odb.seekOid(entry.oid);
265		const file_object = try repository.odb.readObject();
266		if (file_object.type != .blob) return error.InvalidFile;
267		var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch \|e\| {
268		const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
269		errdefer diagnostics.allocator.free(file_name);
270		try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{
271		.code = e,
272		.file_name = file_name,
273		} });
274		continue;
275		};
276		defer file.close(io);
277		try file.writePositionalAll(io, file_object.data, 0);
278		},
279		.symlink => {
280		try repository.odb.seekOid(entry.oid);
281		const symlink_object = try repository.odb.readObject();
282		if (symlink_object.type != .blob) return error.InvalidFile;
283		const link_name = symlink_object.data;
284		dir.symLink(io, link_name, entry.name, .{}) catch \|e\| {
285		const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
286		errdefer diagnostics.allocator.free(file_name);
287		const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
288		errdefer diagnostics.allocator.free(link_name_dup);
289		try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
290		.code = e,
291		.file_name = file_name,
292		.link_name = link_name_dup,
293		} });
294		};
295		},
296		.gitlink => {
297		// Consistent with git archive behavior, create the directory but
298		// do nothing else
299		try dir.createDir(io, entry.name, .default_dir);
300		},
301		}
302		}
303		}
304
305		/// Returns the ID of the tree associated with the given commit (provided as
306		/// raw object data).
307		fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid {
308		if (!mem.startsWith(u8, commit_data, "tree ") or
309		commit_data.len < "tree ".len + format.formattedLength() + "\n".len or
310		commit_data["tree ".len + format.formattedLength()] != '\n')
311		{
312		return error.InvalidCommit;
313		}
314		return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]);
315		}
316
317		const TreeIterator = struct {
318		format: Oid.Format,
319		data: []const u8,
320		pos: usize,
321
322		const Entry = struct {
323		type: Type,
324		executable: bool,
325		name: [:0]const u8,
326		oid: Oid,
327
328		const Type = enum(u4) {
329		directory = 0o4,
330		file = 0o10,
331		symlink = 0o12,
332		gitlink = 0o16,
333		};
334		};
335
336		fn next(iterator: *TreeIterator) !?Entry {
337		if (iterator.pos == iterator.data.len) return null;
338
339		const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
340		const mode: packed struct {
341		permission: u9,
342		unused: u3,
343		type: u4,
344		} = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
345		const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree;
346		const executable = switch (mode.permission) {
347		0 => if (@"type" == .file) return error.InvalidTree else false,
348		0o644 => if (@"type" != .file) return error.InvalidTree else false,
349		0o755 => if (@"type" != .file) return error.InvalidTree else true,
350		else => return error.InvalidTree,
351		};
352		iterator.pos = mode_end + 1;
353
354		const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
355		const name = iterator.data[iterator.pos..name_end :0];
356		iterator.pos = name_end + 1;
357
358		const oid_length = iterator.format.byteLength();
359		if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
360		const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]);
361		iterator.pos += oid_length;
362
363		return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
364		}
365		};
366		};
367
368		/// A Git object database backed by a packfile. A packfile index is also used
369		/// for efficient access to objects in the packfile.
370		///
371		/// The format of the packfile and its associated index are documented in
372		/// [pack-format](https://git-scm.com/docs/pack-format).
373		const Odb = struct {
374		format: Oid.Format,
375		pack_file: *Io.File.Reader,
376		index_header: IndexHeader,
377		index_file: *Io.File.Reader,
378		cache: ObjectCache = .{},
379		allocator: Allocator,
380
381		/// Initializes the database from open pack and index files.
382		fn init(
383		odb: *Odb,
384		allocator: Allocator,
385		format: Oid.Format,
386		pack_file: *Io.File.Reader,
387		index_file: *Io.File.Reader,
388		) !void {
389		try pack_file.seekTo(0);
390		try index_file.seekTo(0);
391		odb.* = .{
392		.format = format,
393		.pack_file = pack_file,
394		.index_header = undefined,
395		.index_file = index_file,
396		.allocator = allocator,
397		};
398		try odb.index_header.read(&index_file.interface);
399		}
400
401		fn deinit(odb: *Odb) void {
402		odb.cache.deinit(odb.allocator);
403		odb.* = undefined;
404		}
405
406		/// Reads the object at the current position in the database.
407		fn readObject(odb: *Odb) !Object {
408		var base_offset = odb.pack_file.logicalPos();
409		var base_header: EntryHeader = undefined;
410		var delta_offsets: std.ArrayList(u64) = .empty;
411		defer delta_offsets.deinit(odb.allocator);
412		const base_object = while (true) {
413		if (odb.cache.get(base_offset)) \|base_object\| break base_object;
414
415		base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface);
416		switch (base_header) {
417		.ofs_delta => \|ofs_delta\| {
418		try delta_offsets.append(odb.allocator, base_offset);
419		base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
420		try odb.pack_file.seekTo(base_offset);
421		},
422		.ref_delta => \|ref_delta\| {
423		try delta_offsets.append(odb.allocator, base_offset);
424		try odb.seekOid(ref_delta.base_object);
425		base_offset = odb.pack_file.logicalPos();
426		},
427		else => {
428		const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
429		errdefer odb.allocator.free(base_data);
430		const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
431		try odb.cache.put(odb.allocator, base_offset, base_object);
432		break base_object;
433		},
434		}
435		};
436
437		const base_data = try resolveDeltaChain(
438		odb.allocator,
439		odb.format,
440		odb.pack_file,
441		base_object,
442		delta_offsets.items,
443		&odb.cache,
444		);
445
446		return .{ .type = base_object.type, .data = base_data };
447		}
448
449		/// Seeks to the beginning of the object with the given ID.
450		fn seekOid(odb: *Odb, oid: Oid) !void {
451		const oid_length = odb.format.byteLength();
452		const key = oid.slice()[0];
453		var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
454		var end_index = odb.index_header.fan_out_table[key];
455		const found_index = while (start_index < end_index) {
456		const mid_index = start_index + (end_index - start_index) / 2;
457		try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
458		const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface);
459		switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
460		.lt => start_index = mid_index + 1,
461		.gt => end_index = mid_index,
462		.eq => break mid_index,
463		}
464		} else return error.ObjectNotFound;
465
466		const n_objects = odb.index_header.fan_out_table[255];
467		const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
468		try odb.index_file.seekTo(offset_values_start + found_index * 4);
469		const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big));
470		const pack_offset = pack_offset: {
471		if (l1_offset.big) {
472		const l2_offset_values_start = offset_values_start + n_objects * 4;
473		try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
474		break :pack_offset try odb.index_file.interface.takeInt(u64, .big);
475		} else {
476		break :pack_offset l1_offset.value;
477		}
478		};
479
480		try odb.pack_file.seekTo(pack_offset);
481		}
482		};
483
484		const Object = struct {
485		type: Type,
486		data: []const u8,
487
488		const Type = enum {
489		commit,
490		tree,
491		blob,
492		tag,
493		};
494		};
495
496		/// A cache for object data.
497		///
498		/// The purpose of this cache is to speed up resolution of deltas by caching the
499		/// results of resolving delta objects, while maintaining a maximum cache size
500		/// to avoid excessive memory usage. If the total size of the objects in the
501		/// cache exceeds the maximum, the cache will begin evicting the least recently
502		/// used objects: when resolving delta chains, the most recently used objects
503		/// will likely be more helpful as they will be further along in the chain
504		/// (skipping earlier reconstruction steps).
505		///
506		/// Object data stored in the cache is managed by the cache. It should not be
507		/// freed by the caller at any point after inserting it into the cache. Any
508		/// objects remaining in the cache will be freed when the cache itself is freed.
509		const ObjectCache = struct {
510		objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty,
511		lru_nodes: std.DoublyLinkedList = .{},
512		lru_nodes_len: usize = 0,
513		byte_size: usize = 0,
514
515		const max_byte_size = 128 * 1024 * 1024; // 128MiB
516		/// A list of offsets stored in the cache, with the most recently used
517		/// entries at the end.
518		const LruListNode = struct {
519		data: u64,
520		node: std.DoublyLinkedList.Node,
521		};
522		const CacheEntry = struct { object: Object, lru_node: *LruListNode };
523
524		fn deinit(cache: *ObjectCache, allocator: Allocator) void {
525		var object_iterator = cache.objects.iterator();
526		while (object_iterator.next()) \|object\| {
527		allocator.free(object.value_ptr.object.data);
528		allocator.destroy(object.value_ptr.lru_node);
529		}
530		cache.objects.deinit(allocator);
531		cache.* = undefined;
532		}
533
534		/// Gets an object from the cache, moving it to the most recently used
535		/// position if it is present.
536		fn get(cache: *ObjectCache, offset: u64) ?Object {
537		if (cache.objects.get(offset)) \|entry\| {
538		cache.lru_nodes.remove(&entry.lru_node.node);
539		cache.lru_nodes.append(&entry.lru_node.node);
540		return entry.object;
541		} else {
542		return null;
543		}
544		}
545
546		/// Puts an object in the cache, possibly evicting older entries if the
547		/// cache exceeds its maximum size. Note that, although old objects may
548		/// be evicted, the object just added to the cache with this function
549		/// will not be evicted before the next call to `put` or `deinit` even if
550		/// it exceeds the maximum cache size.
551		fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
552		const lru_node = try allocator.create(LruListNode);
553		errdefer allocator.destroy(lru_node);
554		lru_node.data = offset;
555
556		const gop = try cache.objects.getOrPut(allocator, offset);
557		if (gop.found_existing) {
558		cache.byte_size -= gop.value_ptr.object.data.len;
559		cache.lru_nodes.remove(&gop.value_ptr.lru_node.node);
560		cache.lru_nodes_len -= 1;
561		allocator.destroy(gop.value_ptr.lru_node);
562		allocator.free(gop.value_ptr.object.data);
563		}
564		gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
565		cache.byte_size += object.data.len;
566		cache.lru_nodes.append(&lru_node.node);
567		cache.lru_nodes_len += 1;
568
569		while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) {
570		// The > 1 check is to make sure that we don't evict the most
571		// recently added node, even if it by itself happens to exceed the
572		// maximum size of the cache.
573		const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?));
574		cache.lru_nodes_len -= 1;
575		const evict_offset = evict_node.data;
576		allocator.destroy(evict_node);
577		const evict_object = cache.objects.get(evict_offset).?.object;
578		cache.byte_size -= evict_object.data.len;
579		allocator.free(evict_object.data);
580		_ = cache.objects.remove(evict_offset);
581		}
582		}
583		};
584
585		/// A single pkt-line in the Git protocol.
586		///
587		/// The format of a pkt-line is documented in
588		/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
589		/// meanings of the delimiter and response-end packets are documented in
590		/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
591		pub const Packet = union(enum) {
592		flush,
593		delimiter,
594		response_end,
595		data: []const u8,
596
597		pub const max_data_length = 65516;
598
599		/// Reads a packet in pkt-line format.
600		fn read(reader: *Io.Reader) !Packet {
601		const packet: Packet = try .peek(reader);
602		switch (packet) {
603		.data => \|data\| reader.toss(data.len),
604		else => {},
605		}
606		return packet;
607		}
608
609		/// Consumes the header of a pkt-line packet and reads any associated data
610		/// into the reader's buffer, but does not consume the data.
611		fn peek(reader: *Io.Reader) !Packet {
612		const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket;
613		switch (length) {
614		0 => return .flush,
615		1 => return .delimiter,
616		2 => return .response_end,
617		3 => return error.InvalidPacket,
618		else => if (length - 4 > max_data_length) return error.InvalidPacket,
619		}
620		return .{ .data = try reader.peek(length - 4) };
621		}
622
623		/// Writes a packet in pkt-line format.
624		fn write(packet: Packet, writer: *Io.Writer) !void {
625		switch (packet) {
626		.flush => try writer.writeAll("0000"),
627		.delimiter => try writer.writeAll("0001"),
628		.response_end => try writer.writeAll("0002"),
629		.data => \|data\| {
630		assert(data.len <= max_data_length);
631		try writer.print("{x:0>4}", .{data.len + 4});
632		try writer.writeAll(data);
633		},
634		}
635		}
636
637		/// Returns the normalized form of textual packet data, stripping any
638		/// trailing '\n'.
639		///
640		/// As documented in
641		/// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format),
642		/// non-binary (textual) pkt-line data should contain a trailing '\n', but
643		/// is not required to do so (implementations must support both forms).
644		fn normalizeText(data: []const u8) []const u8 {
645		return if (mem.endsWith(u8, data, "\n"))
646		data[0 .. data.len - 1]
647		else
648		data;
649		}
650		};
651
652		/// A client session for the Git protocol, currently limited to an HTTP(S)
653		/// transport. Only protocol version 2 is supported, as documented in
654		/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
655		pub const Session = struct {
656		transport: *std.http.Client,
657		location: Location,
658		supports_agent: bool,
659		supports_shallow: bool,
660		object_format: Oid.Format,
661		arena: Allocator,
662
663		const agent = "zig/" ++ @import("builtin").zig_version_string;
664		const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
665
666		/// Initializes a client session and discovers the capabilities of the
667		/// server for optimal transport.
668		pub fn init(
669		arena: Allocator,
670		transport: *std.http.Client,
671		uri: std.Uri,
672		/// Asserted to be at least `Packet.max_data_length`
673		response_buffer: []u8,
674		) !Session {
675		assert(response_buffer.len >= Packet.max_data_length);
676		var session: Session = .{
677		.transport = transport,
678		.location = try .init(arena, uri),
679		.supports_agent = false,
680		.supports_shallow = false,
681		.object_format = .sha1,
682		.arena = arena,
683		};
684		var capability_iterator: CapabilityIterator = undefined;
685		try session.getCapabilities(&capability_iterator, response_buffer);
686		defer capability_iterator.deinit();
687		while (try capability_iterator.next()) \|capability\| {
688		if (mem.eql(u8, capability.key, "agent")) {
689		session.supports_agent = true;
690		} else if (mem.eql(u8, capability.key, "fetch")) {
691		var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
692		while (feature_iterator.next()) \|feature\| {
693		if (mem.eql(u8, feature, "shallow")) {
694		session.supports_shallow = true;
695		}
696		}
697		} else if (mem.eql(u8, capability.key, "object-format")) {
698		if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) \|format\| {
699		session.object_format = format;
700		}
701		}
702		}
703		return session;
704		}
705
706		/// An owned `std.Uri` representing the location of the server (base URI).
707		const Location = struct {
708		uri: std.Uri,
709
710		fn init(arena: Allocator, uri: std.Uri) !Location {
711		const scheme = try arena.dupe(u8, uri.scheme);
712		const user = if (uri.user) \|user\| try std.fmt.allocPrint(arena, "{f}", .{
713		std.fmt.alt(user, .formatUser),
714		}) else null;
715		const password = if (uri.password) \|password\| try std.fmt.allocPrint(arena, "{f}", .{
716		std.fmt.alt(password, .formatPassword),
717		}) else null;
718		const host = if (uri.host) \|host\| try std.fmt.allocPrint(arena, "{f}", .{
719		std.fmt.alt(host, .formatHost),
720		}) else null;
721		const path = try std.fmt.allocPrint(arena, "{f}", .{
722		std.fmt.alt(uri.path, .formatPath),
723		});
724		// The query and fragment are not used as part of the base server URI.
725		return .{
726		.uri = .{
727		.scheme = scheme,
728		.user = if (user) \|s\| .{ .percent_encoded = s } else null,
729		.password = if (password) \|s\| .{ .percent_encoded = s } else null,
730		.host = if (host) \|s\| .{ .percent_encoded = s } else null,
731		.port = uri.port,
732		.path = .{ .percent_encoded = path },
733		},
734		};
735		}
736		};
737
738		/// Returns an iterator over capabilities supported by the server.
739		///
740		/// The `session.location` is updated if the server returns a redirect, so
741		/// that subsequent session functions do not need to handle redirects.
742		fn getCapabilities(session: Session, it: CapabilityIterator, response_buffer: []u8) !void {
743		const arena = session.arena;
744		assert(response_buffer.len >= Packet.max_data_length);
745		var info_refs_uri = session.location.uri;
746		{
747		const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
748		std.fmt.alt(session.location.uri.path, .formatPath),
749		});
750		info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{
751		"/", session_uri_path, "info/refs",
752		}) };
753		}
754		info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" };
755		info_refs_uri.fragment = null;
756
757		const max_redirects = 3;
758		it.* = .{
759		.request = try session.transport.request(.GET, info_refs_uri, .{
760		.redirect_behavior = .init(max_redirects),
761		.extra_headers = &.{
762		.{ .name = "Git-Protocol", .value = "version=2" },
763		},
764		}),
765		.reader = undefined,
766		.decompress = undefined,
767		};
768		errdefer it.deinit();
769		const request = &it.request;
770		try request.sendBodiless();
771
772		var redirect_buffer: [1024]u8 = undefined;
773		var response = try request.receiveHead(&redirect_buffer);
774		if (response.head.status != .ok) return error.ProtocolError;
775		const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
776		if (any_redirects_occurred) {
777		const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
778		std.fmt.alt(request.uri.path, .formatPath),
779		});
780		if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect;
781		var new_uri = request.uri;
782		new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] };
783		session.location = try .init(arena, new_uri);
784		}
785
786		const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
787		it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer);
788		var state: enum { response_start, response_content } = .response_start;
789		while (true) {
790		// Some Git servers (at least GitHub) include an additional
791		// '# service=git-upload-pack' informative response before sending
792		// the expected 'version 2' packet and capability information.
793		// This is not universal: SourceHut, for example, does not do this.
794		// Thus, we need to skip any such useless additional responses
795		// before we get the one we're actually looking for. The responses
796		// will be delimited by flush packets.
797		const packet = Packet.read(it.reader) catch \|err\| switch (err) {
798		error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
799		else => \|e\| return e,
800		};
801		switch (packet) {
802		.flush => state = .response_start,
803		.data => \|data\| switch (state) {
804		.response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) {
805		return;
806		} else {
807		state = .response_content;
808		},
809		else => {},
810		},
811		else => return error.UnexpectedPacket,
812		}
813		}
814		}
815
816		const CapabilityIterator = struct {
817		request: std.http.Client.Request,
818		reader: *Io.Reader,
819		decompress: std.http.Decompress,
820
821		const Capability = struct {
822		key: []const u8,
823		value: ?[]const u8 = null,
824
825		fn parse(data: []const u8) Capability {
826		return if (mem.indexOfScalar(u8, data, '=')) \|separator_pos\|
827		.{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] }
828		else
829		.{ .key = data };
830		}
831		};
832
833		fn deinit(it: *CapabilityIterator) void {
834		it.request.deinit();
835		it.* = undefined;
836		}
837
838		fn next(it: *CapabilityIterator) !?Capability {
839		switch (try Packet.read(it.reader)) {
840		.flush => return null,
841		.data => \|data\| return Capability.parse(Packet.normalizeText(data)),
842		else => return error.UnexpectedPacket,
843		}
844		}
845		};
846
847		const ListRefsOptions = struct {
848		/// The ref prefixes (if any) to use to filter the refs available on the
849		/// server. Note that the client must still check the returned refs
850		/// against its desired filters itself: the server is not required to
851		/// respect these prefix filters and may return other refs as well.
852		ref_prefixes: []const []const u8 = &.{},
853		/// Whether to include symref targets for returned symbolic refs.
854		include_symrefs: bool = false,
855		/// Whether to include the peeled object ID for returned tag refs.
856		include_peeled: bool = false,
857		/// Asserted to be at least `Packet.max_data_length`.
858		buffer: []u8,
859		};
860
861		/// Returns an iterator over refs known to the server.
862		pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void {
863		const arena = session.arena;
864		assert(options.buffer.len >= Packet.max_data_length);
865		var upload_pack_uri = session.location.uri;
866		{
867		const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
868		std.fmt.alt(session.location.uri.path, .formatPath),
869		});
870		upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
871		}
872		upload_pack_uri.query = null;
873		upload_pack_uri.fragment = null;
874
875		var body: Io.Writer = .fixed(options.buffer);
876		try Packet.write(.{ .data = "command=ls-refs\n" }, &body);
877		if (session.supports_agent) {
878		try Packet.write(.{ .data = agent_capability }, &body);
879		}
880		{
881		const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{
882		session.object_format,
883		});
884		try Packet.write(.{ .data = object_format_packet }, &body);
885		}
886		try Packet.write(.delimiter, &body);
887		for (options.ref_prefixes) \|ref_prefix\| {
888		const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix});
889		try Packet.write(.{ .data = ref_prefix_packet }, &body);
890		}
891		if (options.include_symrefs) {
892		try Packet.write(.{ .data = "symrefs\n" }, &body);
893		}
894		if (options.include_peeled) {
895		try Packet.write(.{ .data = "peel\n" }, &body);
896		}
897		try Packet.write(.flush, &body);
898
899		it.* = .{
900		.request = try session.transport.request(.POST, upload_pack_uri, .{
901		.redirect_behavior = .unhandled,
902		.extra_headers = &.{
903		.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
904		.{ .name = "Git-Protocol", .value = "version=2" },
905		},
906		}),
907		.reader = undefined,
908		.format = session.object_format,
909		.decompress = undefined,
910		};
911		const request = &it.request;
912		errdefer request.deinit();
913		try request.sendBodyComplete(body.buffered());
914
915		var response = try request.receiveHead(options.buffer);
916		if (response.head.status != .ok) return error.ProtocolError;
917		const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
918		it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer);
919		}
920
921		pub const RefIterator = struct {
922		format: Oid.Format,
923		request: std.http.Client.Request,
924		reader: *Io.Reader,
925		decompress: std.http.Decompress,
926
927		pub const Ref = struct {
928		oid: Oid,
929		name: []const u8,
930		symref_target: ?[]const u8,
931		peeled: ?Oid,
932		};
933
934		pub fn deinit(iterator: *RefIterator) void {
935		iterator.request.deinit();
936		iterator.* = undefined;
937		}
938
939		pub fn next(it: *RefIterator) !?Ref {
940		switch (try Packet.read(it.reader)) {
941		.flush => return null,
942		.data => \|data\| {
943		const ref_data = Packet.normalizeText(data);
944		const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;
945		const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
946
947		const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;
948		const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];
949
950		var symref_target: ?[]const u8 = null;
951		var peeled: ?Oid = null;
952		var last_sep_pos = name_sep_pos;
953		while (last_sep_pos < ref_data.len) {
954		const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len;
955		const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos];
956		if (mem.startsWith(u8, attribute, "symref-target:")) {
957		symref_target = attribute["symref-target:".len..];
958		} else if (mem.startsWith(u8, attribute, "peeled:")) {
959		peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;
960		}
961		last_sep_pos = next_sep_pos;
962		}
963
964		return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
965		},
966		else => return error.UnexpectedPacket,
967		}
968		}
969		};
970
971		/// Fetches the given refs from the server. A shallow fetch (depth 1) is
972		/// performed if the server supports it.
973		pub fn fetch(
974		session: Session,
975		fs: *FetchStream,
976		wants: []const []const u8,
977		/// Asserted to be at least `Packet.max_data_length`.
978		response_buffer: []u8,
979		) !void {
980		const arena = session.arena;
981		assert(response_buffer.len >= Packet.max_data_length);
982		var upload_pack_uri = session.location.uri;
983		{
984		const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
985		std.fmt.alt(session.location.uri.path, .formatPath),
986		});
987		upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
988		}
989		upload_pack_uri.query = null;
990		upload_pack_uri.fragment = null;
991
992		var body: Io.Writer = .fixed(response_buffer);
993		try Packet.write(.{ .data = "command=fetch\n" }, &body);
994		if (session.supports_agent) {
995		try Packet.write(.{ .data = agent_capability }, &body);
996		}
997		{
998		const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)});
999		try Packet.write(.{ .data = object_format_packet }, &body);
1000		}
1001		try Packet.write(.delimiter, &body);
1002		// Our packfile parser supports the OFS_DELTA object type
1003		try Packet.write(.{ .data = "ofs-delta\n" }, &body);
1004		// We do not currently convey server progress information to the user
1005		try Packet.write(.{ .data = "no-progress\n" }, &body);
1006		if (session.supports_shallow) {
1007		try Packet.write(.{ .data = "deepen 1\n" }, &body);
1008		}
1009		for (wants) \|want\| {
1010		var buf: [Packet.max_data_length]u8 = undefined;
1011		const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
1012		try Packet.write(.{ .data = arg }, &body);
1013		}
1014		try Packet.write(.{ .data = "done\n" }, &body);
1015		try Packet.write(.flush, &body);
1016
1017		fs.* = .{
1018		.request = try session.transport.request(.POST, upload_pack_uri, .{
1019		.redirect_behavior = .not_allowed,
1020		.extra_headers = &.{
1021		.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
1022		.{ .name = "Git-Protocol", .value = "version=2" },
1023		},
1024		}),
1025		.input = undefined,
1026		.reader = undefined,
1027		.remaining_len = undefined,
1028		.decompress = undefined,
1029		};
1030		const request = &fs.request;
1031		errdefer request.deinit();
1032
1033		try request.sendBodyComplete(body.buffered());
1034
1035		var response = try request.receiveHead(&.{});
1036		if (response.head.status != .ok) return error.ProtocolError;
1037
1038		const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
1039		const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer);
1040		// We are not interested in any of the sections of the returned fetch
1041		// data other than the packfile section, since we aren't doing anything
1042		// complex like ref negotiation (this is a fresh clone).
1043		var state: enum { section_start, section_content } = .section_start;
1044		while (true) {
1045		const packet = try Packet.read(reader);
1046		switch (state) {
1047		.section_start => switch (packet) {
1048		.data => \|data\| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) {
1049		fs.input = reader;
1050		fs.reader = .{
1051		.buffer = &.{},
1052		.vtable = &.{ .stream = FetchStream.stream },
1053		.seek = 0,
1054		.end = 0,
1055		};
1056		fs.remaining_len = 0;
1057		return;
1058		} else {
1059		state = .section_content;
1060		},
1061		else => return error.UnexpectedPacket,
1062		},
1063		.section_content => switch (packet) {
1064		.delimiter => state = .section_start,
1065		.data => {},
1066		else => return error.UnexpectedPacket,
1067		},
1068		}
1069		}
1070		}
1071
1072		pub const FetchStream = struct {
1073		request: std.http.Client.Request,
1074		input: *Io.Reader,
1075		reader: Io.Reader,
1076		err: ?Error = null,
1077		remaining_len: usize,
1078		decompress: std.http.Decompress,
1079
1080		pub fn deinit(fs: *FetchStream) void {
1081		fs.request.deinit();
1082		}
1083
1084		pub const Error = error{
1085		InvalidPacket,
1086		ProtocolError,
1087		UnexpectedPacket,
1088		WriteFailed,
1089		ReadFailed,
1090		EndOfStream,
1091		};
1092
1093		const StreamCode = enum(u8) {
1094		pack_data = 1,
1095		progress = 2,
1096		fatal_error = 3,
1097		_,
1098		};
1099
1100		pub fn stream(r: Io.Reader, w: Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
1101		const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r));
1102		const input = fs.input;
1103		if (fs.remaining_len == 0) {
1104		while (true) {
1105		switch (Packet.peek(input) catch \|err\| {
1106		fs.err = err;
1107		return error.ReadFailed;
1108		}) {
1109		.flush => return error.EndOfStream,
1110		.data => \|data\| switch (@as(StreamCode, @enumFromInt(data[0]))) {
1111		.pack_data => {
1112		input.toss(1);
1113		fs.remaining_len = data.len - 1;
1114		break;
1115		},
1116		.fatal_error => {
1117		fs.err = error.ProtocolError;
1118		return error.ReadFailed;
1119		},
1120		else => {
1121		input.toss(data.len);
1122		},
1123		},
1124		else => {
1125		fs.err = error.UnexpectedPacket;
1126		return error.ReadFailed;
1127		},
1128		}
1129		}
1130		}
1131		const buf = limit.slice(try w.writableSliceGreedy(1));
1132		const n = @min(buf.len, fs.remaining_len);
1133		try input.readSliceAll(buf[0..n]);
1134		w.advance(n);
1135		fs.remaining_len -= n;
1136		return n;
1137		}
1138		};
1139		};
1140
1141		const PackHeader = struct {
1142		total_objects: u32,
1143
1144		const signature = "PACK";
1145		const supported_version = 2;
1146
1147		fn read(reader: *Io.Reader) !PackHeader {
1148		const actual_signature = reader.take(4) catch \|e\| switch (e) {
1149		error.EndOfStream => return error.InvalidHeader,
1150		else => \|other\| return other,
1151		};
1152		if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
1153		const version = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1154		error.EndOfStream => return error.InvalidHeader,
1155		else => \|other\| return other,
1156		};
1157		if (version != supported_version) return error.UnsupportedVersion;
1158		const total_objects = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1159		error.EndOfStream => return error.InvalidHeader,
1160		else => \|other\| return other,
1161		};
1162		return .{ .total_objects = total_objects };
1163		}
1164		};
1165
1166		const EntryHeader = union(Type) {
1167		commit: Undeltified,
1168		tree: Undeltified,
1169		blob: Undeltified,
1170		tag: Undeltified,
1171		ofs_delta: OfsDelta,
1172		ref_delta: RefDelta,
1173
1174		const Type = enum(u3) {
1175		commit = 1,
1176		tree = 2,
1177		blob = 3,
1178		tag = 4,
1179		ofs_delta = 6,
1180		ref_delta = 7,
1181		};
1182
1183		const Undeltified = struct {
1184		uncompressed_length: u64,
1185		};
1186
1187		const OfsDelta = struct {
1188		offset: u64,
1189		uncompressed_length: u64,
1190		};
1191
1192		const RefDelta = struct {
1193		base_object: Oid,
1194		uncompressed_length: u64,
1195		};
1196
1197		fn objectType(header: EntryHeader) Object.Type {
1198		return switch (header) {
1199		inline .commit, .tree, .blob, .tag => \|_, tag\| @field(Object.Type, @tagName(tag)),
1200		else => unreachable,
1201		};
1202		}
1203
1204		fn uncompressedLength(header: EntryHeader) u64 {
1205		return switch (header) {
1206		inline else => \|entry\| entry.uncompressed_length,
1207		};
1208		}
1209
1210		fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader {
1211		const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
1212		const initial: InitialByte = @bitCast(reader.takeByte() catch \|e\| switch (e) {
1213		error.EndOfStream => return error.InvalidFormat,
1214		else => \|other\| return other,
1215		});
1216		const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0;
1217		var uncompressed_length: u64 = initial.len;
1218		uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
1219		const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;
1220		return switch (@"type") {
1221		inline .commit, .tree, .blob, .tag => \|tag\| @unionInit(EntryHeader, @tagName(tag), .{
1222		.uncompressed_length = uncompressed_length,
1223		}),
1224		.ofs_delta => .{ .ofs_delta = .{
1225		.offset = try readOffsetVarInt(reader),
1226		.uncompressed_length = uncompressed_length,
1227		} },
1228		.ref_delta => .{ .ref_delta = .{
1229		.base_object = Oid.readBytes(format, reader) catch \|e\| switch (e) {
1230		error.EndOfStream => return error.InvalidFormat,
1231		else => \|other\| return other,
1232		},
1233		.uncompressed_length = uncompressed_length,
1234		} },
1235		};
1236		}
1237		};
1238
1239		fn readOffsetVarInt(r: *Io.Reader) !u64 {
1240		const Byte = packed struct { value: u7, has_next: bool };
1241		var b: Byte = @bitCast(try r.takeByte());
1242		var value: u64 = b.value;
1243		while (b.has_next) {
1244		b = @bitCast(try r.takeByte());
1245		value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
1246		value \|= b.value;
1247		}
1248		return value;
1249		}
1250
1251		const IndexHeader = struct {
1252		fan_out_table: [256]u32,
1253
1254		const signature = "\xFFtOc";
1255		const supported_version = 2;
1256		const size = 4 + 4 + @sizeOf([256]u32);
1257
1258		fn read(index_header: IndexHeader, reader: Io.Reader) !void {
1259		const sig = try reader.take(4);
1260		if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
1261		const version = try reader.takeInt(u32, .big);
1262		if (version != supported_version) return error.UnsupportedVersion;
1263		try reader.readSliceEndian(u32, &index_header.fan_out_table, .big);
1264		}
1265		};
1266
1267		const IndexEntry = struct {
1268		offset: u64,
1269		crc32: u32,
1270		};
1271
1272		/// Writes out a version 2 index for the given packfile, as documented in
1273		/// [pack-format](https://git-scm.com/docs/pack-format).
1274		pub fn indexPack(
1275		allocator: Allocator,
1276		format: Oid.Format,
1277		pack: *Io.File.Reader,
1278		index_writer: *Io.File.Writer,
1279		) !void {
1280		try pack.seekTo(0);
1281
1282		var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
1283		defer index_entries.deinit(allocator);
1284		var pending_deltas: std.ArrayList(IndexEntry) = .empty;
1285		defer pending_deltas.deinit(allocator);
1286
1287		const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas);
1288
1289		var cache: ObjectCache = .{};
1290		defer cache.deinit(allocator);
1291		var remaining_deltas = pending_deltas.items.len;
1292		while (remaining_deltas > 0) {
1293		var i: usize = remaining_deltas;
1294		while (i > 0) {
1295		i -= 1;
1296		const delta = pending_deltas.items[i];
1297		if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) \|oid\| {
1298		try index_entries.put(allocator, oid, delta);
1299		_ = pending_deltas.swapRemove(i);
1300		}
1301		}
1302		if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
1303		remaining_deltas = pending_deltas.items.len;
1304		}
1305
1306		var oids: std.ArrayList(Oid) = .empty;
1307		defer oids.deinit(allocator);
1308		try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
1309		var index_entries_iter = index_entries.iterator();
1310		while (index_entries_iter.next()) \|entry\| {
1311		oids.appendAssumeCapacity(entry.key_ptr.*);
1312		}
1313		mem.sortUnstable(Oid, oids.items, {}, struct {
1314		fn lessThan(_: void, o1: Oid, o2: Oid) bool {
1315		return mem.lessThan(u8, o1.slice(), o2.slice());
1316		}
1317		}.lessThan);
1318
1319		var fan_out_table: [256]u32 = undefined;
1320		var count: u32 = 0;
1321		var fan_out_index: u8 = 0;
1322		for (oids.items) \|oid\| {
1323		const key = oid.slice()[0];
1324		if (key > fan_out_index) {
1325		@memset(fan_out_table[fan_out_index..key], count);
1326		fan_out_index = key;
1327		}
1328		count += 1;
1329		}
1330		@memset(fan_out_table[fan_out_index..], count);
1331
1332		var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{});
1333		const writer = &index_hashed_writer.writer;
1334		try writer.writeAll(IndexHeader.signature);
1335		try writer.writeInt(u32, IndexHeader.supported_version, .big);
1336		for (fan_out_table) \|fan_out_entry\| {
1337		try writer.writeInt(u32, fan_out_entry, .big);
1338		}
1339
1340		for (oids.items) \|oid\| {
1341		try writer.writeAll(oid.slice());
1342		}
1343
1344		for (oids.items) \|oid\| {
1345		try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big);
1346		}
1347
1348		var big_offsets: std.ArrayList(u64) = .empty;
1349		defer big_offsets.deinit(allocator);
1350		for (oids.items) \|oid\| {
1351		const offset = index_entries.get(oid).?.offset;
1352		if (offset <= std.math.maxInt(u31)) {
1353		try writer.writeInt(u32, @intCast(offset), .big);
1354		} else {
1355		const index = big_offsets.items.len;
1356		try big_offsets.append(allocator, offset);
1357		try writer.writeInt(u32, @as(u32, @intCast(index)) \| (1 << 31), .big);
1358		}
1359		}
1360		for (big_offsets.items) \|offset\| {
1361		try writer.writeInt(u64, offset, .big);
1362		}
1363
1364		try writer.writeAll(pack_checksum.slice());
1365		const index_checksum = index_hashed_writer.hasher.finalResult();
1366		try index_writer.interface.writeAll(index_checksum.slice());
1367		try index_writer.end();
1368		}
1369
1370		/// Performs the first pass over the packfile data for index construction.
1371		/// This will index all non-delta objects, queue delta objects for further
1372		/// processing, and return the pack checksum (which is part of the index
1373		/// format).
1374		fn indexPackFirstPass(
1375		allocator: Allocator,
1376		format: Oid.Format,
1377		pack: *Io.File.Reader,
1378		index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1379		pending_deltas: *std.ArrayList(IndexEntry),
1380		) !Oid {
1381		var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1382		var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
1383		var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer);
1384
1385		const pack_header = try PackHeader.read(&pack_hashed.reader);
1386
1387		for (0..pack_header.total_objects) \|_\| {
1388		const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen();
1389		const entry_header = try EntryHeader.read(format, &pack_hashed.reader);
1390		switch (entry_header) {
1391		.commit, .tree, .blob, .tag => \|object\| {
1392		var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{});
1393		var oid_hasher: Oid.Hashing = .init(format, &flate_buffer);
1394		const oid_hasher_w = oid_hasher.writer();
1395		// The object header is not included in the pack data but is
1396		// part of the object's ID
1397		try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length });
1398		const n = try entry_decompress.reader.streamRemaining(oid_hasher_w);
1399		if (n != object.uncompressed_length) return error.InvalidObject;
1400		const oid = oid_hasher.final();
1401		if (!skip_checksums) @compileError("TODO");
1402		try index_entries.put(allocator, oid, .{
1403		.offset = entry_offset,
1404		.crc32 = 0,
1405		});
1406		},
1407		inline .ofs_delta, .ref_delta => \|delta\| {
1408		var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer);
1409		const n = try entry_decompress.reader.discardRemaining();
1410		if (n != delta.uncompressed_length) return error.InvalidObject;
1411		if (!skip_checksums) @compileError("TODO");
1412		try pending_deltas.append(allocator, .{
1413		.offset = entry_offset,
1414		.crc32 = 0,
1415		});
1416		},
1417		}
1418		}
1419
1420		if (!skip_checksums) @compileError("TODO");
1421		return pack_hashed.hasher.finalResult();
1422		}
1423
1424		/// Attempts to determine the final object ID of the given deltified object.
1425		/// May return null if this is not yet possible (if the delta is a ref-based
1426		/// delta and we do not yet know the offset of the base object).
1427		fn indexPackHashDelta(
1428		allocator: Allocator,
1429		format: Oid.Format,
1430		pack: *Io.File.Reader,
1431		delta: IndexEntry,
1432		index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1433		cache: *ObjectCache,
1434		) !?Oid {
1435		// Figure out the chain of deltas to resolve
1436		var base_offset = delta.offset;
1437		var base_header: EntryHeader = undefined;
1438		var delta_offsets: std.ArrayList(u64) = .empty;
1439		defer delta_offsets.deinit(allocator);
1440		const base_object = while (true) {
1441		if (cache.get(base_offset)) \|base_object\| break base_object;
1442
1443		try pack.seekTo(base_offset);
1444		base_header = try EntryHeader.read(format, &pack.interface);
1445		switch (base_header) {
1446		.ofs_delta => \|ofs_delta\| {
1447		try delta_offsets.append(allocator, base_offset);
1448		base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
1449		},
1450		.ref_delta => \|ref_delta\| {
1451		try delta_offsets.append(allocator, base_offset);
1452		base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1453		},
1454		else => {
1455		const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
1456		errdefer allocator.free(base_data);
1457		const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1458		try cache.put(allocator, base_offset, base_object);
1459		break base_object;
1460		},
1461		}
1462		};
1463
1464		const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
1465
1466		var entry_hasher_buffer: [64]u8 = undefined;
1467		var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer);
1468		const entry_hasher_w = entry_hasher.writer();
1469		// Writes to hashers cannot fail.
1470		entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable;
1471		entry_hasher_w.writeAll(base_data) catch unreachable;
1472		return entry_hasher.final();
1473		}
1474
1475		/// Resolves a chain of deltas, returning the final base object data. `pack` is
1476		/// assumed to be looking at the start of the object data for the base object of
1477		/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
1478		/// to obtain the final object.
1479		fn resolveDeltaChain(
1480		allocator: Allocator,
1481		format: Oid.Format,
1482		pack: *Io.File.Reader,
1483		base_object: Object,
1484		delta_offsets: []const u64,
1485		cache: *ObjectCache,
1486		) ![]const u8 {
1487		var base_data = base_object.data;
1488		var i: usize = delta_offsets.len;
1489		while (i > 0) {
1490		i -= 1;
1491
1492		const delta_offset = delta_offsets[i];
1493		try pack.seekTo(delta_offset);
1494		const delta_header = try EntryHeader.read(format, &pack.interface);
1495		const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
1496		defer allocator.free(delta_data);
1497		var delta_reader: Io.Reader = .fixed(delta_data);
1498		_ = try delta_reader.takeLeb128(u64); // base object size
1499		const expanded_size = try delta_reader.takeLeb128(u64);
1500
1501		const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1502		const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1503		errdefer allocator.free(expanded_data);
1504		var expanded_delta_stream: Io.Writer = .fixed(expanded_data);
1505		try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
1506		if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
1507
1508		try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1509		base_data = expanded_data;
1510		}
1511		return base_data;
1512		}
1513
1514		/// Reads the complete contents of an object from `reader`. This function may
1515		/// read more bytes than required from `reader`, so the reader position after
1516		/// returning is not reliable.
1517		fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 {
1518		const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1519		var aw: Io.Writer.Allocating = .init(allocator);
1520		try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len);
1521		defer aw.deinit();
1522		var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
1523		try decompress.reader.streamExact(&aw.writer, alloc_size);
1524		return aw.toOwnedSlice();
1525		}
1526
1527		/// Expands delta data from `delta_reader` to `writer`.
1528		///
1529		/// The format of the delta data is documented in
1530		/// [pack-format](https://git-scm.com/docs/pack-format).
1531		fn expandDelta(base_object: []const u8, delta_reader: Io.Reader, writer: Io.Writer) !void {
1532		while (true) {
1533		const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch \|e\| switch (e) {
1534		error.EndOfStream => return,
1535		else => \|other\| return other,
1536		});
1537		if (inst.copy) {
1538		const available: packed struct {
1539		offset1: bool,
1540		offset2: bool,
1541		offset3: bool,
1542		offset4: bool,
1543		size1: bool,
1544		size2: bool,
1545		size3: bool,
1546		} = @bitCast(inst.value);
1547		const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1548		.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
1549		.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
1550		.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
1551		.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
1552		};
1553		const base_offset: u32 = @bitCast(offset_parts);
1554		const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1555		.size1 = if (available.size1) try delta_reader.takeByte() else 0,
1556		.size2 = if (available.size2) try delta_reader.takeByte() else 0,
1557		.size3 = if (available.size3) try delta_reader.takeByte() else 0,
1558		};
1559		var size: u24 = @bitCast(size_parts);
1560		if (size == 0) size = 0x10000;
1561		try writer.writeAll(base_object[base_offset..][0..size]);
1562		} else if (inst.value != 0) {
1563		try delta_reader.streamExact(writer, inst.value);
1564		} else {
1565		return error.InvalidDeltaInstruction;
1566		}
1567		}
1568		}
1569
1570		/// Runs the packfile indexing and checkout test.
1571		///
1572		/// The two testrepo repositories under testdata contain identical commit
1573		/// histories and contents.
1574		///
1575		/// To verify the contents of the packfiles using Git alone, run the
1576		/// following commands in an empty directory:
1577		///
1578		/// 1. `git init --object-format=(sha1\|sha256)`
1579		/// 2. `git unpack-objects <path/to/testrepo.pack`
1580		/// 3. `git fsck` - will print one "dangling commit":
1581		/// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb`
1582		/// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a`
1583		/// 4. `git checkout $commit`
1584		fn runRepositoryTest(io: Io, comptime format: Oid.Format, head_commit: []const u8) !void {
1585		const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack");
1586
1587		var git_dir = testing.tmpDir(.{});
1588		defer git_dir.cleanup();
1589		var pack_file = try git_dir.dir.createFile(io, "testrepo.pack", .{ .read = true });
1590		defer pack_file.close(io);
1591		try pack_file.writeStreamingAll(io, testrepo_pack);
1592
1593		var pack_file_buffer: [2000]u8 = undefined;
1594		var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
1595
1596		var index_file = try git_dir.dir.createFile(io, "testrepo.idx", .{ .read = true });
1597		defer index_file.close(io);
1598		var index_file_buffer: [2000]u8 = undefined;
1599		var index_file_writer = index_file.writer(io, &index_file_buffer);
1600		try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer);
1601
1602		// Arbitrary size limit on files read while checking the repository contents
1603		// (all files in the test repo are known to be smaller than this)
1604		const max_file_size = 8192;
1605
1606		if (!skip_checksums) {
1607		const index_file_data = try git_dir.dir.readFileAlloc(io, "testrepo.idx", testing.allocator, .limited(max_file_size));
1608		defer testing.allocator.free(index_file_data);
1609		// testrepo.idx is generated by Git. The index created by this file should
1610		// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1611		// this.
1612		const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
1613		try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
1614		}
1615
1616		var index_file_reader = index_file.reader(io, &index_file_buffer);
1617		var repository: Repository = undefined;
1618		try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader);
1619		defer repository.deinit();
1620
1621		var worktree = testing.tmpDir(.{ .iterate = true });
1622		defer worktree.cleanup();
1623
1624		const commit_id = try Oid.parse(format, head_commit);
1625
1626		var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1627		defer diagnostics.deinit();
1628		try repository.checkout(io, worktree.dir, commit_id, &diagnostics);
1629		try testing.expect(diagnostics.errors.items.len == 0);
1630
1631		const expected_files: []const []const u8 = &.{
1632		"dir/file",
1633		"dir/subdir/file",
1634		"dir/subdir/file2",
1635		"dir2/file",
1636		"dir3/file",
1637		"dir3/file2",
1638		"file",
1639		"file2",
1640		"file3",
1641		"file4",
1642		"file5",
1643		"file6",
1644		"file7",
1645		"file8",
1646		"file9",
1647		};
1648		var actual_files: std.ArrayList([]u8) = .empty;
1649		defer actual_files.deinit(testing.allocator);
1650		defer for (actual_files.items) \|file\| testing.allocator.free(file);
1651		var walker = try worktree.dir.walk(testing.allocator);
1652		defer walker.deinit();
1653		while (try walker.next(io)) \|entry\| {
1654		if (entry.kind != .file) continue;
1655		const path = try testing.allocator.dupe(u8, entry.path);
1656		errdefer testing.allocator.free(path);
1657		mem.replaceScalar(u8, path, std.fs.path.sep, '/');
1658		try actual_files.append(testing.allocator, path);
1659		}
1660		mem.sortUnstable([]u8, actual_files.items, {}, struct {
1661		fn lessThan(_: void, a: []u8, b: []u8) bool {
1662		return mem.lessThan(u8, a, b);
1663		}
1664		}.lessThan);
1665		try testing.expectEqualDeep(expected_files, actual_files.items);
1666
1667		const expected_file_contents =
1668		\\revision 1
1669		\\revision 2
1670		\\revision 4
1671		\\revision 5
1672		\\revision 7
1673		\\revision 8
1674		\\revision 9
1675		\\revision 10
1676		\\revision 12
1677		\\revision 13
1678		\\revision 14
1679		\\revision 18
1680		\\revision 19
1681		\\
1682		;
1683		const actual_file_contents = try worktree.dir.readFileAlloc(io, "file", testing.allocator, .limited(max_file_size));
1684		defer testing.allocator.free(actual_file_contents);
1685		try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1686		}
1687
1688		/// Checksum calculation is useful for troubleshooting and debugging, but it's
1689		/// redundant since the package manager already does content hashing at the
1690		/// end. Let's save time by not doing that work, but, I left a cookie crumb
1691		/// trail here if you want to restore the functionality for tinkering purposes.
1692		const skip_checksums = true;
1693
1694		test "SHA-1 packfile indexing and checkout" {
1695		try runRepositoryTest(std.testing.io, .sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
1696		}
1697
1698		test "SHA-256 packfile indexing and checkout" {
1699		try runRepositoryTest(std.testing.io, .sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a");
1700		}
1701
1702		/// Checks out a commit of a packfile. Intended for experimenting with and
1703		/// benchmarking possible optimizations to the indexing and checkout behavior.
1704		pub fn main() !void {
1705		const allocator = std.heap.smp_allocator;
1706
1707		var threaded: Io.Threaded = .init(allocator, .{});
1708		defer threaded.deinit();
1709		const io = threaded.io();
1710
1711		const args = try std.process.argsAlloc(allocator);
1712		defer std.process.argsFree(allocator, args);
1713		if (args.len != 5) {
1714		return error.InvalidArguments; // Arguments: format packfile commit worktree
1715		}
1716
1717		const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat;
1718
1719		var pack_file = try Io.Dir.cwd().openFile(io, args[2], .{});
1720		defer pack_file.close(io);
1721		var pack_file_buffer: [4096]u8 = undefined;
1722		var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
1723
1724		const commit = try Oid.parse(format, args[3]);
1725		var worktree = try Io.Dir.cwd().createDirPathOpen(io, args[4], .{});
1726		defer worktree.close(io);
1727
1728		var git_dir = try worktree.createDirPathOpen(io, ".git", .{});
1729		defer git_dir.close(io);
1730
1731		std.debug.print("Starting index...\n", .{});
1732		var index_file = try git_dir.createFile(io, "idx", .{ .read = true });
1733		defer index_file.close(io);
1734		var index_file_buffer: [4096]u8 = undefined;
1735		var index_file_writer = index_file.writer(io, &index_file_buffer);
1736		try indexPack(allocator, format, &pack_file_reader, &index_file_writer);
1737
1738		std.debug.print("Starting checkout...\n", .{});
1739		var index_file_reader = index_file.reader(io, &index_file_buffer);
1740		var repository: Repository = undefined;
1741		try repository.init(allocator, format, &pack_file_reader, &index_file_reader);
1742		defer repository.deinit();
1743		var diagnostics: Diagnostics = .{ .allocator = allocator };
1744		defer diagnostics.deinit();
1745		try repository.checkout(io, worktree, commit, &diagnostics);
1746
1747		for (diagnostics.errors.items) \|err\| {
1748		std.debug.print("Diagnostic: {}\n", .{err});
1749		}
1750		}

src/Package/Fetch/git/testdata/testrepo-sha1.idx deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha1.idx and /dev/null differ

src/Package/Fetch/git/testdata/testrepo-sha1.pack deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha1.pack and /dev/null differ

src/Package/Fetch/git/testdata/testrepo-sha256.idx deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha256.idx and /dev/null differ

src/Package/Fetch/git/testdata/testrepo-sha256.pack deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha256.pack and /dev/null differ

src/Package/Manifest.zig deleted-734

...	...	@@ -1,734 +0,0 @@
1		const Manifest = @This();
2
3		const std = @import("std");
4		const Io = std.Io;
5		const mem = std.mem;
6		const Allocator = std.mem.Allocator;
7		const assert = std.debug.assert;
8		const Ast = std.zig.Ast;
9		const testing = std.testing;
10
11		const Package = @import("../Package.zig");
12
13		pub const max_bytes = 10 * 1024 * 1024;
14		pub const basename = "build.zig.zon";
15		pub const max_name_len = 32;
16		pub const max_version_len = 32;
17
18		pub const Dependency = struct {
19		location: Location,
20		location_tok: Ast.TokenIndex,
21		location_node: Ast.Node.Index,
22		hash: ?[]const u8,
23		hash_tok: Ast.OptionalTokenIndex,
24		hash_node: Ast.Node.OptionalIndex,
25		node: Ast.Node.Index,
26		name_tok: Ast.TokenIndex,
27		lazy: bool,
28
29		pub const Location = union(enum) {
30		url: []const u8,
31		path: []const u8,
32		};
33		};
34
35		pub const ErrorMessage = struct {
36		msg: []const u8,
37		tok: Ast.TokenIndex,
38		off: u32,
39		};
40
41		name: []const u8,
42		id: u32,
43		version: std.SemanticVersion,
44		version_node: Ast.Node.Index,
45		dependencies: std.array_hash_map.String(Dependency),
46		dependencies_node: Ast.Node.OptionalIndex,
47		paths: std.array_hash_map.String(void),
48		minimum_zig_version: ?std.SemanticVersion,
49
50		errors: []ErrorMessage,
51		arena_state: std.heap.ArenaAllocator.State,
52
53		pub const ParseOptions = struct {
54		allow_missing_paths_field: bool = false,
55		};
56
57		pub const Error = Allocator.Error;
58
59		pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
60		const main_node_index = ast.nodeData(.root).node;
61
62		var arena_instance = std.heap.ArenaAllocator.init(gpa);
63		errdefer arena_instance.deinit();
64
65		var p: Parse = .{
66		.gpa = gpa,
67		.ast = ast.*,
68		.arena = arena_instance.allocator(),
69		.errors = .empty,
70
71		.name = undefined,
72		.id = 0,
73		.version = undefined,
74		.version_node = undefined,
75		.dependencies = .{},
76		.dependencies_node = .none,
77		.paths = .empty,
78		.allow_missing_paths_field = options.allow_missing_paths_field,
79		.minimum_zig_version = null,
80		.buf = .empty,
81		};
82		defer p.buf.deinit(gpa);
83		defer p.errors.deinit(gpa);
84		defer p.dependencies.deinit(gpa);
85		defer p.paths.deinit(gpa);
86
87		p.parseRoot(main_node_index, rng) catch \|err\| switch (err) {
88		error.ParseFailure => assert(p.errors.items.len > 0),
89		else => \|e\| return e,
90		};
91
92		return .{
93		.name = p.name,
94		.id = p.id,
95		.version = p.version,
96		.version_node = p.version_node,
97		.dependencies = try p.dependencies.clone(p.arena),
98		.dependencies_node = p.dependencies_node,
99		.paths = try p.paths.clone(p.arena),
100		.minimum_zig_version = p.minimum_zig_version,
101		.errors = try p.arena.dupe(ErrorMessage, p.errors.items),
102		.arena_state = arena_instance.state,
103		};
104		}
105
106		pub fn deinit(man: *Manifest, gpa: Allocator) void {
107		man.arena_state.promote(gpa).deinit();
108		man.* = undefined;
109		}
110
111		pub fn copyErrorsIntoBundle(
112		man: Manifest,
113		ast: Ast,
114		/// ErrorBundle null-terminated string index
115		src_path: u32,
116		eb: *std.zig.ErrorBundle.Wip,
117		) Allocator.Error!void {
118		for (man.errors) \|msg\| {
119		const start_loc = ast.tokenLocation(0, msg.tok);
120
121		try eb.addRootErrorMessage(.{
122		.msg = try eb.addString(msg.msg),
123		.src_loc = try eb.addSourceLocation(.{
124		.src_path = src_path,
125		.span_start = ast.tokenStart(msg.tok),
126		.span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len),
127		.span_main = ast.tokenStart(msg.tok) + msg.off,
128		.line = @intCast(start_loc.line),
129		.column = @intCast(start_loc.column),
130		.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
131		}),
132		});
133		}
134		}
135
136		const Parse = struct {
137		gpa: Allocator,
138		ast: Ast,
139		arena: Allocator,
140		buf: std.ArrayList(u8),
141		errors: std.ArrayList(ErrorMessage),
142
143		name: []const u8,
144		id: u32,
145		version: std.SemanticVersion,
146		version_node: Ast.Node.Index,
147		dependencies: std.array_hash_map.String(Dependency),
148		dependencies_node: Ast.Node.OptionalIndex,
149		paths: std.array_hash_map.String(void),
150		allow_missing_paths_field: bool,
151		minimum_zig_version: ?std.SemanticVersion,
152
153		const InnerError = error{ ParseFailure, OutOfMemory };
154
155		fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void {
156		const ast = p.ast;
157		const main_token = ast.nodeMainToken(node);
158
159		var buf: [2]Ast.Node.Index = undefined;
160		const struct_init = ast.fullStructInit(&buf, node) orelse {
161		return fail(p, main_token, "expected top level expression to be a struct", .{});
162		};
163
164		var have_name = false;
165		var have_version = false;
166		var have_included_paths = false;
167		var fingerprint: ?Package.Fingerprint = null;
168
169		for (struct_init.ast.fields) \|field_init\| {
170		const name_token = ast.firstToken(field_init) - 2;
171		const field_name = try identifierTokenString(p, name_token);
172		// We could get fancy with reflection and comptime logic here but doing
173		// things manually provides an opportunity to do any additional verification
174		// that is desirable on a per-field basis.
175		if (mem.eql(u8, field_name, "dependencies")) {
176		p.dependencies_node = field_init.toOptional();
177		try parseDependencies(p, field_init);
178		} else if (mem.eql(u8, field_name, "paths")) {
179		have_included_paths = true;
180		try parseIncludedPaths(p, field_init);
181		} else if (mem.eql(u8, field_name, "name")) {
182		p.name = try parseName(p, field_init);
183		have_name = true;
184		} else if (mem.eql(u8, field_name, "fingerprint")) {
185		fingerprint = try parseFingerprint(p, field_init);
186		} else if (mem.eql(u8, field_name, "version")) {
187		p.version_node = field_init;
188		const version_text = try parseString(p, field_init);
189		if (version_text.len > max_version_len) {
190		try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len });
191		}
192		p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
193		try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
194		break :v undefined;
195		};
196		have_version = true;
197		} else if (mem.eql(u8, field_name, "minimum_zig_version")) {
198		const version_text = try parseString(p, field_init);
199		p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
200		try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
201		break :v null;
202		};
203		} else {
204		// Ignore unknown fields so that we can add fields in future zig
205		// versions without breaking older zig versions.
206		}
207		}
208
209		if (!have_name) {
210		try appendError(p, main_token, "missing top-level 'name' field", .{});
211		} else {
212		if (fingerprint) \|n\| {
213		if (!n.validate(p.name)) {
214		return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{
215		n.int(), Package.Fingerprint.generate(rng, p.name).int(),
216		});
217		}
218		p.id = n.id;
219		} else {
220		try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
221		Package.Fingerprint.generate(rng, p.name).int(),
222		});
223		}
224		}
225
226		if (!have_version) {
227		try appendError(p, main_token, "missing top-level 'version' field", .{});
228		}
229
230		if (!have_included_paths) {
231		if (p.allow_missing_paths_field) {
232		try p.paths.put(p.gpa, "", {});
233		} else {
234		try appendError(p, main_token, "missing top-level 'paths' field", .{});
235		}
236		}
237		}
238
239		fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
240		const ast = p.ast;
241
242		var buf: [2]Ast.Node.Index = undefined;
243		const struct_init = ast.fullStructInit(&buf, node) orelse {
244		const tok = ast.nodeMainToken(node);
245		return fail(p, tok, "expected dependencies expression to be a struct", .{});
246		};
247
248		for (struct_init.ast.fields) \|field_init\| {
249		const name_token = ast.firstToken(field_init) - 2;
250		const dep_name = try identifierTokenString(p, name_token);
251		const dep = try parseDependency(p, field_init);
252		try p.dependencies.put(p.gpa, dep_name, dep);
253		}
254		}
255
256		fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
257		const ast = p.ast;
258
259		var buf: [2]Ast.Node.Index = undefined;
260		const struct_init = ast.fullStructInit(&buf, node) orelse {
261		const tok = ast.nodeMainToken(node);
262		return fail(p, tok, "expected dependency expression to be a struct", .{});
263		};
264
265		var dep: Dependency = .{
266		.location = undefined,
267		.location_tok = undefined,
268		.location_node = undefined,
269		.hash = null,
270		.hash_tok = .none,
271		.hash_node = .none,
272		.node = node,
273		.name_tok = undefined,
274		.lazy = false,
275		};
276		var has_location = false;
277
278		for (struct_init.ast.fields) \|field_init\| {
279		const name_token = ast.firstToken(field_init) - 2;
280		dep.name_tok = name_token;
281		const field_name = try identifierTokenString(p, name_token);
282		// We could get fancy with reflection and comptime logic here but doing
283		// things manually provides an opportunity to do any additional verification
284		// that is desirable on a per-field basis.
285		if (mem.eql(u8, field_name, "url")) {
286		if (has_location) {
287		return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
288		}
289		dep.location = .{
290		.url = parseString(p, field_init) catch \|err\| switch (err) {
291		error.ParseFailure => continue,
292		else => \|e\| return e,
293		},
294		};
295		has_location = true;
296		dep.location_tok = ast.nodeMainToken(field_init);
297		dep.location_node = field_init;
298		} else if (mem.eql(u8, field_name, "path")) {
299		if (has_location) {
300		return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
301		}
302		dep.location = .{
303		.path = parseString(p, field_init) catch \|err\| switch (err) {
304		error.ParseFailure => continue,
305		else => \|e\| return e,
306		},
307		};
308		has_location = true;
309		dep.location_tok = ast.nodeMainToken(field_init);
310		dep.location_node = field_init;
311		} else if (mem.eql(u8, field_name, "hash")) {
312		dep.hash = parseHash(p, field_init) catch \|err\| switch (err) {
313		error.ParseFailure => continue,
314		else => \|e\| return e,
315		};
316		dep.hash_tok = .fromToken(ast.nodeMainToken(field_init));
317		dep.hash_node = field_init.toOptional();
318		} else if (mem.eql(u8, field_name, "lazy")) {
319		dep.lazy = parseBool(p, field_init) catch \|err\| switch (err) {
320		error.ParseFailure => continue,
321		else => \|e\| return e,
322		};
323		} else {
324		// Ignore unknown fields so that we can add fields in future zig
325		// versions without breaking older zig versions.
326		}
327		}
328
329		if (!has_location) {
330		try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{});
331		}
332
333		return dep;
334		}
335
336		fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
337		const ast = p.ast;
338
339		var buf: [2]Ast.Node.Index = undefined;
340		const array_init = ast.fullArrayInit(&buf, node) orelse {
341		const tok = ast.nodeMainToken(node);
342		return fail(p, tok, "expected paths expression to be a list of strings", .{});
343		};
344
345		for (array_init.ast.elements) \|elem_node\| {
346		const path_string = try parseString(p, elem_node);
347		// This is normalized so that it can be used in string comparisons
348		// against file system paths.
349		const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
350		try p.paths.put(p.gpa, normalized, {});
351		}
352		}
353
354		fn parseBool(p: *Parse, node: Ast.Node.Index) !bool {
355		const ast = p.ast;
356		if (ast.nodeTag(node) != .identifier) {
357		return fail(p, ast.nodeMainToken(node), "expected identifier", .{});
358		}
359		const ident_token = ast.nodeMainToken(node);
360		const token_bytes = ast.tokenSlice(ident_token);
361		if (mem.eql(u8, token_bytes, "true")) {
362		return true;
363		} else if (mem.eql(u8, token_bytes, "false")) {
364		return false;
365		} else {
366		return fail(p, ident_token, "expected boolean", .{});
367		}
368		}
369
370		fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint {
371		const ast = p.ast;
372		const main_token = ast.nodeMainToken(node);
373		if (ast.nodeTag(node) != .number_literal) {
374		return fail(p, main_token, "expected integer literal", .{});
375		}
376		const token_bytes = ast.tokenSlice(main_token);
377		const parsed = std.zig.parseNumberLiteral(token_bytes);
378		switch (parsed) {
379		.int => \|n\| return @bitCast(n),
380		.big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{
381		@tagName(parsed),
382		}),
383		.failure => \|err\| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
384		}
385		}
386
387		fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
388		const ast = p.ast;
389		const main_token = ast.nodeMainToken(node);
390
391		if (ast.nodeTag(node) != .enum_literal)
392		return fail(p, main_token, "expected enum literal", .{});
393
394		const ident_name = ast.tokenSlice(main_token);
395		if (mem.startsWith(u8, ident_name, "@"))
396		return fail(p, main_token, "name must be a valid bare zig identifier", .{});
397
398		if (ident_name.len > max_name_len)
399		return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{
400		std.zig.fmtId(ident_name), max_name_len,
401		});
402
403		return ident_name;
404		}
405
406		fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
407		const ast = p.ast;
408		if (ast.nodeTag(node) != .string_literal) {
409		return fail(p, ast.nodeMainToken(node), "expected string literal", .{});
410		}
411		const str_lit_token = ast.nodeMainToken(node);
412		const token_bytes = ast.tokenSlice(str_lit_token);
413		p.buf.clearRetainingCapacity();
414		try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
415		const duped = try p.arena.dupe(u8, p.buf.items);
416		return duped;
417		}
418
419		fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
420		const ast = p.ast;
421		const tok = ast.nodeMainToken(node);
422		const h = try parseString(p, node);
423		switch (Package.Hash.validate(h)) {
424		.ok => return h,
425		else => \|t\| return fail(p, tok, "invalid hash: {t}", .{t}),
426		}
427		}
428
429		/// TODO: try to DRY this with AstGen.identifierTokenString
430		fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
431		const ast = p.ast;
432		assert(ast.tokenTag(token) == .identifier);
433		const ident_name = ast.tokenSlice(token);
434		if (!mem.startsWith(u8, ident_name, "@")) {
435		return ident_name;
436		}
437		p.buf.clearRetainingCapacity();
438		try parseStrLit(p, token, &p.buf, ident_name, 1);
439		const duped = try p.arena.dupe(u8, p.buf.items);
440		return duped;
441		}
442
443		/// TODO: try to DRY this with AstGen.parseStrLit
444		fn parseStrLit(
445		p: *Parse,
446		token: Ast.TokenIndex,
447		buf: *std.ArrayList(u8),
448		bytes: []const u8,
449		offset: u32,
450		) InnerError!void {
451		const raw_string = bytes[offset..];
452		const result = r: {
453		var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf);
454		defer buf.* = aw.toArrayList();
455		break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch \|err\| switch (err) {
456		error.WriteFailed => return error.OutOfMemory,
457		};
458		};
459		switch (result) {
460		.success => {},
461		.failure => \|err\| try p.appendStrLitError(err, token, bytes, offset),
462		}
463		}
464
465		/// TODO: try to DRY this with AstGen.failWithStrLitError
466		fn appendStrLitError(
467		p: *Parse,
468		err: std.zig.string_literal.Error,
469		token: Ast.TokenIndex,
470		bytes: []const u8,
471		offset: u32,
472		) Allocator.Error!void {
473		const raw_string = bytes[offset..];
474		switch (err) {
475		.invalid_escape_character => \|bad_index\| {
476		try p.appendErrorOff(
477		token,
478		offset + @as(u32, @intCast(bad_index)),
479		"invalid escape character: '{c}'",
480		.{raw_string[bad_index]},
481		);
482		},
483		.expected_hex_digit => \|bad_index\| {
484		try p.appendErrorOff(
485		token,
486		offset + @as(u32, @intCast(bad_index)),
487		"expected hex digit, found '{c}'",
488		.{raw_string[bad_index]},
489		);
490		},
491		.empty_unicode_escape_sequence => \|bad_index\| {
492		try p.appendErrorOff(
493		token,
494		offset + @as(u32, @intCast(bad_index)),
495		"empty unicode escape sequence",
496		.{},
497		);
498		},
499		.expected_hex_digit_or_rbrace => \|bad_index\| {
500		try p.appendErrorOff(
501		token,
502		offset + @as(u32, @intCast(bad_index)),
503		"expected hex digit or '}}', found '{c}'",
504		.{raw_string[bad_index]},
505		);
506		},
507		.invalid_unicode_codepoint => \|bad_index\| {
508		try p.appendErrorOff(
509		token,
510		offset + @as(u32, @intCast(bad_index)),
511		"unicode escape does not correspond to a valid unicode scalar value",
512		.{},
513		);
514		},
515		.expected_lbrace => \|bad_index\| {
516		try p.appendErrorOff(
517		token,
518		offset + @as(u32, @intCast(bad_index)),
519		"expected '{{', found '{c}",
520		.{raw_string[bad_index]},
521		);
522		},
523		.expected_rbrace => \|bad_index\| {
524		try p.appendErrorOff(
525		token,
526		offset + @as(u32, @intCast(bad_index)),
527		"expected '}}', found '{c}",
528		.{raw_string[bad_index]},
529		);
530		},
531		.expected_single_quote => \|bad_index\| {
532		try p.appendErrorOff(
533		token,
534		offset + @as(u32, @intCast(bad_index)),
535		"expected single quote ('), found '{c}",
536		.{raw_string[bad_index]},
537		);
538		},
539		.invalid_character => \|bad_index\| {
540		try p.appendErrorOff(
541		token,
542		offset + @as(u32, @intCast(bad_index)),
543		"invalid byte in string or character literal: '{c}'",
544		.{raw_string[bad_index]},
545		);
546		},
547		.empty_char_literal => {
548		try p.appendErrorOff(token, offset, "empty character literal", .{});
549		},
550		}
551		}
552
553		fn fail(
554		p: *Parse,
555		tok: Ast.TokenIndex,
556		comptime fmt: []const u8,
557		args: anytype,
558		) InnerError {
559		try appendError(p, tok, fmt, args);
560		return error.ParseFailure;
561		}
562
563		fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
564		return appendErrorOff(p, tok, 0, fmt, args);
565		}
566
567		fn appendErrorOff(
568		p: *Parse,
569		tok: Ast.TokenIndex,
570		byte_offset: u32,
571		comptime fmt: []const u8,
572		args: anytype,
573		) Allocator.Error!void {
574		try p.errors.append(p.gpa, .{
575		.msg = try std.fmt.allocPrint(p.arena, fmt, args),
576		.tok = tok,
577		.off = byte_offset,
578		});
579		}
580		};
581
582		pub fn load(
583		io: Io,
584		arena: Allocator,
585		manifest_path: std.Build.Cache.Path,
586		ast: *std.zig.Ast,
587		error_bundle: *std.zig.ErrorBundle.Wip,
588		manifest: *Manifest,
589		allow_missing_paths_field: bool,
590		) !void {
591		const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions(
592		io,
593		manifest_path.sub_path,
594		arena,
595		.limited(max_bytes),
596		.@"1",
597		0,
598		);
599
600		ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
601
602		if (ast.errors.len > 0) {
603		const file_path = try manifest_path.joinString(arena, "");
604		try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle);
605		return error.ErrorsBundled;
606		}
607
608		const rng: std.Random.IoSource = .{ .io = io };
609
610		manifest.* = try parse(arena, ast, rng.interface(), .{
611		.allow_missing_paths_field = allow_missing_paths_field,
612		});
613
614		if (manifest.errors.len > 0) {
615		const src_path = try error_bundle.printString("{f}", .{manifest_path});
616		try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle);
617		return error.ErrorsBundled;
618		}
619		}
620
621		test "basic" {
622		const gpa = testing.allocator;
623
624		const example =
625		\\.{
626		\\ .name = .foo,
627		\\ .fingerprint = 0x8c736521490b23df,
628		\\ .version = "3.2.1",
629		\\ .paths = .{""},
630		\\ .dependencies = .{
631		\\ .bar = .{
632		\\ .url = "https://example.com/baz.tar.gz",
633		\\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
634		\\ },
635		\\ },
636		\\}
637		;
638
639		var ast = try Ast.parse(gpa, example, .zon);
640		defer ast.deinit(gpa);
641
642		try testing.expect(ast.errors.len == 0);
643
644		var rng = std.Random.DefaultPrng.init(0);
645
646		var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
647		defer manifest.deinit(gpa);
648
649		try testing.expect(manifest.errors.len == 0);
650		try testing.expectEqualStrings("foo", manifest.name);
651
652		try testing.expectEqual(@as(std.SemanticVersion, .{
653		.major = 3,
654		.minor = 2,
655		.patch = 1,
656		}), manifest.version);
657
658		try testing.expect(manifest.dependencies.count() == 1);
659		try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
660		try testing.expectEqualStrings(
661		"https://example.com/baz.tar.gz",
662		manifest.dependencies.values()[0].location.url,
663		);
664		try testing.expectEqualStrings(
665		"libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
666		manifest.dependencies.values()[0].hash orelse return error.TestFailed,
667		);
668
669		try testing.expect(manifest.minimum_zig_version == null);
670		}
671
672		test "minimum_zig_version" {
673		const gpa = testing.allocator;
674
675		const example =
676		\\.{
677		\\ .name = .foo,
678		\\ .fingerprint = 0x8c736521490b23df,
679		\\ .version = "3.2.1",
680		\\ .paths = .{""},
681		\\ .minimum_zig_version = "0.11.1",
682		\\}
683		;
684
685		var ast = try Ast.parse(gpa, example, .zon);
686		defer ast.deinit(gpa);
687
688		try testing.expect(ast.errors.len == 0);
689
690		var rng = std.Random.DefaultPrng.init(0);
691
692		var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
693		defer manifest.deinit(gpa);
694
695		try testing.expect(manifest.errors.len == 0);
696		try testing.expect(manifest.dependencies.count() == 0);
697
698		try testing.expect(manifest.minimum_zig_version != null);
699
700		try testing.expectEqual(@as(std.SemanticVersion, .{
701		.major = 0,
702		.minor = 11,
703		.patch = 1,
704		}), manifest.minimum_zig_version.?);
705		}
706
707		test "minimum_zig_version - invalid version" {
708		const gpa = testing.allocator;
709
710		const example =
711		\\.{
712		\\ .name = .foo,
713		\\ .fingerprint = 0x8c736521490b23df,
714		\\ .version = "3.2.1",
715		\\ .minimum_zig_version = "X.11.1",
716		\\ .paths = .{""},
717		\\}
718		;
719
720		var ast = try Ast.parse(gpa, example, .zon);
721		defer ast.deinit(gpa);
722
723		try testing.expect(ast.errors.len == 0);
724
725		var rng = std.Random.DefaultPrng.init(0);
726
727		var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
728		defer manifest.deinit(gpa);
729
730		try testing.expect(manifest.errors.len == 1);
731		try testing.expect(manifest.dependencies.count() == 0);
732
733		try testing.expect(manifest.minimum_zig_version == null);
734		}

src/Package/Module.zig deleted-529

...	...	@@ -1,529 +0,0 @@
1		//! Corresponds to something that Zig source code can `@import`.
2
3		/// The root directory of the module. Only files inside this directory can be imported.
4		root: Compilation.Path,
5		/// Path to the root source file of this module. Relative to `root`. May contain path separators.
6		root_src_path: []const u8,
7		/// Name used in compile errors. Looks like "root.foo.bar".
8		fully_qualified_name: []const u8,
9		/// The dependency table of this module. The shared dependencies 'std' and
10		/// 'root' are not specified in every module dependency table, but are stored
11		/// separately in `Zcu`. 'builtin' is also not stored here, although it is
12		/// not necessarily the same between all modules. Handling of `@import` in
13		/// the rest of the compiler must detect these special names and use the
14		/// correct module instead of consulting `deps`.
15		deps: Deps = .{},
16
17		resolved_target: ResolvedTarget,
18		optimize_mode: std.lang.OptimizeMode,
19		code_model: std.lang.CodeModel,
20		single_threaded: bool,
21		error_tracing: bool,
22		valgrind: bool,
23		pic: bool,
24		strip: bool,
25		omit_frame_pointer: bool,
26		stack_check: bool,
27		stack_protector: u32,
28		red_zone: bool,
29		sanitize_c: std.zig.SanitizeC,
30		sanitize_thread: bool,
31		fuzz: bool,
32		unwind_tables: std.lang.UnwindTables,
33		cc_argv: []const []const u8,
34		/// (SPIR-V) whether to generate a structured control flow graph or not
35		structured_cfg: bool,
36		no_builtin: bool,
37
38		pub const Deps = std.array_hash_map.String(*Module);
39
40		pub const Tree = struct {
41		/// Each `Package` exposes a `Module` with build.zig as its root source file.
42		build_module_table: std.array_hash_map.Auto(MultiHashHexDigest, *Module),
43		};
44
45		pub const CreateOptions = struct {
46		paths: Paths,
47		fully_qualified_name: []const u8,
48
49		cc_argv: []const []const u8,
50		inherited: Inherited,
51		global: Compilation.Config,
52		/// If this is null then `resolved_target` must be non-null.
53		parent: ?*Package.Module,
54
55		pub const Paths = struct {
56		root: Compilation.Path,
57		/// Relative to `root`. May contain path separators.
58		root_src_path: []const u8,
59		};
60
61		pub const Inherited = struct {
62		/// If this is null then `parent` must be non-null.
63		resolved_target: ?ResolvedTarget = null,
64		optimize_mode: ?std.lang.OptimizeMode = null,
65		code_model: ?std.lang.CodeModel = null,
66		single_threaded: ?bool = null,
67		error_tracing: ?bool = null,
68		valgrind: ?bool = null,
69		pic: ?bool = null,
70		strip: ?bool = null,
71		omit_frame_pointer: ?bool = null,
72		stack_check: ?bool = null,
73		/// null means default.
74		/// 0 means no stack protector.
75		/// other number means stack protection with that buffer size.
76		stack_protector: ?u32 = null,
77		red_zone: ?bool = null,
78		unwind_tables: ?std.lang.UnwindTables = null,
79		sanitize_c: ?std.zig.SanitizeC = null,
80		sanitize_thread: ?bool = null,
81		fuzz: ?bool = null,
82		structured_cfg: ?bool = null,
83		no_builtin: ?bool = null,
84		};
85		};
86
87		pub const ResolvedTarget = struct {
88		result: std.Target,
89		is_native_os: bool,
90		is_native_abi: bool,
91		is_explicit_dynamic_linker: bool,
92		llvm_cpu_features: ?[*:0]const u8 = null,
93		};
94
95		pub const CreateError = error{
96		OutOfMemory,
97		ValgrindUnsupportedOnTarget,
98		TargetRequiresSingleThreaded,
99		BackendRequiresSingleThreaded,
100		TargetRequiresPic,
101		PieRequiresPic,
102		DynamicLinkingRequiresPic,
103		TargetHasNoRedZone,
104		StackCheckUnsupportedByTarget,
105		StackProtectorUnsupportedByTarget,
106		StackProtectorUnavailableWithoutLibC,
107		};
108
109		/// At least one of `parent` and `resolved_target` must be non-null.
110		pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module {
111		if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread);
112		if (options.inherited.fuzz == true) assert(options.global.any_fuzz);
113		if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded);
114		if (options.inherited.unwind_tables) \|uwt\| if (uwt != .none) assert(options.global.any_unwind_tables);
115		if (options.inherited.sanitize_c) \|sc\| if (sc != .off) assert(options.global.any_sanitize_c != .off);
116		if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing);
117
118		const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target;
119		const target = &resolved_target.result;
120
121		const optimize_mode = options.inherited.optimize_mode orelse
122		if (options.parent) \|p\| p.optimize_mode else options.global.root_optimize_mode;
123
124		const strip = b: {
125		if (options.inherited.strip) \|x\| break :b x;
126		if (options.parent) \|p\| break :b p.strip;
127		break :b options.global.root_strip;
128		};
129
130		const zig_backend = target_util.zigBackend(target, options.global.use_llvm);
131
132		const valgrind = b: {
133		if (!target_util.hasValgrindSupport(target, zig_backend)) {
134		if (options.inherited.valgrind == true)
135		return error.ValgrindUnsupportedOnTarget;
136		break :b false;
137		}
138		if (options.inherited.valgrind) \|x\| break :b x;
139		if (options.parent) \|p\| break :b p.valgrind;
140		if (strip) break :b false;
141		break :b optimize_mode == .Debug;
142		};
143
144		const single_threaded = b: {
145		if (target_util.alwaysSingleThreaded(target)) {
146		if (options.inherited.single_threaded == false)
147		return error.TargetRequiresSingleThreaded;
148		break :b true;
149		}
150
151		if (options.global.have_zcu) {
152		if (!target_util.supportsThreads(target, zig_backend)) {
153		if (options.inherited.single_threaded == false)
154		return error.BackendRequiresSingleThreaded;
155		break :b true;
156		}
157		}
158
159		if (options.inherited.single_threaded) \|x\| break :b x;
160		if (options.parent) \|p\| break :b p.single_threaded;
161		break :b target_util.defaultSingleThreaded(target);
162		};
163
164		const error_tracing = b: {
165		if (options.inherited.error_tracing) \|x\| break :b x;
166		if (options.parent) \|p\| break :b p.error_tracing;
167		break :b options.global.root_error_tracing;
168		};
169
170		const pic = b: {
171		if (target_util.requiresPic(target, options.global.link_libc)) {
172		if (options.inherited.pic == false)
173		return error.TargetRequiresPic;
174		break :b true;
175		}
176		if (options.global.pie) {
177		if (options.inherited.pic == false)
178		return error.PieRequiresPic;
179		break :b true;
180		}
181		if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) {
182		if (options.inherited.pic == false)
183		return error.DynamicLinkingRequiresPic;
184		break :b true;
185		}
186		if (options.inherited.pic) \|x\| break :b x;
187		if (options.parent) \|p\| break :b p.pic;
188
189		// Default to PIC on targets where we default to producing PIEs to make
190		// the common case of linking objects and static libraries into an
191		// executable work out of the box.
192		break :b target_util.defaultPie(target);
193		};
194
195		const red_zone = b: {
196		if (!target_util.hasRedZone(target)) {
197		if (options.inherited.red_zone == true)
198		return error.TargetHasNoRedZone;
199		break :b false;
200		}
201		if (options.inherited.red_zone) \|x\| break :b x;
202		if (options.parent) \|p\| break :b p.red_zone;
203		break :b true;
204		};
205
206		const omit_frame_pointer = b: {
207		if (options.inherited.omit_frame_pointer) \|x\| break :b x;
208		if (options.parent) \|p\| break :b p.omit_frame_pointer;
209		if (optimize_mode == .ReleaseSmall) {
210		// On x86, in most cases, keeping the frame pointer usually results in smaller binary size.
211		// This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer)
212		// are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer).
213		break :b !target.cpu.arch.isX86();
214		}
215		break :b false;
216		};
217
218		const sanitize_thread = b: {
219		if (options.inherited.sanitize_thread) \|x\| break :b x;
220		if (options.parent) \|p\| break :b p.sanitize_thread;
221		break :b false;
222		};
223
224		const unwind_tables = b: {
225		if (options.inherited.unwind_tables) \|x\| break :b x;
226		if (options.parent) \|p\| break :b p.unwind_tables;
227
228		break :b target_util.defaultUnwindTables(
229		target,
230		options.global.link_libunwind,
231		sanitize_thread or options.global.any_sanitize_thread,
232		);
233		};
234
235		const fuzz = b: {
236		if (options.inherited.fuzz) \|x\| break :b x;
237		if (options.parent) \|p\| break :b p.fuzz;
238		break :b false;
239		};
240
241		const code_model: std.lang.CodeModel = b: {
242		if (options.inherited.code_model) \|x\| break :b x;
243		if (options.parent) \|p\| break :b p.code_model;
244		break :b .default;
245		};
246
247		const is_safe_mode = switch (optimize_mode) {
248		.Debug, .ReleaseSafe => true,
249		.ReleaseFast, .ReleaseSmall => false,
250		};
251
252		const sanitize_c: std.zig.SanitizeC = b: {
253		if (options.inherited.sanitize_c) \|x\| break :b x;
254		if (options.parent) \|p\| break :b p.sanitize_c;
255		break :b switch (optimize_mode) {
256		.Debug => .full,
257		// It's recommended to use the minimal runtime in production
258		// environments due to the security implications of the full runtime.
259		// The minimal runtime doesn't provide much benefit over simply
260		// trapping, however, so we do that instead.
261		.ReleaseSafe => .trap,
262		.ReleaseFast, .ReleaseSmall => .off,
263		};
264		};
265
266		const stack_check = b: {
267		if (!target_util.supportsStackProbing(target, zig_backend)) {
268		if (options.inherited.stack_check == true)
269		return error.StackCheckUnsupportedByTarget;
270		break :b false;
271		}
272		if (options.inherited.stack_check) \|x\| break :b x;
273		if (options.parent) \|p\| break :b p.stack_check;
274		break :b is_safe_mode;
275		};
276
277		const stack_protector: u32 = sp: {
278		const use_zig_backend = options.global.have_zcu or
279		(options.global.any_c_source_files and options.global.c_frontend == .aro);
280		if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) {
281		if (options.inherited.stack_protector) \|x\| {
282		if (x > 0) return error.StackProtectorUnsupportedByTarget;
283		}
284		break :sp 0;
285		}
286
287		if (options.global.any_c_source_files and options.global.c_frontend == .clang and
288		!target_util.clangSupportsStackProtector(target))
289		{
290		if (options.inherited.stack_protector) \|x\| {
291		if (x > 0) return error.StackProtectorUnsupportedByTarget;
292		}
293		break :sp 0;
294		}
295
296		// This logic is checking for linking libc because otherwise our start code
297		// which is trying to set up TLS (i.e. the fs/gs registers) but the stack
298		// protection code depends on fs/gs registers being already set up.
299		// If we were able to annotate start code, or perhaps the entire std lib,
300		// as being exempt from stack protection checks, we could change this logic
301		// to supporting stack protection even when not linking libc.
302		// TODO file issue about this
303		if (!options.global.link_libc) {
304		if (options.inherited.stack_protector) \|x\| {
305		if (x > 0) return error.StackProtectorUnavailableWithoutLibC;
306		}
307		break :sp 0;
308		}
309
310		if (options.inherited.stack_protector) \|x\| break :sp x;
311		if (options.parent) \|p\| break :sp p.stack_protector;
312		if (!is_safe_mode) break :sp 0;
313
314		break :sp target_util.default_stack_protector_buffer_size;
315		};
316
317		const structured_cfg = b: {
318		if (options.inherited.structured_cfg) \|x\| break :b x;
319		if (options.parent) \|p\| break :b p.structured_cfg;
320		// We always want a structured control flow in shaders. This option is
321		// only relevant for OpenCL kernels.
322		break :b switch (target.os.tag) {
323		.opencl => false,
324		else => true,
325		};
326		};
327
328		const no_builtin = b: {
329		if (options.inherited.no_builtin) \|x\| break :b x;
330		if (options.parent) \|p\| break :b p.no_builtin;
331
332		break :b target.cpu.arch.isBpf();
333		};
334
335		const llvm_cpu_features: ?[*:0]const u8 = b: {
336		if (resolved_target.llvm_cpu_features) \|x\| break :b x;
337		if (!options.global.use_llvm) break :b null;
338
339		var buf = std.array_list.Managed(u8).init(arena);
340		var disabled_features = std.array_list.Managed(u8).init(arena);
341		defer disabled_features.deinit();
342
343		// Append disabled features after enabled ones, so that their effects aren't overwritten.
344		for (target.cpu.arch.allFeaturesList()) \|feature\| {
345		if (feature.llvm_name) \|llvm_name\| {
346		// Ignore these until we figure out how to handle the concept of omitting features.
347		// See https://github.com/ziglang/zig/issues/23539
348		if (target_util.isDynamicAMDGCNFeature(target, feature)) continue;
349
350		if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue;
351		if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue;
352
353		var is_enabled = target.cpu.features.isEnabled(feature.index);
354		if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) {
355		is_enabled = !omit_frame_pointer;
356		}
357
358		if (is_enabled) {
359		try buf.ensureUnusedCapacity(2 + llvm_name.len);
360		buf.appendAssumeCapacity('+');
361		buf.appendSliceAssumeCapacity(llvm_name);
362		buf.appendAssumeCapacity(',');
363		} else {
364		try disabled_features.ensureUnusedCapacity(2 + llvm_name.len);
365		disabled_features.appendAssumeCapacity('-');
366		disabled_features.appendSliceAssumeCapacity(llvm_name);
367		disabled_features.appendAssumeCapacity(',');
368		}
369		}
370		}
371
372		try buf.appendSlice(disabled_features.items);
373		if (buf.items.len == 0) break :b "";
374		assert(std.mem.endsWith(u8, buf.items, ","));
375		buf.items[buf.items.len - 1] = 0;
376		buf.shrinkAndFree(buf.items.len);
377		break :b buf.items[0 .. buf.items.len - 1 :0].ptr;
378		};
379
380		const mod = try arena.create(Module);
381		mod.* = .{
382		.root = options.paths.root,
383		.root_src_path = options.paths.root_src_path,
384		.fully_qualified_name = options.fully_qualified_name,
385		.resolved_target = .{
386		.result = target.*,
387		.is_native_os = resolved_target.is_native_os,
388		.is_native_abi = resolved_target.is_native_abi,
389		.is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker,
390		.llvm_cpu_features = llvm_cpu_features,
391		},
392		.optimize_mode = optimize_mode,
393		.single_threaded = single_threaded,
394		.error_tracing = error_tracing,
395		.valgrind = valgrind,
396		.pic = pic,
397		.strip = strip,
398		.omit_frame_pointer = omit_frame_pointer,
399		.stack_check = stack_check,
400		.stack_protector = stack_protector,
401		.code_model = code_model,
402		.red_zone = red_zone,
403		.sanitize_c = sanitize_c,
404		.sanitize_thread = sanitize_thread,
405		.fuzz = fuzz,
406		.unwind_tables = unwind_tables,
407		.cc_argv = options.cc_argv,
408		.structured_cfg = structured_cfg,
409		.no_builtin = no_builtin,
410		};
411		return mod;
412		}
413
414		/// All fields correspond to `CreateOptions`.
415		pub const LimitedOptions = struct {
416		root: Compilation.Path,
417		root_src_path: []const u8,
418		fully_qualified_name: []const u8,
419		};
420
421		/// This one can only be used if the Module will only be used for AstGen and earlier in
422		/// the pipeline. Illegal behavior occurs if a limited module touches Sema.
423		pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module {
424		const mod = try gpa.create(Module);
425		mod.* = .{
426		.root = options.root,
427		.root_src_path = options.root_src_path,
428		.fully_qualified_name = options.fully_qualified_name,
429
430		.resolved_target = undefined,
431		.optimize_mode = undefined,
432		.code_model = undefined,
433		.single_threaded = undefined,
434		.error_tracing = undefined,
435		.valgrind = undefined,
436		.pic = undefined,
437		.strip = undefined,
438		.omit_frame_pointer = undefined,
439		.stack_check = undefined,
440		.stack_protector = undefined,
441		.red_zone = undefined,
442		.sanitize_c = undefined,
443		.sanitize_thread = undefined,
444		.fuzz = undefined,
445		.unwind_tables = undefined,
446		.cc_argv = undefined,
447		.structured_cfg = undefined,
448		.no_builtin = undefined,
449		};
450		return mod;
451		}
452
453		/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task.
454		pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: Compilation.Directories) Allocator.Error!*Module {
455		const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash());
456		const new = try arena.create(Module);
457		new.* = .{
458		.root = try .fromRoot(arena, dirs, .global_cache, sub_path),
459		.root_src_path = "builtin.zig",
460		.fully_qualified_name = "builtin",
461		.resolved_target = .{
462		.result = opts.target,
463		// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
464		.is_native_os = false,
465		.is_native_abi = false,
466		.is_explicit_dynamic_linker = false,
467		.llvm_cpu_features = null,
468		},
469		.optimize_mode = opts.optimize_mode,
470		.single_threaded = opts.single_threaded,
471		.error_tracing = opts.error_tracing,
472		.valgrind = opts.valgrind,
473		.pic = opts.pic,
474		.strip = opts.strip,
475		.omit_frame_pointer = opts.omit_frame_pointer,
476		.code_model = opts.code_model,
477		.sanitize_thread = opts.sanitize_thread,
478		.fuzz = opts.fuzz,
479		.unwind_tables = opts.unwind_tables,
480		.cc_argv = &.{},
481		// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
482		.stack_check = false,
483		.stack_protector = 0,
484		.red_zone = false,
485		.sanitize_c = .off,
486		.structured_cfg = false,
487		.no_builtin = false,
488		};
489		return new;
490		}
491
492		/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module.
493		pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin {
494		assert(global.have_zcu);
495		return .{
496		.target = m.resolved_target.result,
497		.zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm),
498		.output_mode = global.output_mode,
499		.link_mode = global.link_mode,
500		.unwind_tables = m.unwind_tables,
501		.is_test = global.is_test,
502		.single_threaded = m.single_threaded,
503		.link_libc = global.link_libc,
504		.link_libcpp = global.link_libcpp,
505		.optimize_mode = m.optimize_mode,
506		.error_tracing = m.error_tracing,
507		.valgrind = m.valgrind,
508		.sanitize_thread = m.sanitize_thread,
509		.fuzz = m.fuzz,
510		.pic = m.pic,
511		.pie = global.pie,
512		.strip = m.strip,
513		.code_model = m.code_model,
514		.omit_frame_pointer = m.omit_frame_pointer,
515		.wasi_exec_model = global.wasi_exec_model,
516		};
517		}
518
519		const Module = @This();
520		const Package = @import("../Package.zig");
521		const std = @import("std");
522		const Allocator = std.mem.Allocator;
523		const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest;
524		const target_util = @import("../target.zig");
525		const Cache = std.Build.Cache;
526		const Builtin = @import("../Builtin.zig");
527		const assert = std.debug.assert;
528		const Compilation = @import("../Compilation.zig");
529		const File = @import("../Zcu.zig").File;

src/Sema.zig+2-2

...	...	@@ -25,7 +25,6 @@ const SemaError = Zcu.SemaError;
25	25	const LazySrcLoc = Zcu.LazySrcLoc;
26	26	const RangeSet = @import("RangeSet.zig");
27	27	const target_util = @import("target.zig");
28		const Package = @import("Package.zig");
29	28	const crash_report = @import("crash_report.zig");
30	29	const build_options = @import("build_options");
31	30	const Compilation = @import("Compilation.zig");
...	...	@@ -36,6 +35,7 @@ const ComptimeAllocIndex = InternPool.ComptimeAllocIndex;
36	35	const Cache = std.Build.Cache;
37	36	const LowerZon = @import("Sema/LowerZon.zig");
38	37	const arith = @import("Sema/arith.zig");
	38	const Module = @import("Module.zig");
39	39
40	40	pt: Zcu.PerThread,
41	41	/// Alias to `zcu.gpa`.
...	...	@@ -839,7 +839,7 @@ pub const Block = struct {
839	839	return result_index;
840	840	}
841	841
842		pub fn ownerModule(block: Block) *Package.Module {
	842	pub fn ownerModule(block: Block) *Module {
843	843	const zcu = block.sema.pt.zcu;
844	844	return zcu.namespacePtr(block.namespace).fileScope(zcu).mod.?;
845	845	}

src/Zcu.zig+15-16

...	...	@@ -25,7 +25,7 @@ const Compilation = @import("Compilation.zig");
25	25	const Cache = std.Build.Cache;
26	26	pub const Value = @import("Value.zig");
27	27	pub const Type = @import("Type.zig");
28		const Package = @import("Package.zig");
	28	const Module = @import("Module.zig");
29	29	const link = @import("link.zig");
30	30	const Air = @import("Air.zig");
31	31	const Zir = std.zig.Zir;
...	...	@@ -34,7 +34,6 @@ const AstGen = std.zig.AstGen;
34	34	const Sema = @import("Sema.zig");
35	35	const target_util = @import("target.zig");
36	36	const build_options = @import("build_options");
37		const isUpDir = @import("introspect.zig").isUpDir;
38	37	const InternPool = @import("InternPool.zig");
39	38	const Alignment = InternPool.Alignment;
40	39	const AnalUnit = InternPool.AnalUnit;
...	...	@@ -65,11 +64,11 @@ comp: *Compilation,
65	64	llvm_object: ?LlvmObject.Ptr,
66	65
67	66	/// Pointer to externally managed resource.
68		root_mod: *Package.Module,
	67	root_mod: *Module,
69	68	/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which
70	69	/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests.
71		main_mod: *Package.Module,
72		std_mod: *Package.Module,
	70	main_mod: *Module,
	71	std_mod: *Module,
73	72	sema_prog_node: std.Progress.Node = .none,
74	73	codegen_prog_node: std.Progress.Node = .none,
75	74	/// The number of codegen jobs which are pending or in-progress. Whichever thread drops this value
...	...	@@ -106,11 +105,11 @@ multi_exports: std.array_hash_map.Auto(AnalUnit, extern struct {
106	105	}) = .{},
107	106
108	107	/// Key is the digest returned by `Builtin.hash`; value is the corresponding module.
109		builtin_modules: std.array_hash_map.Auto(Cache.BinDigest, *Package.Module) = .empty,
	108	builtin_modules: std.array_hash_map.Auto(Cache.BinDigest, *Module) = .empty,
110	109
111	110	/// Populated as soon as the `Compilation` is created. Guaranteed to contain all modules, even builtin ones.
112	111	/// Modules whose root file is not a Zig or ZON file have the value `.none`.
113		module_roots: std.array_hash_map.Auto(*Package.Module, File.Index.Optional) = .empty,
	112	module_roots: std.array_hash_map.Auto(*Module, File.Index.Optional) = .empty,
114	113
115	114	/// The set of all the Zig source files in the Zig Compilation Unit. Tracked in
116	115	/// order to iterate over it and check which source files have been modified on
...	...	@@ -149,7 +148,7 @@ alive_files: std.array_hash_map.Auto(File.Index, File.Reference) = .empty,
149	148	/// Cleared and recomputed every update, after AstGen and before Sema.
150	149	multi_module_err: ?struct {
151	150	file: File.Index,
152		modules: [2]*Package.Module,
	151	modules: [2]*Module,
153	152	refs: [2]File.Reference,
154	153	} = null,
155	154
...	...	@@ -293,7 +292,7 @@ retryable_failures: std.ArrayList(AnalUnit) = .empty,
293	292
294	293	/// These are the modules which we initially queue for analysis in `Compilation.update`.
295	294	/// `resolveReferences` will use these as the root of its reachability traversal.
296		analysis_roots_buffer: [5]*Package.Module,
	295	analysis_roots_buffer: [5]*Module,
297	296	analysis_roots_len: usize = 0,
298	297	/// This is the cached result of `Zcu.resolveReferences`. It is computed on-demand, and
299	298	/// reset to `null` when any semantic analysis occurs (since this invalidates the data).
...	...	@@ -986,7 +985,7 @@ pub const File = struct {
986	985	/// tell, and invalidate dependencies as needed (see `module_changed`).
987	986	/// During semantic analysis, this is always non-`null` for alive files (i.e. those which
988	987	/// have imports targeting them).
989		mod: ?*Package.Module,
	988	mod: ?*Module,
990	989	/// Relative to the root directory of `mod`. If `mod == null`, this field is `undefined`.
991	990	/// This memory is managed externally and must not be directly freed.
992	991	/// Its lifetime is at least equal to that of this `File`.
...	...	@@ -1029,13 +1028,13 @@ pub const File = struct {
1029	1028
1030	1029	/// A single reference to a file.
1031	1030	pub const Reference = union(enum) {
1032		analysis_root: *Package.Module,
	1031	analysis_root: *Module,
1033	1032	import: struct {
1034	1033	importer: Zcu.File.Index,
1035	1034	tok: Ast.TokenIndex,
1036	1035	/// If the file is imported as the root of a module, this is that module.
1037	1036	/// `null` means the file was imported directly by path.
1038		module: ?*Package.Module,
	1037	module: ?*Module,
1039	1038	},
1040	1039	};
1041	1040
...	...	@@ -3710,7 +3709,7 @@ pub const ImportResult = struct {
3710	3709	/// If this import was a simple file path, this is `null`; the imported file should exist within
3711	3710	/// the importer's module. Otherwise, it's the module which the import resolved to. This module
3712	3711	/// could match the module of `cur_file`, since a module can depend on itself.
3713		module: ?*Package.Module,
	3712	module: ?*Module,
3714	3713	};
3715	3714
3716	3715	/// Prepares `unit` for re-analysis by clearing all of the following state:
...	...	@@ -4407,7 +4406,7 @@ fn resolveReferencesInner(zcu: *Zcu) Allocator.Error!std.array_hash_map.Auto(Ana
4407	4406	return units.move();
4408	4407	}
4409	4408
4410		pub fn analysisRoots(zcu: Zcu) []Package.Module {
	4409	pub fn analysisRoots(zcu: Zcu) []Module {
4411	4410	return zcu.analysis_roots_buffer[0..zcu.analysis_roots_len];
4412	4411	}
4413	4412
...	...	@@ -4821,7 +4820,7 @@ fn explainWhyFileIsInModule(
4821	4820	eb: *std.zig.ErrorBundle.Wip,
4822	4821	notes_out: *std.ArrayList(std.zig.ErrorBundle.MessageIndex),
4823	4822	file: File.Index,
4824		in_module: *Package.Module,
	4823	in_module: *Module,
4825	4824	ref: File.Reference,
4826	4825	) Allocator.Error!void {
4827	4826	const gpa = zcu.gpa;
...	...	@@ -4867,7 +4866,7 @@ fn explainWhyFileIsInModule(
4867	4866	const import_src = try importer_file.errorBundleTokenSrc(import.tok, zcu, eb);
4868	4867
4869	4868	const importer_ref = zcu.alive_files.get(import.importer).?;
4870		const importer_root: ?*Package.Module = switch (importer_ref) {
	4869	const importer_root: ?*Module = switch (importer_ref) {
4871	4870	.analysis_root => \|mod\| mod,
4872	4871	.import => \|i\| i.module,
4873	4872	};

src/Zcu/PerThread.zig+1-2

...	...	@@ -23,8 +23,7 @@ const builtin = @import("builtin");
23	23	const dev = @import("../dev.zig");
24	24	const InternPool = @import("../InternPool.zig");
25	25	const AnalUnit = InternPool.AnalUnit;
26		const introspect = @import("../introspect.zig");
27		const Module = @import("../Package.zig").Module;
	26	const Module = @import("../Module.zig");
28	27	const Sema = @import("../Sema.zig");
29	28	const target_util = @import("../target.zig");
30	29	const tracy = @import("../tracy.zig");

src/codegen/aarch64/Select.zig+2-2

...	...	@@ -7592,7 +7592,7 @@ pub fn layout(
7592	7592	is_sysv_var_args: bool,
7593	7593	saved_gra_len: u7,
7594	7594	saved_vra_len: u7,
7595		mod: *const Package.Module,
	7595	mod: *const Module,
7596	7596	) !usize {
7597	7597	const zcu = isel.pt.zcu;
7598	7598	const ip = &zcu.intern_pool;
...	...	@@ -12513,7 +12513,7 @@ const assert = std.debug.assert;
12513	12513	const codegen = @import("../../codegen.zig");
12514	12514	const Constant = @import("../../Value.zig");
12515	12515	const InternPool = @import("../../InternPool.zig");
12516		const Package = @import("../../Package.zig");
	12516	const Module = @import("../../Module.zig");
12517	12517	const Register = codegen.aarch64.encoding.Register;
12518	12518	const Select = @This();
12519	12519	const std = @import("std");

src/codegen/c.zig+1-1

...	...	@@ -9,7 +9,7 @@ const Writer = std.Io.Writer;
9	9	const dev = @import("../dev.zig");
10	10	const link = @import("../link.zig");
11	11	const Zcu = @import("../Zcu.zig");
12		const Module = @import("../Package/Module.zig");
	12	const Module = @import("../Module.zig");
13	13	const Compilation = @import("../Compilation.zig");
14	14	const Value = @import("../Value.zig");
15	15	const Type = @import("../Type.zig");

src/codegen/llvm.zig+2-2

...	...	@@ -13,7 +13,7 @@ const Compilation = @import("../Compilation.zig");
13	13	const dev = @import("../dev.zig");
14	14	const InternPool = @import("../InternPool.zig");
15	15	const link = @import("../link.zig");
16		const Package = @import("../Package.zig");
	16	const Module = @import("../Module.zig");
17	17	const target_util = @import("../target.zig");
18	18	const Type = @import("../Type.zig");
19	19	const Value = @import("../Value.zig");
...	...	@@ -2772,7 +2772,7 @@ pub const Object = struct {
2772	2772	fn addCommonFnAttributes(
2773	2773	o: *Object,
2774	2774	attributes: *Builder.FunctionAttributes.Wip,
2775		owner_mod: *Package.Module,
	2775	owner_mod: *Module,
2776	2776	omit_frame_pointer: bool,
2777	2777	) Allocator.Error!void {
2778	2778	if (!owner_mod.red_zone) {

src/codegen/llvm/FuncGen.zig+2-2

...	...	@@ -80,7 +80,7 @@ fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError {
80	80	);
81	81	}
82	82
83		fn ownerModule(fg: const FuncGen) Package.Module {
	83	fn ownerModule(fg: const FuncGen) Module {
84	84	return fg.object.zcu.navFileScope(fg.nav_index).mod.?;
85	85	}
86	86
...	...	@@ -7739,7 +7739,7 @@ const mips_c_abi = @import("../mips/abi.zig");
7739	7739
7740	7740	const Zcu = @import("../../Zcu.zig");
7741	7741	const Air = @import("../../Air.zig");
7742		const Package = @import("../../Package.zig");
	7742	const Module = @import("../../Module.zig");
7743	7743	const InternPool = @import("../../InternPool.zig");
7744	7744	const Value = @import("../../Value.zig");
7745	7745	const Type = @import("../../Type.zig");

src/codegen/riscv64/CodeGen.zig+2-2

...	...	@@ -14,7 +14,7 @@ const Type = @import("../../Type.zig");
14	14	const Value = @import("../../Value.zig");
15	15	const link = @import("../../link.zig");
16	16	const Zcu = @import("../../Zcu.zig");
17		const Package = @import("../../Package.zig");
	17	const Module = @import("../../Module.zig");
18	18	const InternPool = @import("../../InternPool.zig");
19	19	const Compilation = @import("../../Compilation.zig");
20	20	const target_util = @import("../../target.zig");
...	...	@@ -66,7 +66,7 @@ liveness: Air.Liveness,
66	66	bin_file: *link.File,
67	67	gpa: Allocator,
68	68
69		mod: *Package.Module,
	69	mod: *Module,
70	70	target: *const std.Target,
71	71	args: []MCValue,
72	72	ret_mcv: InstTracking,

src/codegen/x86_64/CodeGen.zig+1-1

...	...	@@ -14,7 +14,7 @@ const Emit = @import("Emit.zig");
14	14	const Lower = @import("Lower.zig");
15	15	const Mir = @import("Mir.zig");
16	16	const Zcu = @import("../../Zcu.zig");
17		const Module = @import("../../Package/Module.zig");
	17	const Module = @import("../../Module.zig");
18	18	const InternPool = @import("../../InternPool.zig");
19	19	const Type = @import("../../Type.zig");
20	20	const Value = @import("../../Value.zig");

src/dev.zig+1-8

...	...	@@ -76,7 +76,6 @@ pub const Env = enum {
76	76	.test_command,
77	77	.run_command,
78	78	.ar_command,
79		.build_command,
80	79	.clang_command,
81	80	.stdio_listen,
82	81	.build_import_lib,
...	...	@@ -108,12 +107,11 @@ pub const Env = enum {
108	107	.wasm_linker,
109	108	.spirv_linker,
110	109	.plan9_linker,
	110	.jit_command,
111	111	=> true,
112	112	.cc_command,
113	113	.translate_c_command,
114	114	.fmt_command,
115		.jit_command,
116		.fetch_command,
117	115	.init_command,
118	116	.targets_command,
119	117	.version_command,
...	...	@@ -162,7 +160,6 @@ pub const Env = enum {
162	160	else => Env.ast_gen.supports(feature),
163	161	},
164	162	.@"aarch64-linux" => switch (feature) {
165		.build_command,
166	163	.stdio_listen,
167	164	.incremental,
168	165	.aarch64_backend,
...	...	@@ -179,7 +176,6 @@ pub const Env = enum {
179	176	else => Env.sema.supports(feature),
180	177	},
181	178	.@"powerpc-linux" => switch (feature) {
182		.build_command,
183	179	.stdio_listen,
184	180	.incremental,
185	181	.x86_64_backend,
...	...	@@ -210,7 +206,6 @@ pub const Env = enum {
210	206	else => Env.sema.supports(feature),
211	207	},
212	208	.@"x86_64-linux" => switch (feature) {
213		.build_command,
214	209	.stdio_listen,
215	210	.incremental,
216	211	.legalize,
...	...	@@ -251,13 +246,11 @@ pub const Feature = enum {
251	246	test_command,
252	247	run_command,
253	248	ar_command,
254		build_command,
255	249	clang_command,
256	250	cc_command,
257	251	translate_c_command,
258	252	fmt_command,
259	253	jit_command,
260		fetch_command,
261	254	init_command,
262	255	targets_command,
263	256	version_command,

src/introspect.zig deleted-220

...	...	@@ -1,220 +0,0 @@
1		const builtin = @import("builtin");
2
3		const std = @import("std");
4		const Io = std.Io;
5		const Dir = std.Io.Dir;
6		const mem = std.mem;
7		const Allocator = std.mem.Allocator;
8		const Cache = std.Build.Cache;
9		const assert = std.debug.assert;
10
11		const build_options = @import("build_options");
12
13		const Compilation = @import("Compilation.zig");
14		const Package = @import("Package.zig");
15
16		/// Returns the sub_path that worked, or `null` if none did.
17		/// The path of the returned Directory is relative to `base`.
18		/// The handle of the returned Directory is open.
19		fn testZigInstallPrefix(io: Io, base_dir: Io.Dir) ?Cache.Directory {
20		const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig";
21
22		zig_dir: {
23		// Try lib/zig/std/std.zig
24		const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig";
25		var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir;
26		const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
27		test_zig_dir.close(io);
28		break :zig_dir;
29		};
30		file.close(io);
31		return .{ .handle = test_zig_dir, .path = lib_zig };
32		}
33
34		// Try lib/std/std.zig
35		var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null;
36		const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
37		test_zig_dir.close(io);
38		return null;
39		};
40		file.close(io);
41		return .{ .handle = test_zig_dir, .path = "lib" };
42		}
43
44		/// Both the directory handle and the path are newly allocated resources which the caller now owns.
45		pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
46		const cwd_path = try getResolvedCwd(io, gpa);
47		defer gpa.free(cwd_path);
48		const self_exe_path = try std.process.executablePathAlloc(io, gpa);
49		defer gpa.free(self_exe_path);
50
51		return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path);
52		}
53
54		/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
55		/// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
56		/// On WASI, "" is returned instead of ".".
57		pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
58		if (builtin.target.os.tag == .wasi) {
59		if (std.debug.runtime_safety) {
60		const cwd = try std.process.currentPathAlloc(io, gpa);
61		defer gpa.free(cwd);
62		assert(mem.eql(u8, cwd, "."));
63		}
64		return "";
65		}
66		const cwd = try std.process.currentPathAlloc(io, gpa);
67		defer gpa.free(cwd);
68		const resolved = try Dir.path.resolve(gpa, &.{cwd});
69		assert(Dir.path.isAbsolute(resolved));
70		return resolved;
71		}
72
73		/// Both the directory handle and the path are newly allocated resources which the caller now owns.
74		pub fn findZigLibDirFromSelfExe(
75		allocator: Allocator,
76		io: Io,
77		/// The return value of `getResolvedCwd`.
78		/// Passed as an argument to avoid pointlessly repeating the call.
79		cwd_path: []const u8,
80		self_exe_path: []const u8,
81		) error{ OutOfMemory, FileNotFound }!Cache.Directory {
82		const cwd = Io.Dir.cwd();
83		var cur_path: []const u8 = self_exe_path;
84		while (Dir.path.dirname(cur_path)) \|dirname\| : (cur_path = dirname) {
85		var base_dir = cwd.openDir(io, dirname, .{}) catch continue;
86		defer base_dir.close(io);
87
88		const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue;
89		const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? });
90		defer allocator.free(p);
91
92		const resolved = try resolvePath(allocator, cwd_path, &.{p});
93		return .{
94		.handle = sub_directory.handle,
95		.path = if (resolved.len == 0) null else resolved,
96		};
97		}
98		return error.FileNotFound;
99		}
100
101		pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 {
102		if (std.zig.EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) \|value\| return value;
103
104		const app_name = "zig";
105
106		switch (builtin.os.tag) {
107		.wasi => @compileError("on WASI the global cache dir must be resolved with preopens"),
108		.windows => {
109		const local_app_data_dir = std.zig.EnvVar.LOCALAPPDATA.get(environ_map) orelse
110		return error.AppDataDirUnavailable;
111		return Dir.path.join(arena, &.{ local_app_data_dir, app_name });
112		},
113		else => {
114		if (std.zig.EnvVar.XDG_CACHE_HOME.get(environ_map)) \|cache_root\| {
115		if (cache_root.len > 0) {
116		return Dir.path.join(arena, &.{ cache_root, app_name });
117		}
118		}
119		if (std.zig.EnvVar.HOME.get(environ_map)) \|home\| {
120		if (home.len > 0) {
121		return Dir.path.join(arena, &.{ home, ".cache", app_name });
122		}
123		}
124		return error.AppDataDirUnavailable;
125		},
126		}
127		}
128
129		/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would
130		/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd
131		/// returns the empty string ("") instead of ".".
132		pub fn resolvePath(
133		gpa: Allocator,
134		/// The return value of `getResolvedCwd`.
135		/// Passed as an argument to avoid pointlessly repeating the call.
136		cwd_resolved: []const u8,
137		paths: []const []const u8,
138		) Allocator.Error![]u8 {
139		if (builtin.target.os.tag == .wasi) {
140		assert(mem.eql(u8, cwd_resolved, ""));
141		const res = try Dir.path.resolve(gpa, paths);
142		if (mem.eql(u8, res, ".")) {
143		gpa.free(res);
144		return "";
145		}
146		return res;
147		}
148
149		// Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`.
150		for (paths) \|p\| {
151		if (Dir.path.isAbsolute(p)) break; // absolute path
152		if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir
153		} else {
154		// no absolute path, no "..".
155		const res = try Dir.path.resolve(gpa, paths);
156		if (mem.eql(u8, res, ".")) {
157		gpa.free(res);
158		return "";
159		}
160		assert(!Dir.path.isAbsolute(res));
161		assert(!isUpDir(res));
162		return res;
163		}
164
165		// The fast path failed; resolve the whole thing.
166		// Optimization: `paths` often has just one element.
167		const path_resolved = switch (paths.len) {
168		0 => unreachable,
169		1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }),
170		else => r: {
171		const all_paths = try gpa.alloc([]const u8, paths.len + 1);
172		defer gpa.free(all_paths);
173		all_paths[0] = cwd_resolved;
174		@memcpy(all_paths[1..], paths);
175		break :r try Dir.path.resolve(gpa, all_paths);
176		},
177		};
178		errdefer gpa.free(path_resolved);
179
180		assert(Dir.path.isAbsolute(path_resolved));
181		assert(Dir.path.isAbsolute(cwd_resolved));
182
183		if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd
184		if (path_resolved.len == cwd_resolved.len) {
185		// equal to cwd
186		gpa.free(path_resolved);
187		return "";
188		}
189		if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs)
190
191		// in cwd; extract sub path
192		const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]);
193		gpa.free(path_resolved);
194		return sub_path;
195		}
196
197		pub fn isUpDir(p: []const u8) bool {
198		return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep);
199		}
200
201		pub const default_local_zig_cache_basename = ".zig-cache";
202
203		/// Searches upwards from `cwd` for a directory containing a `build.zig` file.
204		/// If such a directory is found, returns the path to it joined to the `.zig_cache` name.
205		/// Otherwise, returns `null`, indicating no suitable local cache location.
206		pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 {
207		var cur_dir = cwd;
208		while (true) {
209		const joined = try Dir.path.join(arena, &.{ cur_dir, Package.build_zig_basename });
210		if (Io.Dir.cwd().access(io, joined, .{})) \|_\| {
211		return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename });
212		} else \|err\| switch (err) {
213		error.FileNotFound => {
214		cur_dir = Dir.path.dirname(cur_dir) orelse return null;
215		continue;
216		},
217		else => return null,
218		}
219		}
220		}

src/libs/freebsd.zig+1-1

...	...	@@ -12,7 +12,7 @@ const Compilation = @import("../Compilation.zig");
12	12	const build_options = @import("build_options");
13	13	const trace = @import("../tracy.zig").trace;
14	14	const Cache = std.Build.Cache;
15		const Module = @import("../Package/Module.zig");
	15	const Module = @import("../Module.zig");
16	16	const link = @import("../link.zig");
17	17
18	18	pub const CrtFile = enum {

src/libs/glibc.zig+1-1

...	...	@@ -12,7 +12,7 @@ const Compilation = @import("../Compilation.zig");
12	12	const build_options = @import("build_options");
13	13	const trace = @import("../tracy.zig").trace;
14	14	const Cache = std.Build.Cache;
15		const Module = @import("../Package/Module.zig");
	15	const Module = @import("../Module.zig");
16	16	const link = @import("../link.zig");
17	17
18	18	pub const Lib = struct {

src/libs/libcxx.zig+1-1

...	...	@@ -6,7 +6,7 @@ const target_util = @import("../target.zig");
6	6	const Compilation = @import("../Compilation.zig");
7	7	const build_options = @import("build_options");
8	8	const trace = @import("../tracy.zig").trace;
9		const Module = @import("../Package/Module.zig");
	9	const Module = @import("../Module.zig");
10	10
11	11	const libcxxabi_files = [_][]const u8{
12	12	"src/cxa_aux_runtime.cpp",

src/libs/libtsan.zig+1-1

...	...	@@ -4,7 +4,7 @@ const assert = std.debug.assert;
4	4	const Compilation = @import("../Compilation.zig");
5	5	const build_options = @import("build_options");
6	6	const trace = @import("../tracy.zig").trace;
7		const Module = @import("../Package/Module.zig");
	7	const Module = @import("../Module.zig");
8	8
9	9	pub const BuildError = error{
10	10	OutOfMemory,

src/libs/libunwind.zig+1-1

...	...	@@ -4,7 +4,7 @@ const assert = std.debug.assert;
4	4
5	5	const target_util = @import("../target.zig");
6	6	const Compilation = @import("../Compilation.zig");
7		const Module = @import("../Package/Module.zig");
	7	const Module = @import("../Module.zig");
8	8	const build_options = @import("build_options");
9	9	const trace = @import("../tracy.zig").trace;
10	10

src/libs/musl.zig+1-1

...	...	@@ -3,7 +3,7 @@ const Allocator = std.mem.Allocator;
3	3	const mem = std.mem;
4	4	const path = std.fs.path;
5	5	const assert = std.debug.assert;
6		const Module = @import("../Package/Module.zig");
	6	const Module = @import("../Module.zig");
7	7
8	8	const Compilation = @import("../Compilation.zig");
9	9	const build_options = @import("build_options");

src/libs/netbsd.zig+1-1

...	...	@@ -12,7 +12,7 @@ const Compilation = @import("../Compilation.zig");
12	12	const build_options = @import("build_options");
13	13	const trace = @import("../tracy.zig").trace;
14	14	const Cache = std.Build.Cache;
15		const Module = @import("../Package/Module.zig");
	15	const Module = @import("../Module.zig");
16	16	const link = @import("../link.zig");
17	17
18	18	pub const CrtFile = enum {

src/libs/openbsd.zig+1-1

...	...	@@ -13,7 +13,7 @@ const Compilation = @import("../Compilation.zig");
13	13	const build_options = @import("build_options");
14	14	const trace = @import("../tracy.zig").trace;
15	15	const Cache = std.Build.Cache;
16		const Module = @import("../Package/Module.zig");
	16	const Module = @import("../Module.zig");
17	17	const link = @import("../link.zig");
18	18
19	19	pub const CrtFile = enum {

src/link.zig-1

...	...	@@ -21,7 +21,6 @@ const Zcu = @import("Zcu.zig");
21	21	const InternPool = @import("InternPool.zig");
22	22	const Type = @import("Type.zig");
23	23	const Value = @import("Value.zig");
24		const Package = @import("Package.zig");
25	24	const dev = @import("dev.zig");
26	25	const target_util = @import("target.zig");
27	26	const codegen = @import("codegen.zig");

src/link/C.zig+1-1

...	...	@@ -13,7 +13,7 @@ const Path = std.Build.Cache.Path;
13	13
14	14	const build_options = @import("build_options");
15	15	const Zcu = @import("../Zcu.zig");
16		const Module = @import("../Package/Module.zig");
	16	const Module = @import("../Module.zig");
17	17	const InternPool = @import("../InternPool.zig");
18	18	const Alignment = InternPool.Alignment;
19	19	const Compilation = @import("../Compilation.zig");

src/link/Dwarf.zig+1-1

...	...	@@ -10,7 +10,7 @@ const log = std.log.scoped(.dwarf);
10	10	const Writer = std.Io.Writer;
11	11
12	12	const InternPool = @import("../InternPool.zig");
13		const Module = @import("../Package.zig").Module;
	13	const Module = @import("../Module.zig");
14	14	const Type = @import("../Type.zig");
15	15	const Value = @import("../Value.zig");
16	16	const Zcu = @import("../Zcu.zig");

src/link/Lld.zig+45-46

...	...	@@ -436,23 +436,23 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
436	436	try argv.append("-DEBUG");
437	437
438	438	const out_ext = std.fs.path.extension(full_out_path);
439		const out_pdb = coff.pdb_out_path orelse try allocPrint(arena, "{s}.pdb", .{
	439	const out_pdb = coff.pdb_out_path orelse try arena.print("{s}.pdb", .{
440	440	full_out_path[0 .. full_out_path.len - out_ext.len],
441	441	});
442	442	const out_pdb_basename = std.fs.path.basename(out_pdb);
443	443
444		try argv.append(try allocPrint(arena, "-PDB:{s}", .{out_pdb}));
445		try argv.append(try allocPrint(arena, "-PDBALTPATH:{s}", .{out_pdb_basename}));
	444	try argv.append(try arena.print("-PDB:{s}", .{out_pdb}));
	445	try argv.append(try arena.print("-PDBALTPATH:{s}", .{out_pdb_basename}));
446	446	}
447	447	if (comp.version) \|version\| {
448		try argv.append(try allocPrint(arena, "-VERSION:{d}.{d}", .{ version.major, version.minor }));
	448	try argv.append(try arena.print("-VERSION:{d}.{d}", .{ version.major, version.minor }));
449	449	}
450	450
451	451	if (target_util.llvmMachineAbi(target)) \|mabi\| {
452		try argv.append(try allocPrint(arena, "-MLLVM:-target-abi={s}", .{mabi}));
	452	try argv.append(try arena.print("-MLLVM:-target-abi={s}", .{mabi}));
453	453	}
454	454
455		try argv.append(try allocPrint(arena, "-MLLVM:-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"}));
	455	try argv.append(try arena.print("-MLLVM:-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"}));
456	456
457	457	if (comp.config.lto != .none) {
458	458	switch (optimize_mode) {
...	...	@@ -462,9 +462,9 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
462	462	}
463	463	}
464	464	if (comp.config.output_mode == .Exe) {
465		try argv.append(try allocPrint(arena, "-STACK:{d}", .{base.stack_size}));
	465	try argv.append(try arena.print("-STACK:{d}", .{base.stack_size}));
466	466	}
467		try argv.append(try allocPrint(arena, "-BASE:{d}", .{coff.image_base}));
	467	try argv.append(try arena.print("-BASE:{d}", .{coff.image_base}));
468	468
469	469	switch (base.build_id) {
470	470	.none => try argv.append("-BUILD-ID:NO"),
...	...	@@ -483,7 +483,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
483	483	}
484	484
485	485	for (comp.force_undefined_symbols.keys()) \|symbol\| {
486		try argv.append(try allocPrint(arena, "-INCLUDE:{s}", .{symbol}));
	486	try argv.append(try arena.print("-INCLUDE:{s}", .{symbol}));
487	487	}
488	488
489	489	if (is_dyn_lib) {
...	...	@@ -491,7 +491,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
491	491	}
492	492
493	493	if (entry_name) \|name\| {
494		try argv.append(try allocPrint(arena, "-ENTRY:{s}", .{name}));
	494	try argv.append(try arena.print("-ENTRY:{s}", .{name}));
495	495	}
496	496
497	497	if (coff.repro) {
...	...	@@ -511,26 +511,26 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
511	511	try argv.append("-FORCE:UNRESOLVED");
512	512	}
513	513
514		try argv.append(try allocPrint(arena, "-OUT:{s}", .{full_out_path}));
	514	try argv.append(try arena.print("-OUT:{s}", .{full_out_path}));
515	515
516	516	if (comp.emit_implib) \|raw_emit_path\| {
517	517	const path = try comp.resolveEmitPathFlush(arena, .artifact, raw_emit_path);
518		try argv.append(try allocPrint(arena, "-IMPLIB:{f}", .{path}));
	518	try argv.append(try arena.print("-IMPLIB:{f}", .{path}));
519	519	}
520	520
521	521	if (comp.config.link_libc) {
522	522	if (comp.libc_installation) \|libc_installation\| {
523		try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.crt_dir.?}));
	523	try argv.append(try arena.print("-LIBPATH:{s}", .{libc_installation.crt_dir.?}));
524	524
525	525	if (target.abi == .msvc or target.abi == .itanium) {
526		try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?}));
527		try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?}));
	526	try argv.append(try arena.print("-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?}));
	527	try argv.append(try arena.print("-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?}));
528	528	}
529	529	}
530	530	}
531	531
532	532	for (coff.lib_directories) \|lib_directory\| {
533		try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{lib_directory.path orelse "."}));
	533	try argv.append(try arena.print("-LIBPATH:{s}", .{lib_directory.path orelse "."}));
534	534	}
535	535
536	536	try argv.ensureUnusedCapacity(comp.link_inputs.len);
...	...	@@ -541,7 +541,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
541	541	},
542	542	.object, .archive => \|obj\| {
543	543	if (obj.must_link) {
544		argv.appendAssumeCapacity(try allocPrint(arena, "-WHOLEARCHIVE:{f}", .{@as(Cache.Path, obj.path)}));
	544	argv.appendAssumeCapacity(try arena.print("-WHOLEARCHIVE:{f}", .{@as(Cache.Path, obj.path)}));
545	545	} else {
546	546	argv.appendAssumeCapacity(try obj.path.toString(arena));
547	547	}
...	...	@@ -561,7 +561,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
561	561	}
562	562
563	563	if (coff.module_definition_file) \|def\| {
564		try argv.append(try allocPrint(arena, "-DEF:{s}", .{def}));
	564	try argv.append(try arena.print("-DEF:{s}", .{def}));
565	565	}
566	566
567	567	const resolved_subsystem: ?std.zig.Subsystem = blk: {
...	...	@@ -590,7 +590,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
590	590	const Mode = enum { uefi, win32 };
591	591	const mode: Mode = mode: {
592	592	if (resolved_subsystem) \|subsystem\| {
593		try argv.append(try allocPrint(arena, "-SUBSYSTEM:{s},{d}.{d}", .{
	593	try argv.append(try arena.print("-SUBSYSTEM:{s},{d}.{d}", .{
594	594	@tagName(subsystem),
595	595	coff.major_subsystem_version,
596	596	coff.minor_subsystem_version,
...	...	@@ -645,8 +645,8 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
645	645	.static => "lib",
646	646	.dynamic => "",
647	647	};
648		try argv.append(try allocPrint(arena, "{s}vcruntime.lib", .{lib_str}));
649		try argv.append(try allocPrint(arena, "{s}ucrt.lib", .{lib_str}));
	648	try argv.append(try arena.print("{s}vcruntime.lib", .{lib_str}));
	649	try argv.append(try arena.print("{s}ucrt.lib", .{lib_str}));
650	650
651	651	//Visual C++ 2015 Conformance Changes
652	652	//https://msdn.microsoft.com/en-us/library/bb531344.aspx
...	...	@@ -712,7 +712,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
712	712
713	713	try argv.ensureUnusedCapacity(comp.windows_libs.count());
714	714	for (comp.windows_libs.keys()) \|key\| {
715		const lib_basename = try allocPrint(arena, "{s}.lib", .{key});
	715	const lib_basename = try arena.print("{s}.lib", .{key});
716	716	if (comp.crt_files.get(lib_basename)) \|crt_file\| {
717	717	argv.appendAssumeCapacity(try crt_file.full_object_path.toString(arena));
718	718	continue;
...	...	@@ -722,7 +722,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
722	722	continue;
723	723	}
724	724	if (target.abi.isGnu()) {
725		const fallback_name = try allocPrint(arena, "lib{s}.dll.a", .{key});
	725	const fallback_name = try arena.print("lib{s}.dll.a", .{key});
726	726	if (try findLib(arena, io, fallback_name, coff.lib_directories)) \|full_path\| {
727	727	argv.appendAssumeCapacity(full_path);
728	728	continue;
...	...	@@ -843,19 +843,19 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
843	843	try argv.append("--error-limit=0");
844	844
845	845	if (comp.sysroot) \|sysroot\| {
846		try argv.append(try std.fmt.allocPrint(arena, "--sysroot={s}", .{sysroot}));
	846	try argv.append(try arena.print("--sysroot={s}", .{sysroot}));
847	847	}
848	848
849	849	if (target_util.llvmMachineAbi(target)) \|mabi\| {
850	850	try argv.appendSlice(&.{
851	851	"-mllvm",
852		try std.fmt.allocPrint(arena, "-target-abi={s}", .{mabi}),
	852	try arena.print("-target-abi={s}", .{mabi}),
853	853	});
854	854	}
855	855
856	856	try argv.appendSlice(&.{
857	857	"-mllvm",
858		try std.fmt.allocPrint(arena, "-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"}),
	858	try arena.print("-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"}),
859	859	});
860	860
861	861	switch (target.cpu.arch) {
...	...	@@ -894,19 +894,19 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
894	894	if (output_mode == .Exe) {
895	895	try argv.appendSlice(&.{
896	896	"-z",
897		try std.fmt.allocPrint(arena, "stack-size={d}", .{base.stack_size}),
	897	try arena.print("stack-size={d}", .{base.stack_size}),
898	898	});
899	899	}
900	900
901	901	switch (base.build_id) {
902	902	.none => try argv.append("--build-id=none"),
903		.fast, .uuid, .sha1, .md5 => try argv.append(try std.fmt.allocPrint(arena, "--build-id={s}", .{
	903	.fast, .uuid, .sha1, .md5 => try argv.append(try arena.print("--build-id={s}", .{
904	904	@tagName(base.build_id),
905	905	})),
906		.hexstring => \|hs\| try argv.append(try std.fmt.allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()})),
	906	.hexstring => \|hs\| try argv.append(try arena.print("--build-id=0x{x}", .{hs.toSlice()})),
907	907	}
908	908
909		try argv.append(try std.fmt.allocPrint(arena, "--image-base={d}", .{elf.image_base}));
	909	try argv.append(try arena.print("--image-base={d}", .{elf.image_base}));
910	910
911	911	if (elf.linker_script) \|linker_script\| {
912	912	try argv.append("-T");
...	...	@@ -914,7 +914,7 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
914	914	}
915	915
916	916	if (elf.sort_section) \|how\| {
917		const arg = try std.fmt.allocPrint(arena, "--sort-section={s}", .{@tagName(how)});
	917	const arg = try arena.print("--sort-section={s}", .{@tagName(how)});
918	918	try argv.append(arg);
919	919	}
920	920
...	...	@@ -980,11 +980,11 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
980	980	}
981	981	if (elf.z_common_page_size) \|size\| {
982	982	try argv.append("-z");
983		try argv.append(try std.fmt.allocPrint(arena, "common-page-size={d}", .{size}));
	983	try argv.append(try arena.print("common-page-size={d}", .{size}));
984	984	}
985	985	if (elf.z_max_page_size) \|size\| {
986	986	try argv.append("-z");
987		try argv.append(try std.fmt.allocPrint(arena, "max-page-size={d}", .{size}));
	987	try argv.append(try arena.print("max-page-size={d}", .{size}));
988	988	}
989	989
990	990	if (getLDMOption(target)) \|ldm\| {
...	...	@@ -1190,7 +1190,7 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
1190	1190	if (target.os.versionRange().gnuLibCVersion().?.order(rem_in) != .lt) continue;
1191	1191	}
1192	1192
1193		const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so.{d}", .{
	1193	const lib_path = try arena.print("{f}{c}lib{s}.so.{d}", .{
1194	1194	comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover,
1195	1195	});
1196	1196	try argv.append(lib_path);
...	...	@@ -1207,21 +1207,21 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
1207	1207	if (target.os.version_range.semver.min.order(add_in) == .lt) continue;
1208	1208	}
1209	1209
1210		const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so.{d}", .{
	1210	const lib_path = try arena.print("{f}{c}lib{s}.so.{d}", .{
1211	1211	comp.freebsd_so_files.?.dir_path, fs.path.sep, lib.name, lib.getSoVersion(&target.os),
1212	1212	});
1213	1213	try argv.append(lib_path);
1214	1214	}
1215	1215	} else if (target.isNetBSDLibC()) {
1216	1216	for (netbsd.libs) \|lib\| {
1217		const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so.{d}", .{
	1217	const lib_path = try arena.print("{f}{c}lib{s}.so.{d}", .{
1218	1218	comp.netbsd_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover,
1219	1219	});
1220	1220	try argv.append(lib_path);
1221	1221	}
1222	1222	} else if (target.isOpenBSDLibC()) {
1223	1223	for (openbsd.libs) \|lib\| {
1224		const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so", .{
	1224	const lib_path = try arena.print("{f}{c}lib{s}.so", .{
1225	1225	comp.openbsd_so_files.?.dir_path, fs.path.sep, lib.name,
1226	1226	});
1227	1227	try argv.append(lib_path);
...	...	@@ -1451,12 +1451,12 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void {
1451	1451	}
1452	1452
1453	1453	if (wasm.initial_memory) \|initial_memory\| {
1454		const arg = try std.fmt.allocPrint(arena, "--initial-memory={d}", .{initial_memory});
	1454	const arg = try arena.print("--initial-memory={d}", .{initial_memory});
1455	1455	try argv.append(arg);
1456	1456	}
1457	1457
1458	1458	if (wasm.max_memory) \|max_memory\| {
1459		const arg = try std.fmt.allocPrint(arena, "--max-memory={d}", .{max_memory});
	1459	const arg = try arena.print("--max-memory={d}", .{max_memory});
1460	1460	try argv.append(arg);
1461	1461	}
1462	1462
...	...	@@ -1465,7 +1465,7 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void {
1465	1465	}
1466	1466
1467	1467	if (wasm.global_base) \|global_base\| {
1468		const arg = try std.fmt.allocPrint(arena, "--global-base={d}", .{global_base});
	1468	const arg = try arena.print("--global-base={d}", .{global_base});
1469	1469	try argv.append(arg);
1470	1470	} else {
1471	1471	// We prepend it by default, so when a stack overflow happens the runtime will trap correctly,
...	...	@@ -1477,7 +1477,7 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void {
1477	1477
1478	1478	// Users are allowed to specify which symbols they want to export to the wasm host.
1479	1479	for (wasm.export_symbol_names) \|symbol_name\| {
1480		const arg = try std.fmt.allocPrint(arena, "--export={s}", .{symbol_name});
	1480	const arg = try arena.print("--export={s}", .{symbol_name});
1481	1481	try argv.append(arg);
1482	1482	}
1483	1483
...	...	@@ -1493,15 +1493,15 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void {
1493	1493
1494	1494	try argv.appendSlice(&.{
1495	1495	"-z",
1496		try std.fmt.allocPrint(arena, "stack-size={d}", .{base.stack_size}),
	1496	try arena.print("stack-size={d}", .{base.stack_size}),
1497	1497	});
1498	1498
1499	1499	switch (base.build_id) {
1500	1500	.none => try argv.append("--build-id=none"),
1501		.fast, .uuid, .sha1 => try argv.append(try std.fmt.allocPrint(arena, "--build-id={s}", .{
	1501	.fast, .uuid, .sha1 => try argv.append(try arena.print("--build-id={s}", .{
1502	1502	@tagName(base.build_id),
1503	1503	})),
1504		.hexstring => \|hs\| try argv.append(try std.fmt.allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()})),
	1504	.hexstring => \|hs\| try argv.append(try arena.print("--build-id=0x{x}", .{hs.toSlice()})),
1505	1505	.md5 => {},
1506	1506	}
1507	1507
...	...	@@ -1685,7 +1685,7 @@ fn spawnLld(comp: *Compilation, arena: Allocator, argv: []const []const u8) !voi
1685	1685	.argv = &.{
1686	1686	argv[0],
1687	1687	argv[1],
1688		try std.fmt.allocPrint(arena, "@{s}", .{
	1688	try arena.print("@{s}", .{
1689	1689	try comp.dirs.local_cache.join(arena, &.{rsp_path}),
1690	1690	}),
1691	1691	},
...	...	@@ -1740,7 +1740,6 @@ const std = @import("std");
1740	1740	const Io = std.Io;
1741	1741	const Allocator = std.mem.Allocator;
1742	1742	const Cache = std.Build.Cache;
1743		const allocPrint = std.fmt.allocPrint;
1744	1743	const assert = std.debug.assert;
1745	1744	const fs = std.fs;
1746	1745	const log = std.log.scoped(.link);

src/main.zig+227-2081

...	...	@@ -25,18 +25,16 @@ const stringToEnum = std.meta.stringToEnum;
25	25	pub const tracy = @import("tracy.zig");
26	26	const Compilation = @import("Compilation.zig");
27	27	const link = @import("link.zig");
28		const Package = @import("Package.zig");
29	28	const build_options = @import("build_options");
30		const introspect = @import("introspect.zig");
31	29	const wasi_libc = @import("libs/wasi_libc.zig");
32	30	const target_util = @import("target.zig");
33	31	const crash_report = @import("crash_report.zig");
34	32	const Zcu = @import("Zcu.zig");
35	33	const mingw = @import("libs/mingw.zig");
36	34	const dev = @import("dev.zig");
	35	const Module = @import("Module.zig");
37	36
38	37	test {
39		_ = Package;
40	38	_ = @import("codegen.zig");
41	39	}
42	40
...	...	@@ -353,9 +351,18 @@ fn mainArgs(
353	351	dev.check(.ar_command);
354	352	return process.exit(try llvmArMain(arena, args));
355	353	},
356		.build => {
357		dev.check(.build_command);
358		return cmdBuild(gpa, arena, io, cmd_args, environ_map);
	354	.build, .fetch, .init, .libc => {
	355	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
	356	.cmd_name = "maker",
	357	.root_src_path = "Maker.zig",
	358	.prepend_cmd = cmd,
	359	.prepend_zig_lib_dir_path = true,
	360	.prepend_global_cache_path = true,
	361	.prepend_zig_exe_path = true,
	362	.prepend_seed = true,
	363	.debug_env_var = .ZIG_DEBUG_MAKER,
	364	.release_mode = .ReleaseSafe,
	365	});
359	366	},
360	367	.clang, .@"-cc1", .@"-cc1as" => {
361	368	dev.check(.clang_command);
...	...	@@ -385,7 +392,6 @@ fn mainArgs(
385	392	.depend_on_aro = true,
386	393	.prepend_zig_lib_dir_path = true,
387	394	.server = use_server,
388		.color = Color.settingFromEnvironment(environ_map),
389	395	});
390	396	},
391	397	.fmt => {
...	...	@@ -396,25 +402,12 @@ fn mainArgs(
396	402	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
397	403	.cmd_name = "objcopy",
398	404	.root_src_path = "objcopy.zig",
399		.color = Color.settingFromEnvironment(environ_map),
400	405	});
401	406	},
402	407	.objdump => {
403	408	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
404	409	.cmd_name = "objdump",
405	410	.root_src_path = "objdump.zig",
406		.color = Color.settingFromEnvironment(environ_map),
407		});
408		},
409		.fetch => {
410		return cmdFetch(gpa, arena, io, cmd_args, environ_map);
411		},
412		.libc => {
413		return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
414		.cmd_name = "libc",
415		.root_src_path = "libc.zig",
416		.prepend_zig_lib_dir_path = true,
417		.color = Color.settingFromEnvironment(environ_map),
418	411	});
419	412	},
420	413	.std => {
...	...	@@ -424,12 +417,8 @@ fn mainArgs(
424	417	.prepend_zig_lib_dir_path = true,
425	418	.prepend_zig_exe_path = true,
426	419	.prepend_global_cache_path = true,
427		.color = Color.settingFromEnvironment(environ_map),
428	420	});
429	421	},
430		.init => {
431		return cmdInit(gpa, arena, io, cmd_args);
432		},
433	422	.targets => {
434	423	dev.check(.targets_command);
435	424	const host = std.zig.resolveTargetQueryOrFatal(io, .{});
...	...	@@ -461,7 +450,6 @@ fn mainArgs(
461	450	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
462	451	.cmd_name = "reduce",
463	452	.root_src_path = "reduce.zig",
464		.color = Color.settingFromEnvironment(environ_map),
465	453	});
466	454	},
467	455	.zen => {
...	...	@@ -884,13 +872,13 @@ const CliModule = struct {
884	872	root_path: []const u8,
885	873	root_src_path: []const u8,
886	874	cc_argv: []const []const u8,
887		inherited: Package.Module.CreateOptions.Inherited,
	875	inherited: Module.CreateOptions.Inherited,
888	876	target_arch_os_abi: ?[]const u8,
889	877	target_mcpu: ?[]const u8,
890	878	dynamic_linker: ?[]const u8,
891	879
892	880	deps: []const Dep,
893		resolved: ?*Package.Module,
	881	resolved: ?*Module,
894	882
895	883	c_source_files_start: usize,
896	884	c_source_files_end: usize,
...	...	@@ -1041,7 +1029,7 @@ fn buildOutputType(
1041	1029
1042	1030	// These get set by CLI flags and then snapshotted when a `-M` flag is
1043	1031	// encountered.
1044		var mod_opts: Package.Module.CreateOptions.Inherited = .{};
	1032	var mod_opts: Module.CreateOptions.Inherited = .{};
1045	1033
1046	1034	// These get appended to by CLI flags and then slurped when a `-M` flag
1047	1035	// is encountered.
...	...	@@ -2977,7 +2965,7 @@ fn buildOutputType(
2977	2965	while (preprocessor_args_it.next()) \|arg\| {
2978	2966	if (mem.eql(u8, arg, "-MD") or mem.eql(u8, arg, "-MMD") or mem.eql(u8, arg, "-MT")) {
2979	2967	disable_c_depfile = true;
2980		const cc_arg = try std.fmt.allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() });
	2968	const cc_arg = try arena.print("-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() });
2981	2969	try cc_argv.append(arena, cc_arg);
2982	2970	} else {
2983	2971	fatal("unsupported preprocessor arg: {s}", .{arg});
...	...	@@ -3222,10 +3210,10 @@ fn buildOutputType(
3222	3210	else => process.executablePathAlloc(io, arena) catch \|err\| fatal("unable to find zig self exe path: {t}", .{err}),
3223	3211	};
3224	3212
3225		const cwd_path = try introspect.getResolvedCwd(io, arena);
	3213	const cwd_path = try std.zig.getResolvedCwd(io, arena);
3226	3214
3227	3215	// This `init` calls `fatal` on error.
3228		var dirs: Compilation.Directories = .init(
	3216	var dirs: std.zig.Directories = .init(
3229	3217	arena,
3230	3218	io,
3231	3219	override_lib_dir,
...	...	@@ -3272,7 +3260,7 @@ fn buildOutputType(
3272	3260	const root_mod = switch (arg_mode) {
3273	3261	.zig_test, .zig_test_obj => root_mod: {
3274	3262	const test_mod = if (test_runner_path) \|test_runner\| test_mod: {
3275		const test_mod = try Package.Module.create(arena, .{
	3263	const test_mod = try Module.create(arena, .{
3276	3264	.paths = .{
3277	3265	.root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(test_runner) orelse "."}),
3278	3266	.root_src_path = fs.path.basename(test_runner),
...	...	@@ -3285,7 +3273,7 @@ fn buildOutputType(
3285	3273	});
3286	3274	test_mod.deps = try main_mod.deps.clone(arena);
3287	3275	break :test_mod test_mod;
3288		} else try Package.Module.create(arena, .{
	3276	} else try Module.create(arena, .{
3289	3277	.paths = .{
3290	3278	.root = try .fromRoot(arena, dirs, .zig_lib, "compiler"),
3291	3279	.root_src_path = "test_runner.zig",
...	...	@@ -3421,9 +3409,9 @@ fn buildOutputType(
3421	3409	.yes_default_value => if (create_module.resolved_options.output_mode == .Lib and
3422	3410	create_module.resolved_options.link_mode == .dynamic and target.ofmt == .elf)
3423	3411	if (have_version)
3424		try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major })
	3412	try arena.print("lib{s}.so.{d}", .{ root_name, version.major })
3425	3413	else
3426		try std.fmt.allocPrint(arena, "lib{s}.so", .{root_name})
	3414	try arena.print("lib{s}.so", .{root_name})
3427	3415	else
3428	3416	null,
3429	3417	};
...	...	@@ -3433,7 +3421,7 @@ fn buildOutputType(
3433	3421	.yes_default_path => emit: {
3434	3422	if (output_to_cache != null) break :emit .yes_cache;
3435	3423	const name = switch (clang_preprocessor_mode) {
3436		.pch => try std.fmt.allocPrint(arena, "{s}.pch", .{root_name}),
	3424	.pch => try arena.print("{s}.pch", .{root_name}),
3437	3425	else => try std.zig.binNameAlloc(arena, .{
3438	3426	.root_name = root_name,
3439	3427	.cpu_arch = target.cpu.arch,
...	...	@@ -3469,16 +3457,16 @@ fn buildOutputType(
3469	3457	},
3470	3458	};
3471	3459
3472		const default_h_basename = try std.fmt.allocPrint(arena, "{s}.h", .{root_name});
	3460	const default_h_basename = try arena.print("{s}.h", .{root_name});
3473	3461	const emit_h_resolved = emit_h.resolve(io, default_h_basename, output_to_cache);
3474	3462
3475		const default_asm_basename = try std.fmt.allocPrint(arena, "{s}.s", .{root_name});
	3463	const default_asm_basename = try arena.print("{s}.s", .{root_name});
3476	3464	const emit_asm_resolved = emit_asm.resolve(io, default_asm_basename, output_to_cache);
3477	3465
3478		const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{s}.ll", .{root_name});
	3466	const default_llvm_ir_basename = try arena.print("{s}.ll", .{root_name});
3479	3467	const emit_llvm_ir_resolved = emit_llvm_ir.resolve(io, default_llvm_ir_basename, output_to_cache);
3480	3468
3481		const default_llvm_bc_basename = try std.fmt.allocPrint(arena, "{s}.bc", .{root_name});
	3469	const default_llvm_bc_basename = try arena.print("{s}.bc", .{root_name});
3482	3470	const emit_llvm_bc_resolved = emit_llvm_bc.resolve(io, default_llvm_bc_basename, output_to_cache);
3483	3471
3484	3472	const emit_docs_resolved = emit_docs.resolve(io, "docs", output_to_cache);
...	...	@@ -3499,7 +3487,7 @@ fn buildOutputType(
3499	3487	fatal("the argument -femit-implib is allowed only when building a Windows DLL", .{});
3500	3488	}
3501	3489	}
3502		const default_implib_basename = try std.fmt.allocPrint(arena, "{s}.lib", .{root_name});
	3490	const default_implib_basename = try arena.print("{s}.lib", .{root_name});
3503	3491	const emit_implib_resolved: Compilation.CreateOptions.Emit = switch (emit_implib) {
3504	3492	.no => .no,
3505	3493	.yes => emit_implib.resolve(io, default_implib_basename, output_to_cache),
...	...	@@ -3528,7 +3516,7 @@ fn buildOutputType(
3528	3516
3529	3517	// "-" is stdin. Dump it to a real file.
3530	3518	const sep = fs.path.sep_str;
3531		const dump_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{
	3519	const dump_path = try arena.print("tmp" ++ sep ++ "{x}-dump-stdin{s}", .{
3532	3520	randInt(io, u64), ext.canonicalName(target),
3533	3521	});
3534	3522	try dirs.local_cache.handle.createDirPath(io, "tmp");
...	...	@@ -3557,7 +3545,7 @@ fn buildOutputType(
3557	3545
3558	3546	const bin_digest: Cache.BinDigest = hasher.hasher.finalResult();
3559	3547
3560		const sub_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{
	3548	const sub_path = try arena.print("tmp" ++ sep ++ "{x}-stdin{s}", .{
3561	3549	&bin_digest, ext.canonicalName(target),
3562	3550	});
3563	3551	try dirs.local_cache.handle.rename(dump_path, dirs.local_cache.handle, sub_path, io);
...	...	@@ -3889,9 +3877,9 @@ fn buildOutputType(
3889	3877	for (mod.cc_argv) \|cc_arg\| test_exec_args.appendAssumeCapacity(cc_arg);
3890	3878	for (mod.deps) \|dep\| try test_exec_args.appendSlice(arena, &.{
3891	3879	"--dep",
3892		if (std.mem.eql(u8, dep.key, dep.value)) dep.value else try std.fmt.allocPrint(arena, "{s}={s}", .{ dep.key, dep.value }),
	3880	if (std.mem.eql(u8, dep.key, dep.value)) dep.value else try arena.print("{s}={s}", .{ dep.key, dep.value }),
3893	3881	});
3894		try test_exec_args.append(arena, try std.fmt.allocPrint(arena, "-M{s}", .{mod_name}));
	3882	try test_exec_args.append(arena, try arena.print("-M{s}", .{mod_name}));
3895	3883	}
3896	3884
3897	3885	try test_exec_args.ensureUnusedCapacity(arena, comp.global_cc_argv.len);
...	...	@@ -3935,7 +3923,7 @@ fn buildOutputType(
3935	3923	}
3936	3924
3937	3925	const CreateModule = struct {
3938		dirs: Compilation.Directories,
	3926	dirs: std.zig.Directories,
3939	3927	modules: std.array_hash_map.String(CliModule),
3940	3928	opts: Compilation.Config.Options,
3941	3929	object_format: ?[]const u8,
...	...	@@ -3979,10 +3967,10 @@ fn createModule(
3979	3967	io: Io,
3980	3968	create_module: *CreateModule,
3981	3969	index: usize,
3982		parent: ?*Package.Module,
	3970	parent: ?*Module,
3983	3971	color: std.zig.Color,
3984	3972	environ_map: *process.Environ.Map,
3985		) Allocator.Error!*Package.Module {
	3973	) Allocator.Error!*Module {
3986	3974	const cli_mod = &create_module.modules.values()[index];
3987	3975	if (cli_mod.resolved) \|m\| return m;
3988	3976
...	...	@@ -4258,7 +4246,7 @@ fn createModule(
4258	4246
4259	4247	const root: Compilation.Path = try .fromUnresolved(arena, create_module.dirs, &.{cli_mod.root_path});
4260	4248
4261		const mod = Package.Module.create(arena, .{
	4249	const mod = Module.create(arena, .{
4262	4250	.paths = .{
4263	4251	.root = root,
4264	4252	.root_src_path = cli_mod.root_src_path,
...	...	@@ -4586,7 +4574,7 @@ fn runOrTest(
4586	4574	try argv.append(exe_path);
4587	4575	if (arg_mode == .zig_test) {
4588	4576	try argv.append(
4589		try std.fmt.allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}),
	4577	try arena.print("--seed=0x{x}", .{randInt(io, u32)}),
4590	4578	);
4591	4579	}
4592	4580	} else {
...	...	@@ -4794,7 +4782,7 @@ fn cmdTranslateC(
4794	4782	assert(comp.c_source_files.len == 1);
4795	4783	const c_source_file = comp.c_source_files[0];
4796	4784
4797		const translated_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{comp.root_name});
	4785	const translated_basename = try arena.print("{s}.zig", .{comp.root_name});
4798	4786
4799	4787	var man: Cache.Manifest = comp.obtainCObjectCacheManifest(comp.root_mod);
4800	4788	man.want_shared_lock = false;
...	...	@@ -4872,1389 +4860,223 @@ pub fn translateC(
4872	4860	.root_src_path = "translate-c/main.zig",
4873	4861	.depend_on_aro = true,
4874	4862	.capture = capture,
4875		.color = Color.settingFromEnvironment(environ_map),
4876	4863	});
4877	4864	}
4878	4865
4879		const usage_init =
4880		\\Usage: zig init
4881		\\
4882		\\ Initializes a `zig build` project in the current working
4883		\\ directory.
4884		\\
4885		\\Options:
4886		\\ -m, --minimal Use minimal init template
4887		\\ -h, --help Print this help and exit
4888		\\
4889		\\
4890		;
4891
4892		fn cmdInit(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8) !void {
4893		dev.check(.init_command);
4894
4895		var template: enum { example, minimal } = .example;
4896		{
4897		var i: usize = 0;
4898		while (i < args.len) : (i += 1) {
4899		const arg = args[i];
4900		if (mem.startsWith(u8, arg, "-")) {
4901		if (mem.eql(u8, arg, "-m") or mem.eql(u8, arg, "--minimal")) {
4902		template = .minimal;
4903		} else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
4904		try Io.File.stdout().writeStreamingAll(io, usage_init);
4905		return cleanExit(io);
4906		} else {
4907		fatal("unrecognized parameter: {q}", .{arg});
4908		}
4909		} else {
4910		fatal("unexpected extra parameter: {q}", .{arg});
4911		}
4912		}
4913		}
4914
4915		const cwd_path = try introspect.getResolvedCwd(io, arena);
4916		const cwd_basename = fs.path.basename(cwd_path);
4917		const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename);
4918
4919		const rng: std.Random.IoSource = .{ .io = io };
4920		const fingerprint: Package.Fingerprint = .generate(rng.interface(), sanitized_root_name);
4921
4922		switch (template) {
4923		.example => {
4924		var templates = findTemplates(gpa, arena, io);
4925		defer templates.deinit(io);
4926
4927		const s = fs.path.sep_str;
4928		const template_paths = [_][]const u8{
4929		Package.build_zig_basename,
4930		Package.Manifest.basename,
4931		"src" ++ s ++ "main.zig",
4932		"src" ++ s ++ "root.zig",
4933		};
4934		var ok_count: usize = 0;
4935
4936		for (template_paths) \|template_path\| {
4937		if (templates.write(arena, io, Io.Dir.cwd(), sanitized_root_name, template_path, fingerprint)) \|_\| {
4938		std.log.info("created {s}", .{template_path});
4939		ok_count += 1;
4940		} else \|err\| switch (err) {
4941		error.PathAlreadyExists => std.log.info("preserving already existing file: {s}", .{
4942		template_path,
4943		}),
4944		else => std.log.err("unable to write {s}: {s}\n", .{ template_path, @errorName(err) }),
4945		}
4946		}
	4866	const JitCmdOptions = struct {
	4867	cmd_name: []const u8,
	4868	root_src_path: []const u8,
	4869	prepend_cmd: ?[]const u8 = null,
	4870	prepend_zig_lib_dir_path: bool = false,
	4871	prepend_global_cache_path: bool = false,
	4872	prepend_zig_exe_path: bool = false,
	4873	prepend_seed: bool = false,
	4874	depend_on_aro: bool = false,
	4875	capture: ?*[]u8 = null,
	4876	/// Send error bundles via std.zig.Server over stdout
	4877	server: bool = false,
	4878	debug_env_var: EnvVar = .ZIG_DEBUG_CMD,
	4879	release_mode: std.lang.OptimizeMode = .ReleaseFast,
	4880	};
4947	4881
4948		if (ok_count == template_paths.len) {
4949		std.log.info("see `zig build --help` for a menu of options", .{});
4950		}
4951		return cleanExit(io);
4952		},
4953		.minimal => {
4954		writeSimpleTemplateFile(io, Package.Manifest.basename,
4955		\\.{{
4956		\\ .name = .{s},
4957		\\ .version = "0.0.1",
4958		\\ .minimum_zig_version = "{s}",
4959		\\ .paths = .{{""}},
4960		\\ .fingerprint = 0x{x},
4961		\\}}
4962		\\
4963		, .{
4964		sanitized_root_name,
4965		build_options.version,
4966		fingerprint.int(),
4967		}) catch \|err\| switch (err) {
4968		else => fatal("failed to create {q}: {t}", .{ Package.Manifest.basename, err }),
4969		error.PathAlreadyExists => fatal("refusing to overwrite {q}", .{Package.Manifest.basename}),
4970		};
4971		writeSimpleTemplateFile(io, Package.build_zig_basename,
4972		\\const std = @import("std");
4973		\\
4974		\\pub fn build(b: *std.Build) void {{
4975		\\ _ = b; // stub
4976		\\}}
4977		\\
4978		, .{}) catch \|err\| switch (err) {
4979		else => fatal("failed to create {q}: {t}", .{ Package.build_zig_basename, err }),
4980		// `build.zig` already existing is okay: the user has just used `zig init` to set up
4981		// their `build.zig.zon` after writing their `build.zig`. So this one isn't fatal.
4982		error.PathAlreadyExists => {
4983		std.log.info("successfully populated {q}, preserving existing {q}", .{
4984		Package.Manifest.basename, Package.build_zig_basename,
4985		});
4986		return cleanExit(io);
4987		},
4988		};
4989		std.log.info("successfully populated {q} and {q}", .{ Package.Manifest.basename, Package.build_zig_basename });
4990		return cleanExit(io);
4991		},
4992		}
4993		}
	4882	fn jitCmd(
	4883	gpa: Allocator,
	4884	arena: Allocator,
	4885	io: Io,
	4886	args: []const []const u8,
	4887	environ_map: *const process.Environ.Map,
	4888	options: JitCmdOptions,
	4889	) !void {
	4890	dev.check(.jit_command);
4994	4891
4995		fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 {
4996		var result: std.ArrayList(u8) = .empty;
4997		for (bytes, 0..) \|byte, i\| switch (byte) {
4998		'0'...'9' => {
4999		if (i == 0) try result.append(arena, '_');
5000		try result.append(arena, byte);
5001		},
5002		'_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte),
5003		'-', '.', ' ' => try result.append(arena, '_'),
5004		else => continue,
5005		};
5006		if (!std.zig.isValidId(result.items)) return "foo";
5007		if (result.items.len > Package.Manifest.max_name_len)
5008		result.shrinkRetainingCapacity(Package.Manifest.max_name_len);
	4892	const color = Color.settingFromEnvironment(environ_map);
5009	4893
5010		return result.toOwnedSlice(arena);
5011		}
	4894	const root_prog_node = std.Progress.start(io, .{
	4895	.disable_printing = (color == .off),
	4896	.root_name = try arena.print("Compiling {s} (first time setup)", .{options.cmd_name}),
	4897	});
	4898	defer root_prog_node.end();
5012	4899
5013		test sanitizeExampleName {
5014		var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator);
5015		defer arena_instance.deinit();
5016		const arena = arena_instance.allocator();
	4900	const thread_limit = @min(
	4901	@max(std.Thread.getCpuCount() catch 1, 1),
	4902	std.math.maxInt(Zcu.PerThread.IdBacking),
	4903	);
	4904	try setThreadLimit(arena, thread_limit);
5017	4905
5018		try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+"));
5019		try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, ""));
5020		try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!"));
5021		try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a"));
5022		try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!"));
5023		try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234"));
5024		try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error"));
5025		try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test"));
5026		try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests"));
5027		try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project"));
	4906	return jitCmdInner(gpa, arena, io, args, environ_map, root_prog_node, thread_limit, options);
5028	4907	}
5029	4908
5030		fn cmdBuild(
	4909	fn jitCmdInner(
5031	4910	gpa: Allocator,
5032	4911	arena: Allocator,
5033	4912	io: Io,
5034	4913	args: []const []const u8,
5035		environ_map: *process.Environ.Map,
	4914	environ_map: *const process.Environ.Map,
	4915	root_prog_node: std.Progress.Node,
	4916	thread_limit: usize,
	4917	options: JitCmdOptions,
5036	4918	) !void {
5037		var build_file: ?[]const u8 = null;
5038		var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
5039		var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
5040		var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
5041		var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
5042		var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map))
	4919	const target_query: std.Target.Query = .{};
	4920	const resolved_target: Module.ResolvedTarget = .{
	4921	.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
	4922	.is_native_os = true,
	4923	.is_native_abi = true,
	4924	.is_explicit_dynamic_linker = false,
	4925	};
	4926
	4927	const self_exe_path = process.executablePathAlloc(io, arena) catch \|err\|
	4928	fatal("unable to find self exe path: {t}", .{err});
	4929
	4930	const optimize_mode: std.lang.OptimizeMode = if (options.debug_env_var.isSet(environ_map))
5043	4931	.Debug
5044	4932	else
5045		.ReleaseSafe;
5046		var configure_argv: std.ArrayList([]const u8) = .empty;
5047		var make_argv: std.ArrayList([]const u8) = .empty;
5048		var cached_passthru_configure: std.ArrayList(u32) = .empty;
5049		var forks: std.ArrayList(Fork) = .empty;
5050		var reference_trace: ?u32 = null;
5051		var debug_compile_errors = false;
5052		var verbose_link = (native_os != .wasi or builtin.link_libc) and
5053		EnvVar.ZIG_VERBOSE_LINK.isSet(environ_map);
5054		var verbose_cc = (native_os != .wasi or builtin.link_libc) and
5055		EnvVar.ZIG_VERBOSE_CC.isSet(environ_map);
5056		var verbose_air = false;
5057		var verbose_intern_pool = false;
5058		var verbose_generic_instances = false;
5059		var verbose_llvm_ir: ?[]const u8 = null;
5060		var verbose_llvm_bc: ?[]const u8 = null;
5061		var verbose_llvm_cpu_features = false;
5062		var fetch_only = false;
5063		var fetch_mode: Package.Fetch.JobQueue.Mode = .needed;
5064		var system_pkg_dir_path: ?[]const u8 = null;
5065		var debug_target: ?[]const u8 = null;
5066		var debug_libc_paths_file: ?[]const u8 = null;
5067		var cache_poison: std.Build.Graph.CachePoison = .pure;
5068		var print_configuration_path: bool = false;
	4933	options.release_mode;
	4934	const strip = optimize_mode != .Debug;
	4935	const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
	4936	const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
5069	4937
5070		const self_exe_path = try process.executablePathAlloc(io, arena);
5071		const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)});
	4938	const cwd_path = try std.zig.getResolvedCwd(io, arena);
5072	4939
5073		try configure_argv.ensureUnusedCapacity(arena, 16);
5074		try make_argv.ensureUnusedCapacity(arena, 16);
5075		try cached_passthru_configure.ensureUnusedCapacity(arena, 16);
	4940	// This `init` calls `fatal` on error.
	4941	var dirs: std.zig.Directories = .init(
	4942	arena,
	4943	io,
	4944	override_lib_dir,
	4945	override_global_cache_dir,
	4946	.global,
	4947	preopens,
	4948	self_exe_path,
	4949	environ_map,
	4950	cwd_path,
	4951	);
	4952	defer dirs.deinit(io);
5076	4953
5077		_ = configure_argv.addOneAssumeCapacity(); // configurer executable
5078		_ = make_argv.addOneAssumeCapacity(); // maker executable
	4954	var child_argv: std.ArrayList([]const u8) = .empty;
	4955	try child_argv.ensureUnusedCapacity(arena, args.len + 6);
5079	4956
5080		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path };
5081		configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path };
	4957	// We want to release all the locks before executing the child process, so we make a nice
	4958	// big block here to ensure the cleanup gets run when we extract out our argv.
	4959	{
	4960	const main_mod_paths: Module.CreateOptions.Paths = .{
	4961	.root = try .fromRoot(arena, dirs, .zig_lib, "compiler"),
	4962	.root_src_path = options.root_src_path,
	4963	};
5082	4964
5083		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined };
5084		const make_argv_index_zig_lib_dir = make_argv.items.len - 1;
	4965	const config = try Compilation.Config.resolve(.{
	4966	.output_mode = .Exe,
	4967	.root_strip = strip,
	4968	.root_optimize_mode = optimize_mode,
	4969	.resolved_target = resolved_target,
	4970	.have_zcu = true,
	4971	.emit_bin = true,
	4972	.is_test = false,
	4973	});
5085	4974
5086		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined };
5087		const make_argv_index_build_root = make_argv.items.len - 1;
	4975	const root_mod = try Module.create(arena, .{
	4976	.paths = main_mod_paths,
	4977	.fully_qualified_name = "root",
	4978	.cc_argv = &.{},
	4979	.inherited = .{
	4980	.resolved_target = resolved_target,
	4981	.optimize_mode = optimize_mode,
	4982	.strip = strip,
	4983	},
	4984	.global = config,
	4985	.parent = null,
	4986	});
5088	4987
5089		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined };
5090		const make_argv_index_cache_dir = make_argv.items.len - 1;
	4988	if (options.depend_on_aro) {
	4989	const aro_mod = try Module.create(arena, .{
	4990	.paths = .{
	4991	.root = try .fromRoot(arena, dirs, .zig_lib, "compiler/aro"),
	4992	.root_src_path = "aro.zig",
	4993	},
	4994	.fully_qualified_name = "aro",
	4995	.cc_argv = &.{},
	4996	.inherited = .{
	4997	.resolved_target = resolved_target,
	4998	.optimize_mode = optimize_mode,
	4999	.strip = strip,
	5000	},
	5001	.global = config,
	5002	.parent = null,
	5003	});
	5004	try root_mod.deps.put(arena, "aro", aro_mod);
	5005	}
5091	5006
5092		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined };
5093		const make_argv_index_global_cache_dir = make_argv.items.len - 1;
	5007	var create_diag: Compilation.CreateDiagnostic = undefined;
	5008	const comp = Compilation.create(gpa, arena, io, &create_diag, .{
	5009	.dirs = dirs,
	5010	.root_name = options.cmd_name,
	5011	.config = config,
	5012	.root_mod = root_mod,
	5013	.main_mod = root_mod,
	5014	.emit_bin = .yes_cache,
	5015	.self_exe_path = self_exe_path,
	5016	.thread_limit = thread_limit,
	5017	.cache_mode = .whole,
	5018	.environ_map = environ_map,
	5019	}) catch \|err\| switch (err) {
	5020	error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}),
	5021	else => fatal("failed to create compilation: {t}", .{err}),
	5022	};
	5023	defer comp.destroy();
5094	5024
5095		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined };
5096		const argv_index_configuration_file = make_argv.items.len - 1;
	5025	if (options.server) {
	5026	var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
	5027	var server: std.zig.Server = .{
	5028	.out = &stdout_writer.interface,
	5029	.in = undefined, // won't be receiving messages
	5030	};
5097	5031
5098		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed };
5099		const argv_index_seed = make_argv.items.len - 1;
	5032	try comp.update(root_prog_node);
5100	5033
5101		configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined };
5102		const conf_argv_index_build_root = configure_argv.items.len - 1;
	5034	var error_bundle = try comp.getAllErrorsAlloc();
	5035	defer error_bundle.deinit(comp.gpa);
	5036	if (error_bundle.errorMessageCount() > 0) {
	5037	try server.serveErrorBundle(error_bundle);
	5038	process.exit(2);
	5039	}
	5040	} else {
	5041	const color = Color.settingFromEnvironment(environ_map);
	5042	updateModule(comp, color, root_prog_node) catch \|err\| switch (err) {
	5043	error.CompileErrorsReported => process.exit(2),
	5044	else => \|e\| return e,
	5045	};
	5046	}
5103	5047
5104		var color: Color = Color.settingFromEnvironment(environ_map);
5105		var n_jobs: ?u32 = null;
	5048	const exe_path = try dirs.global_cache.join(arena, &.{
	5049	"o",
	5050	&Cache.binToHex(comp.digest.?),
	5051	comp.emit_bin.?,
	5052	});
	5053	child_argv.appendAssumeCapacity(exe_path);
	5054	}
5106	5055
5107		{
5108		var i: usize = 0;
5109		while (i < args.len) : (i += 1) {
5110		const arg = args[i];
5111		if (mem.startsWith(u8, arg, "-")) {
5112		try configure_argv.ensureUnusedCapacity(arena, 2);
5113
5114		if (mem.startsWith(u8, arg, "-D") or
5115		mem.startsWith(u8, arg, "-fsys=") or
5116		mem.startsWith(u8, arg, "-fno-sys=") or
5117		mem.startsWith(u8, arg, "--release=") or
5118		mem.eql(u8, arg, "--release"))
5119		{
5120		try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
5121		configure_argv.appendAssumeCapacity(arg);
5122		continue;
5123		} else if (mem.eql(u8, arg, "--system")) {
5124		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5125		i += 1;
5126		system_pkg_dir_path = args[i];
	5056	if (options.prepend_cmd) \|cmd\|
	5057	child_argv.appendAssumeCapacity(cmd);
	5058	if (options.prepend_zig_lib_dir_path)
	5059	child_argv.appendAssumeCapacity(try arena.print("--zig-lib={s}", .{dirs.zig_lib.path.?}));
	5060	if (options.prepend_zig_exe_path)
	5061	child_argv.appendAssumeCapacity(try arena.print("--zig={s}", .{self_exe_path}));
	5062	if (options.prepend_global_cache_path)
	5063	child_argv.appendAssumeCapacity(try arena.print("--global-cache={s}", .{dirs.global_cache.path.?}));
	5064	if (options.prepend_seed)
	5065	child_argv.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{randInt(io, u32)}));
5127	5066
5128		try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
5129		configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path.
5130		continue;
5131		} else if (mem.cutPrefix(u8, arg, "--color=")) \|rest\| {
5132		color = stringToEnum(Color, rest) orelse
5133		fatal("expected --color=[auto\|on\|off]; found {q}", .{arg});
	5067	child_argv.appendSliceAssumeCapacity(args);
5134	5068
5135		try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
5136		configure_argv.appendAssumeCapacity(arg);
5137		continue;
5138		} else if (mem.eql(u8, arg, "--cache-poison")) {
5139		cache_poison = .poisoned;
5140		configure_argv.appendAssumeCapacity("--cache-poison=poisoned");
5141		continue;
5142		} else if (mem.cutPrefix(u8, arg, "--cache-poison=")) \|rest\| {
5143		// Allow the configurer process to report parse failure.
5144		if (stringToEnum(std.Build.Graph.CachePoison, rest)) \|poison\| {
5145		cache_poison = poison;
5146		}
5147		configure_argv.appendAssumeCapacity(arg);
5148		continue;
5149		} else if (mem.eql(u8, arg, "--verbose")) {
5150		// Intentionally is added both to make and configure but
5151		// does not go into the cache hash.
5152		configure_argv.appendAssumeCapacity(arg);
5153		} else if (mem.eql(u8, arg, "--search-prefix")) {
5154		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5155		i += 1;
5156		// This argument is cache poisonous: it does not go into
5157		// the cache and configurer must set the poison bit when
5158		// choosing to observe it.
5159		configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, args[i] };
5160		(try make_argv.addManyAsArray(arena, 2)).* = .{ arg, args[i] };
5161		continue;
5162		} else if (mem.eql(u8, arg, "--build-file")) {
5163		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5164		i += 1;
5165		build_file = args[i];
5166		continue;
5167		} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
5168		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5169		i += 1;
5170		override_lib_dir = args[i];
5171		continue;
5172		} else if (mem.eql(u8, arg, "--cache-dir")) {
5173		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5174		i += 1;
5175		override_local_cache_dir = args[i];
5176		continue;
5177		} else if (mem.eql(u8, arg, "--pkg-dir")) {
5178		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5179		i += 1;
5180		override_pkg_dir = args[i];
5181		continue;
5182		} else if (mem.eql(u8, arg, "--global-cache-dir")) {
5183		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5184		i += 1;
5185		override_global_cache_dir = args[i];
5186		continue;
5187		} else if (mem.eql(u8, arg, "--print-configuration-path")) {
5188		print_configuration_path = true;
5189		continue;
5190		} else if (mem.eql(u8, arg, "-freference-trace")) {
5191		reference_trace = 256;
5192		} else if (mem.eql(u8, arg, "--fetch")) {
5193		fetch_only = true;
5194		} else if (mem.cutPrefix(u8, arg, "--fetch=")) \|sub_arg\| {
5195		fetch_only = true;
5196		fetch_mode = stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse
5197		fatal("expected [needed\|all] after \"--fetch=\", found: {s}", .{sub_arg});
5198		} else if (mem.cutPrefix(u8, arg, "--fork=")) \|sub_arg\| {
5199		try forks.append(arena, .init(sub_arg));
5200		continue;
5201		} else if (mem.eql(u8, arg, "--fork")) {
5202		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5203		i += 1;
5204		try forks.append(arena, .init(args[i]));
5205		continue;
5206		} else if (mem.cutPrefix(u8, arg, "-freference-trace=")) \|num\| {
5207		reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch \|err\| {
5208		fatal("unable to parse reference_trace count {q}: {t}", .{ num, err });
5209		};
5210		} else if (mem.eql(u8, arg, "-fno-reference-trace")) {
5211		reference_trace = null;
5212		} else if (mem.cutPrefix(u8, arg, "--maker-opt=")) \|rest\| {
5213		maker_optimize_mode = parseOptimizeMode(rest);
5214		continue;
5215		} else if (mem.eql(u8, arg, "--debug-log")) {
5216		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5217		try make_argv.appendSlice(arena, args[i .. i + 2]);
5218		i += 1;
5219		try addDebugLog(arena, args[i]);
5220		continue;
5221		} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
5222		if (build_options.enable_debug_extensions) {
5223		debug_compile_errors = true;
5224		} else {
5225		warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{});
5226		}
5227		} else if (mem.eql(u8, arg, "--debug-target")) {
5228		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5229		i += 1;
5230		if (build_options.enable_debug_extensions) {
5231		debug_target = args[i];
5232		} else {
5233		warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{});
5234		}
5235		continue;
5236		} else if (mem.eql(u8, arg, "--debug-libc")) {
5237		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5238		i += 1;
5239		if (build_options.enable_debug_extensions) {
5240		debug_libc_paths_file = args[i];
5241		} else {
5242		warn("Zig was compiled without debug extensions. --debug-libc has no effect.", .{});
5243		}
5244		continue;
5245		} else if (mem.eql(u8, arg, "--verbose-link")) {
5246		verbose_link = true;
5247		} else if (mem.eql(u8, arg, "--verbose-cc")) {
5248		verbose_cc = true;
5249		} else if (mem.eql(u8, arg, "--verbose-air")) {
5250		verbose_air = true;
5251		} else if (mem.eql(u8, arg, "--verbose-intern-pool")) {
5252		verbose_intern_pool = true;
5253		} else if (mem.eql(u8, arg, "--verbose-generic-instances")) {
5254		verbose_generic_instances = true;
5255		} else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
5256		verbose_llvm_ir = "-";
5257		} else if (mem.cutPrefix(u8, arg, "--verbose-llvm-ir=")) \|rest\| {
5258		verbose_llvm_ir = rest;
5259		} else if (mem.cutPrefix(u8, arg, "--verbose-llvm-bc=")) \|rest\| {
5260		verbose_llvm_bc = rest;
5261		} else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
5262		verbose_llvm_cpu_features = true;
5263		} else if (mem.cutPrefix(u8, arg, "-j")) \|str\| {
5264		const num = std.fmt.parseUnsigned(u32, str, 10) catch \|err\|
5265		fatal("unable to parse jobs count {s}: {t}", .{ str, err });
5266		if (num < 1) {
5267		fatal("number of jobs must be at least 1", .{});
5268		}
5269		n_jobs = num;
5270		} else if (mem.eql(u8, arg, "--seed")) {
5271		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5272		i += 1;
5273		make_argv.items[argv_index_seed] = args[i];
5274		continue;
5275		} else if (mem.eql(u8, arg, "--")) {
5276		try make_argv.appendSlice(arena, args[i..]);
5277		break;
5278		}
5279		}
5280		try make_argv.append(arena, arg);
5281		}
	5069	if (EnvVar.ZIG_VERBOSE_CMD.isSet(environ_map)) {
	5070	const cmd: std.zig.SubprocessCommand = .{
	5071	.argv = child_argv.items,
	5072	};
	5073	std.log.info("{f}", .{cmd});
5282	5074	}
5283	5075
5284		const root_prog_node = std.Progress.start(io, .{
5285		.disable_printing = (color == .off),
5286		.root_name = "",
5287		});
5288		defer root_prog_node.end();
5289
5290		process.raiseFileDescriptorLimit();
5291
5292		const cwd_path = introspect.getResolvedCwd(io, arena) catch \|err\|
5293		fatal("failed to get current directory path: {t}", .{err});
5294
5295		const build_root = try findBuildRoot(arena, io, .{
5296		.cwd_path = cwd_path,
5297		.build_file = build_file,
5298		});
5299
5300		{
5301		// This `init` calls `fatal` on error.
5302		var dirs: Compilation.Directories = .init(
5303		arena,
5304		io,
5305		override_lib_dir,
5306		override_global_cache_dir,
5307		.{ .override = path: {
5308		if (override_local_cache_dir) \|d\| break :path d;
5309		break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename});
5310		} },
5311		.empty,
5312		self_exe_path,
5313		environ_map,
5314		cwd_path,
5315		);
5316		defer dirs.deinit(io);
5317
5318		const thread_limit = @min(
5319		@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
5320		std.math.maxInt(Zcu.PerThread.IdBacking),
5321		);
5322		try setThreadLimit(arena, thread_limit);
5323
5324		// Cache lookup for configure options. If we get a match, we can skip
5325		// execution of the configure script. If not, we get the file path to pass
5326		// to the configure process.
5327		var local_cache: Cache = .{
5328		.gpa = gpa,
5329		.io = io,
5330		.manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}),
5331		.cwd = cwd_path,
5332		};
5333		local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() });
5334		local_cache.addPrefix(dirs.zig_lib);
5335		local_cache.addPrefix(dirs.local_cache);
5336		local_cache.addPrefix(dirs.global_cache);
5337		defer local_cache.manifest_dir.close(io);
5338
5339		var config_man = local_cache.obtain();
5340		defer config_man.deinit();
5341		config_man.hash.addBytes(build_options.version);
5342
5343		for (cached_passthru_configure.items) \|i\|
5344		config_man.hash.addBytes(configure_argv.items[i]);
5345
5346		// Prevents a `zig build` from getting a false positive cache hit following
5347		// a `zig build --cache-poison=ignored`.
5348		config_man.hash.add(cache_poison == .ignored);
5349
5350		// Normally the build runner is compiled for the host target but here is
5351		// some code to help when debugging edits to the build runner so that you
5352		// can make sure it compiles successfully on other targets.
5353		const resolved_target: Package.Module.ResolvedTarget = t: {
5354		if (build_options.enable_debug_extensions) {
5355		if (debug_target) \|triple\| {
5356		const target_query = try std.Target.Query.parse(.{
5357		.arch_os_abi = triple,
5358		});
5359		config_man.hash.addBytes(triple);
5360		break :t .{
5361		.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
5362		.is_native_os = false,
5363		.is_native_abi = false,
5364		.is_explicit_dynamic_linker = false,
5365		};
5366		}
5367		}
5368		break :t .{
5369		.result = std.zig.resolveTargetQueryOrFatal(io, .{}),
5370		.is_native_os = true,
5371		.is_native_abi = true,
5372		.is_explicit_dynamic_linker = false,
5373		};
5374		};
5375
5376		// Likewise, `--debug-libc` allows overriding the libc installation.
5377		const libc_installation: ?*const LibCInstallation = lci: {
5378		const paths_file = debug_libc_paths_file orelse break :lci null;
5379		if (!build_options.enable_debug_extensions) unreachable;
5380		const lci = try arena.create(LibCInstallation);
5381		lci.* = try .parse(arena, io, paths_file, &resolved_target.result);
5382		LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi);
5383		break :lci lci;
5384		};
5385
5386		// Kick off an optimized compilation of the make runner.
5387		var make_runner_task = if (print_configuration_path) undefined else io.async(compileMakeRunner, .{ gpa, arena, io, .{
5388		.dirs = .{
5389		.cwd = dirs.cwd,
5390		.zig_lib = dirs.zig_lib,
5391		.global_cache = dirs.global_cache,
5392		.local_cache = dirs.global_cache,
5393		},
5394		.environ_map = environ_map,
5395		.parent_prog_node = root_prog_node,
5396		.resolved_target = resolved_target,
5397		.libc_installation = libc_installation,
5398		.thread_limit = thread_limit,
5399		.self_exe_path = self_exe_path,
5400		.color = color,
5401		.reference_trace = reference_trace,
5402		.optimize_mode = maker_optimize_mode,
5403		} });
5404		defer _ = if (!print_configuration_path) make_runner_task.cancel(io) catch {};
5405
5406		const pkg_root: Path = if (override_pkg_dir) \|p\|
5407		.initCwd(p)
5408		else if (system_pkg_dir_path) \|p\|
5409		.initCwd(p)
5410		else
5411		.{
5412		.root_dir = build_root.directory,
5413		.sub_path = "zig-pkg",
5414		};
5415
5416		make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path;
5417		make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path;
5418		make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path;
5419		make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path;
5420
5421		configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path;
5422
5423		// Dummy http client that is not actually used when fetch_command is unsupported.
5424		// Prevents bootstrap from depending on a bunch of unnecessary stuff.
5425		var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct {
5426		allocator: Allocator,
5427		io: Io,
5428		fn deinit(_: @This()) void {}
5429		} = .{ .allocator = gpa, .io = io };
5430		defer http_client.deinit();
5431
5432		var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
5433		var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
5434
5435		{
5436		// Populate fork_set.
5437		var group: Io.Group = .init;
5438		defer group.cancel(io);
5439
5440		for (forks.items) \|*fork\|
5441		group.async(io, Fork.load, .{ io, gpa, fork, color });
5442
5443		try group.await(io);
5444
5445		for (forks.items) \|*fork\| {
5446		if (fork.failed) process.exit(1);
5447		try fork_set.put(arena, .{
5448		.path = fork.path,
5449		.manifest_ast = fork.manifest_ast,
5450		.manifest = fork.manifest,
5451		.uses = 0,
5452		}, {});
5453		}
5454		}
5455		defer Fork.deinitList(forks.items);
5456
5457		// This loop is re-evaluated when the build script exits with an indication that it
5458		// could not continue due to missing lazy dependencies.
5459		const configuration_path: Path, const poisoned: bool = cp: while (true) {
5460		// We want to release all the locks before executing the child process, so we make a nice
5461		// big block here to ensure the cleanup gets run when we extract out our argv.
5462		{
5463		const main_mod_paths: Package.Module.CreateOptions.Paths = .{
5464		.root = try .fromRoot(arena, dirs, .zig_lib, "compiler"),
5465		.root_src_path = "configurer.zig",
5466		};
5467
5468		const config = try Compilation.Config.resolve(.{
5469		.output_mode = .Exe,
5470		.resolved_target = resolved_target,
5471		.have_zcu = true,
5472		.emit_bin = true,
5473		.is_test = false,
5474		});
5475
5476		const root_mod = try Package.Module.create(arena, .{
5477		.paths = main_mod_paths,
5478		.fully_qualified_name = "root",
5479		.cc_argv = &.{},
5480		.inherited = .{
5481		.resolved_target = resolved_target,
5482		.single_threaded = true,
5483		},
5484		.global = config,
5485		.parent = null,
5486		});
5487
5488		const build_mod = try Package.Module.create(arena, .{
5489		.paths = .{
5490		.root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}),
5491		.root_src_path = build_root.build_zig_basename,
5492		},
5493		.fully_qualified_name = "root.@build",
5494		.cc_argv = &.{},
5495		.inherited = .{},
5496		.global = config,
5497		.parent = root_mod,
5498		});
5499
5500		if (dev.env.supports(.fetch_command)) {
5501		const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);
5502		defer fetch_prog_node.end();
5503
5504		// Reset fork match counts.
5505		for (fork_set.keys()) \|*fork\| fork.uses = 0;
5506
5507		var job_queue: Package.Fetch.JobQueue = .{
5508		.io = io,
5509		.http_client = &http_client,
5510		.global_cache = dirs.global_cache,
5511		.local_storage = &.{
5512		.cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5513		.pkg_root = pkg_root,
5514		},
5515		.recursive = true,
5516		.debug_hash = false,
5517		.unlazy_set = unlazy_set,
5518		.fork_set = fork_set,
5519		.mode = fetch_mode,
5520		.prog_node = fetch_prog_node,
5521		.read_only = system_pkg_dir_path != null,
5522		};
5523		defer job_queue.deinit();
5524
5525		if (system_pkg_dir_path == null) {
5526		try http_client.initDefaultProxies(arena, environ_map);
5527		}
5528
5529		try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
5530		try job_queue.table.ensureUnusedCapacity(gpa, 1);
5531
5532		const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory };
5533
5534		var fetch: Package.Fetch = .{
5535		.arena = std.heap.ArenaAllocator.init(gpa),
5536		.location = .{ .relative_path = phantom_package_root },
5537		.location_tok = 0,
5538		.hash_tok = .none,
5539		.name_tok = 0,
5540		.lazy_status = .eager,
5541		.remote_package_root = phantom_package_root,
5542		.parent_package_root = phantom_package_root,
5543		.parent_manifest_ast = null,
5544		.prog_node = fetch_prog_node,
5545		.job_queue = &job_queue,
5546		.omit_missing_hash_error = true,
5547		.allow_missing_paths_field = false,
5548		.use_latest_commit = false,
5549
5550		.package_root = undefined,
5551		.error_bundle = undefined,
5552		.manifest = undefined,
5553		.manifest_ast = undefined,
5554		.have_manifest = false,
5555		.computed_hash = undefined,
5556		.has_build_zig = true,
5557		.oom_flag = false,
5558		.latest_commit = null,
5559
5560		.module = build_mod,
5561		};
5562
5563		job_queue.all_fetches.appendAssumeCapacity(&fetch);
5564
5565		job_queue.table.putAssumeCapacityNoClobber(
5566		Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache),
5567		&fetch,
5568		);
5569
5570		job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
5571		try job_queue.group.await(io);
5572
5573		{
5574		// Ensure that forks were actually used. This is done
5575		// before printing manifest errors because using a fork can
5576		// prevent them.
5577		var any_unused = false;
5578		for (fork_set.keys()) \|*fork\| {
5579		if (fork.uses == 0) {
5580		std.log.err("fork {f} matched no {s} packages", .{
5581		fork.path, fork.manifest.name,
5582		});
5583		any_unused = true;
5584		} else {
5585		std.log.info("fork {f} matched {d} {s} packages", .{
5586		fork.path, fork.uses, fork.manifest.name,
5587		});
5588		}
5589		}
5590		if (any_unused) process.exit(1);
5591		}
5592
5593		try job_queue.consolidateErrors();
5594
5595		if (fetch.error_bundle.root_list.items.len > 0) {
5596		var errors = try fetch.error_bundle.toOwnedBundle("");
5597		errors.renderToStderr(io, .{}, color) catch {};
5598		process.exit(1);
5599		}
5600
5601		if (fetch_only) return cleanExit(io);
5602
5603		var source_buf = std.array_list.Managed(u8).init(gpa);
5604		defer source_buf.deinit();
5605		try job_queue.createDependenciesSource(&source_buf);
5606		const deps_mod = try createDependenciesModule(
5607		arena,
5608		io,
5609		source_buf.items,
5610		root_mod,
5611		dirs,
5612		config,
5613		);
5614
5615		{
5616		// We need a Module for each package's build.zig.
5617		const hashes = job_queue.table.keys();
5618		const fetches = job_queue.table.values();
5619		try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5620		for (hashes, fetches) \|*hash, f\| {
5621		if (f == &fetch) {
5622		// The first one is a dummy package for the current project.
5623		continue;
5624		}
5625		if (!f.has_build_zig)
5626		continue;
5627		const hash_slice = hash.toSlice();
5628		const mod_root_path = try f.package_root.toString(arena);
5629		const m = try Package.Module.create(arena, .{
5630		.paths = .{
5631		.root = try .fromUnresolved(arena, dirs, &.{mod_root_path}),
5632		.root_src_path = Package.build_zig_basename,
5633		},
5634		.fully_qualified_name = try std.fmt.allocPrint(
5635		arena,
5636		"root.@dependencies.{s}",
5637		.{hash_slice},
5638		),
5639		.cc_argv = &.{},
5640		.inherited = .{},
5641		.global = config,
5642		.parent = root_mod,
5643		});
5644		const hash_cloned = try arena.dupe(u8, hash_slice);
5645		deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5646		f.module = m;
5647		}
5648
5649		// Each build.zig module needs access to each of its
5650		// dependencies' build.zig modules by name.
5651		for (fetches) \|f\| {
5652		const mod = f.module orelse continue;
5653		if (!f.have_manifest) continue;
5654		const man = &f.manifest;
5655		const dep_names = man.dependencies.keys();
5656		try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
5657		for (dep_names, man.dependencies.values()) \|name, dep\| {
5658		const dep_digest = Package.Fetch.depDigest(
5659		f.package_root,
5660		dirs.global_cache,
5661		dep,
5662		) orelse continue;
5663		const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue;
5664		const name_cloned = try arena.dupe(u8, name);
5665		mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
5666		}
5667		}
5668		}
5669		} else try createEmptyDependenciesModule(
5670		arena,
5671		io,
5672		root_mod,
5673		dirs,
5674		config,
5675		);
5676
5677		const compile_prog_node = root_prog_node.start("Compile Configure Script", 0);
5678		defer compile_prog_node.end();
5679
5680		try root_mod.deps.put(arena, "@build", build_mod);
5681
5682		var create_diag: Compilation.CreateDiagnostic = undefined;
5683		const comp = Compilation.create(gpa, arena, io, &create_diag, .{
5684		.libc_installation = libc_installation,
5685		.dirs = dirs,
5686		.root_name = "configure",
5687		.config = config,
5688		.root_mod = root_mod,
5689		.main_mod = build_mod,
5690		.emit_bin = .yes_cache,
5691		.self_exe_path = self_exe_path,
5692		.thread_limit = thread_limit,
5693		.verbose_cc = verbose_cc,
5694		.verbose_link = verbose_link,
5695		.verbose_air = verbose_air,
5696		.verbose_intern_pool = verbose_intern_pool,
5697		.verbose_generic_instances = verbose_generic_instances,
5698		.verbose_llvm_ir = verbose_llvm_ir,
5699		.verbose_llvm_bc = verbose_llvm_bc,
5700		.verbose_llvm_cpu_features = verbose_llvm_cpu_features,
5701		.cache_mode = .whole,
5702		.reference_trace = reference_trace,
5703		.debug_compile_errors = debug_compile_errors,
5704		.environ_map = environ_map,
5705		}) catch \|err\| switch (err) {
5706		error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}),
5707		else => \|e\| fatal("failed to create compilation: {t}", .{e}),
5708		};
5709		defer comp.destroy();
5710
5711		updateModule(comp, color, compile_prog_node) catch \|err\| switch (err) {
5712		error.CompileErrorsReported => process.exit(2),
5713		else => \|e\| return e,
5714		};
5715
5716		// Since incremental compilation isn't done yet, we use cache_mode = whole
5717		// above, and thus the output file is already closed.
5718		//try comp.makeBinFileExecutable();
5719		const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?);
5720		const exe_path: Path = .{
5721		.root_dir = dirs.local_cache,
5722		.sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }),
5723		};
5724		_ = try config_man.addFilePath(exe_path, null);
5725		configure_argv.items[0] = try exe_path.toString(arena);
5726
5727		switch (cache_poison) {
5728		.pure, .disallowed, .ignored => if (try config_man.hit()) {
5729		const digest = config_man.final();
5730		break :cp .{
5731		.{
5732		.root_dir = dirs.local_cache,
5733		.sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}),
5734		},
5735		false,
5736		};
5737		},
5738		.poisoned => {}, // Don't bother checking for cache hit.
5739		}
5740		}
5741
5742		if (!process.can_spawn) {
5743		const cmd = try std.mem.join(arena, " ", configure_argv.items);
5744		fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd });
5745		}
5746
5747		const rand_int = randInt(io, u64);
5748		const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
5749		const config_tmp_path: Path = .{
5750		.root_dir = dirs.local_cache,
5751		.sub_path = tmp_dir_sub_path,
5752		};
5753		const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile(
5754		io,
5755		config_tmp_path.sub_path,
5756		.{ .read = true, .exclusive = true },
5757		);
5758		defer config_tmp_file.close(io);
5759
5760		const term = term: {
5761		const child_node = root_prog_node.start("Run Configure Script", 0);
5762		defer child_node.end();
5763		var child = std.process.spawn(io, .{
5764		.argv = configure_argv.items,
5765		.stdout = .{ .file = config_tmp_file },
5766		.progress_node = child_node,
5767		}) catch \|err\| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err });
5768		defer child.kill(io);
5769		break :term child.wait(io) catch \|err\|
5770		fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err });
5771		};
5772		if (!term.success()) {
5773		// Failure to produce the configuration file.
5774		const cmd = try std.mem.join(arena, " ", configure_argv.items);
5775		fatal("the following configure command {f}:\n{s}", .{ term, cmd });
5776		}
5777		// Even though the file is designed to be sent directly to make
5778		// runner, we must load it now because:
5779		// * If it contains additional file dependencies, we need to
5780		// add them to `config_man` before obtaining the final digest.
5781		// * If it contains a set of lazy packages that need to be
5782		// fetched, we need to fetch those now and re-run configure.
5783		var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch \|err\|
5784		fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err });
5785
5786		if (configuration.unlazy_deps.len != 0) {
5787		if (!dev.env.supports(.fetch_command)) process.exit(1);
5788		var any_errors = false;
5789		for (configuration.unlazy_deps) \|hash_string\| {
5790		const hash = hash_string.slice(&configuration);
5791		assert(hash.len != 0);
5792		if (hash.len > Package.Hash.max_len) {
5793		std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
5794		any_errors = true;
5795		continue;
5796		}
5797		try unlazy_set.put(arena, .fromSlice(hash), {});
5798		}
5799		if (any_errors) process.exit(1);
5800		if (system_pkg_dir_path) \|p\| {
5801		// In this mode, the system needs to provide these packages; they
5802		// cannot be fetched by Zig.
5803		const s = fs.path.sep_str;
5804		for (unlazy_set.keys()) \|*hash\| {
5805		std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
5806		}
5807		std.log.info("remote package fetching disabled due to --system mode", .{});
5808		std.log.info("dependencies might be avoidable depending on build configuration", .{});
5809		process.exit(1);
5810		}
5811		continue :cp;
5812		}
5813
5814		for (configuration.path_deps_base, configuration.path_deps_sub) \|base, sub\| {
5815		const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub };
5816		try config_man.addPathPost(conf_path.toCachePath(&configuration, arena));
5817		}
5818
5819		// If it is poisoned, there is no point in moving it to cached
5820		// location. Just leave it in the tmp directory.
5821		if (configuration.poisoned) {
5822		break :cp .{ config_tmp_path, true };
5823		} else {
5824		const digest = config_man.final();
5825		const final_path: Path = .{
5826		.root_dir = dirs.local_cache,
5827		.sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}),
5828		};
5829		Io.Dir.rename(
5830		config_tmp_path.root_dir.handle,
5831		config_tmp_path.sub_path,
5832		final_path.root_dir.handle,
5833		final_path.sub_path,
5834		io,
5835		) catch \|err\| retry: {
5836		const e = switch (err) {
5837		error.FileNotFound => e: {
5838		const dir_path = final_path.dirname().?;
5839		dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch \|e\|
5840		fatal("failed to create directory {f}: {t}", .{ dir_path, e });
5841		if (Io.Dir.rename(
5842		config_tmp_path.root_dir.handle,
5843		config_tmp_path.sub_path,
5844		final_path.root_dir.handle,
5845		final_path.sub_path,
5846		io,
5847		)) \|_\| break :retry else \|e\| break :e e;
5848		},
5849		else => \|e\| e,
5850		};
5851		fatal("failed to rename configuration file from {f} into {f}: {t}", .{
5852		config_tmp_path, final_path, e,
5853		});
5854		};
5855		config_man.writeManifest() catch \|err\| warn("failed to write cache manifest: {t}", .{err});
5856		break :cp .{ final_path, false };
5857		}
5858		};
5859
5860		{
5861		// Release all file system locks just before running the maker process.
5862		var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null;
5863		defer if (configuration_lock) \|*l\| l.release(io);
5864
5865		if (print_configuration_path) {
5866		var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer);
5867		stdout_writer.interface.print("{f}\n", .{configuration_path}) catch
5868		fatal("failed printing cache file path: {t}", .{stdout_writer.err.?});
5869		stdout_writer.flush() catch \|err\|
5870		fatal("failed printing cache file path: {t}", .{err});
5871		return cleanExit(io);
5872		}
5873		const make_runner = make_runner_task.await(io) catch \|err\| fatal("failed compiling maker: {t}", .{err});
5874
5875		make_argv.items[0] = try make_runner.exe_path.toString(arena);
5876		make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena);
5877		}
5878		}
5879
5880		if (!process.can_spawn) {
5881		const cmd = try std.mem.join(arena, " ", make_argv.items);
5882		fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{
5883		native_os, cmd,
5884		});
5885		}
5886
5887		const term = term: {
5888		_ = try io.lockStderr(&.{}, .no_color);
5889		defer io.unlockStderr();
5890		var child = std.process.spawn(io, .{
5891		.argv = make_argv.items,
5892		}) catch \|err\| fatal("failed spawning maker {s}: {t}", .{ make_argv.items[0], err });
5893		defer child.kill(io);
5894		break :term child.wait(io) catch \|err\|
5895		fatal("failed waiting on maker {s}: {t}", .{ make_argv.items[0], err });
5896		};
5897		if (term.success()) return cleanExit(io);
5898		const cmd = try std.mem.join(arena, " ", make_argv.items);
5899		fatal("the following maker command {f}:\n{s}", .{ term, cmd });
5900		}
5901
5902		const MakeRunner = struct {
5903		exe_path: Path,
5904
5905		const Options = struct {
5906		environ_map: *const process.Environ.Map,
5907		dirs: Compilation.Directories,
5908		parent_prog_node: std.Progress.Node,
5909		resolved_target: Package.Module.ResolvedTarget,
5910		libc_installation: ?*const LibCInstallation,
5911		self_exe_path: []const u8,
5912		thread_limit: usize,
5913		color: Color,
5914		reference_trace: ?u32,
5915		optimize_mode: std.builtin.OptimizeMode,
5916		};
5917		};
5918
5919		fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner {
5920		const compile_prog_node = options.parent_prog_node.start("Compiling Maker (first time setup)", 0);
5921		defer compile_prog_node.end();
5922
5923		const strip = options.optimize_mode != .Debug;
5924
5925		const main_mod_paths: Package.Module.CreateOptions.Paths = .{
5926		.root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"),
5927		.root_src_path = "Maker.zig",
5928		};
5929
5930		const config = try Compilation.Config.resolve(.{
5931		.output_mode = .Exe,
5932		.root_strip = strip,
5933		.root_optimize_mode = options.optimize_mode,
5934		.resolved_target = options.resolved_target,
5935		.have_zcu = true,
5936		.emit_bin = true,
5937		.is_test = false,
5938		});
5939
5940		const root_mod = try Package.Module.create(arena, .{
5941		.paths = main_mod_paths,
5942		.fully_qualified_name = "root",
5943		.cc_argv = &.{},
5944		.inherited = .{
5945		.resolved_target = options.resolved_target,
5946		.optimize_mode = options.optimize_mode,
5947		.strip = strip,
5948		},
5949		.global = config,
5950		.parent = null,
5951		});
5952
5953		var create_diag: Compilation.CreateDiagnostic = undefined;
5954		const comp = Compilation.create(gpa, arena, io, &create_diag, .{
5955		.dirs = options.dirs,
5956		.root_name = "maker",
5957		.config = config,
5958		.root_mod = root_mod,
5959		.main_mod = root_mod,
5960		.emit_bin = .yes_cache,
5961		.self_exe_path = options.self_exe_path,
5962		.thread_limit = options.thread_limit,
5963		.cache_mode = .whole,
5964		.environ_map = options.environ_map,
5965		.reference_trace = options.reference_trace,
5966		}) catch \|err\| switch (err) {
5967		error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}),
5968		error.Canceled => \|e\| return e,
5969		else => \|e\| fatal("failed to create compilation: {t}", .{e}),
5970		};
5971		defer comp.destroy();
5972
5973		try updateModule(comp, options.color, compile_prog_node);
5974
5975		const exe_path: Path = .{
5976		.root_dir = options.dirs.global_cache,
5977		.sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{
5978		&Cache.binToHex(comp.digest.?), comp.emit_bin.?,
5979		}),
5980		};
5981
5982		return .{
5983		.exe_path = exe_path,
5984		};
5985		}
5986
5987		const Fork = struct {
5988		path: Path,
5989		manifest_ast: std.zig.Ast,
5990		manifest: Package.Manifest,
5991		error_bundle: std.zig.ErrorBundle.Wip,
5992		failed: bool,
5993		arena_allocator: std.heap.ArenaAllocator,
5994
5995		fn init(cwd_relative_path: []const u8) Fork {
5996		return .{
5997		.manifest_ast = undefined,
5998		.manifest = undefined,
5999		.error_bundle = undefined,
6000		.arena_allocator = undefined,
6001		.path = .{
6002		.root_dir = .cwd(),
6003		.sub_path = cwd_relative_path,
6004		},
6005		.failed = false,
6006		};
6007		}
6008
6009		fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void {
6010		loadFallible(io, gpa, fork, color) catch \|err\| switch (err) {
6011		error.Canceled => \|e\| return e,
6012		error.AlreadyReported => fork.failed = true,
6013		else => \|e\| {
6014		std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e });
6015		fork.failed = true;
6016		},
6017		};
6018		}
6019
6020		fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void {
6021		fork.arena_allocator = .init(gpa);
6022		const arena = fork.arena_allocator.allocator();
6023
6024		var error_bundle: std.zig.ErrorBundle.Wip = undefined;
6025		try error_bundle.init(gpa);
6026		defer error_bundle.deinit();
6027
6028		const manifest_path = try fork.path.join(arena, Package.Manifest.basename);
6029
6030		Package.Manifest.load(
6031		io,
6032		arena,
6033		manifest_path,
6034		&fork.manifest_ast,
6035		&error_bundle,
6036		&fork.manifest,
6037		true,
6038		) catch \|err\| switch (err) {
6039		error.Canceled => \|e\| return e,
6040		error.ErrorsBundled => {
6041		assert(error_bundle.root_list.items.len > 0);
6042		var errors = try error_bundle.toOwnedBundle("");
6043		errors.renderToStderr(io, .{}, color) catch {};
6044		return error.AlreadyReported;
6045		},
6046		else => \|e\| {
6047		std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e });
6048		return error.AlreadyReported;
6049		},
6050		};
6051		}
6052
6053		fn deinitList(forks: []Fork) void {
6054		for (forks) \|*fork\| fork.arena_allocator.deinit();
6055		}
6056		};
6057
6058		const JitCmdOptions = struct {
6059		cmd_name: []const u8,
6060		root_src_path: []const u8,
6061		prepend_zig_lib_dir_path: bool = false,
6062		prepend_global_cache_path: bool = false,
6063		prepend_zig_exe_path: bool = false,
6064		depend_on_aro: bool = false,
6065		capture: ?*[]u8 = null,
6066		/// Send error bundles via std.zig.Server over stdout
6067		server: bool = false,
6068		color: Color = .auto,
6069		};
6070
6071		fn jitCmd(
6072		gpa: Allocator,
6073		arena: Allocator,
6074		io: Io,
6075		args: []const []const u8,
6076		environ_map: *const process.Environ.Map,
6077		options: JitCmdOptions,
6078		) !void {
6079		dev.check(.jit_command);
6080
6081		const root_prog_node = std.Progress.start(io, .{
6082		.disable_printing = (options.color == .off),
6083		});
6084		defer root_prog_node.end();
6085
6086		const thread_limit = @min(
6087		@max(std.Thread.getCpuCount() catch 1, 1),
6088		std.math.maxInt(Zcu.PerThread.IdBacking),
6089		);
6090		try setThreadLimit(arena, thread_limit);
6091
6092		return jitCmdInner(gpa, arena, io, args, environ_map, root_prog_node, thread_limit, options);
6093		}
6094
6095		fn jitCmdInner(
6096		gpa: Allocator,
6097		arena: Allocator,
6098		io: Io,
6099		args: []const []const u8,
6100		environ_map: *const process.Environ.Map,
6101		root_prog_node: std.Progress.Node,
6102		thread_limit: usize,
6103		options: JitCmdOptions,
6104		) !void {
6105		const target_query: std.Target.Query = .{};
6106		const resolved_target: Package.Module.ResolvedTarget = .{
6107		.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
6108		.is_native_os = true,
6109		.is_native_abi = true,
6110		.is_explicit_dynamic_linker = false,
6111		};
6112
6113		const self_exe_path = process.executablePathAlloc(io, arena) catch \|err\|
6114		fatal("unable to find self exe path: {t}", .{err});
6115
6116		const optimize_mode: std.lang.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map))
6117		.Debug
6118		else
6119		.ReleaseFast;
6120		const strip = optimize_mode != .Debug;
6121		const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
6122		const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
6123
6124		const cwd_path = try introspect.getResolvedCwd(io, arena);
6125
6126		// This `init` calls `fatal` on error.
6127		var dirs: Compilation.Directories = .init(
6128		arena,
6129		io,
6130		override_lib_dir,
6131		override_global_cache_dir,
6132		.global,
6133		preopens,
6134		self_exe_path,
6135		environ_map,
6136		cwd_path,
6137		);
6138		defer dirs.deinit(io);
6139
6140		var child_argv: std.ArrayList([]const u8) = .empty;
6141		try child_argv.ensureUnusedCapacity(arena, args.len + 4);
6142
6143		// We want to release all the locks before executing the child process, so we make a nice
6144		// big block here to ensure the cleanup gets run when we extract out our argv.
6145		{
6146		const main_mod_paths: Package.Module.CreateOptions.Paths = .{
6147		.root = try .fromRoot(arena, dirs, .zig_lib, "compiler"),
6148		.root_src_path = options.root_src_path,
6149		};
6150
6151		const config = try Compilation.Config.resolve(.{
6152		.output_mode = .Exe,
6153		.root_strip = strip,
6154		.root_optimize_mode = optimize_mode,
6155		.resolved_target = resolved_target,
6156		.have_zcu = true,
6157		.emit_bin = true,
6158		.is_test = false,
6159		});
6160
6161		const root_mod = try Package.Module.create(arena, .{
6162		.paths = main_mod_paths,
6163		.fully_qualified_name = "root",
6164		.cc_argv = &.{},
6165		.inherited = .{
6166		.resolved_target = resolved_target,
6167		.optimize_mode = optimize_mode,
6168		.strip = strip,
6169		},
6170		.global = config,
6171		.parent = null,
6172		});
6173
6174		if (options.depend_on_aro) {
6175		const aro_mod = try Package.Module.create(arena, .{
6176		.paths = .{
6177		.root = try .fromRoot(arena, dirs, .zig_lib, "compiler/aro"),
6178		.root_src_path = "aro.zig",
6179		},
6180		.fully_qualified_name = "aro",
6181		.cc_argv = &.{},
6182		.inherited = .{
6183		.resolved_target = resolved_target,
6184		.optimize_mode = optimize_mode,
6185		.strip = strip,
6186		},
6187		.global = config,
6188		.parent = null,
6189		});
6190		try root_mod.deps.put(arena, "aro", aro_mod);
6191		}
6192
6193		var create_diag: Compilation.CreateDiagnostic = undefined;
6194		const comp = Compilation.create(gpa, arena, io, &create_diag, .{
6195		.dirs = dirs,
6196		.root_name = options.cmd_name,
6197		.config = config,
6198		.root_mod = root_mod,
6199		.main_mod = root_mod,
6200		.emit_bin = .yes_cache,
6201		.self_exe_path = self_exe_path,
6202		.thread_limit = thread_limit,
6203		.cache_mode = .whole,
6204		.environ_map = environ_map,
6205		}) catch \|err\| switch (err) {
6206		error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}),
6207		else => fatal("failed to create compilation: {t}", .{err}),
6208		};
6209		defer comp.destroy();
6210
6211		if (options.server) {
6212		var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
6213		var server: std.zig.Server = .{
6214		.out = &stdout_writer.interface,
6215		.in = undefined, // won't be receiving messages
6216		};
6217
6218		try comp.update(root_prog_node);
6219
6220		var error_bundle = try comp.getAllErrorsAlloc();
6221		defer error_bundle.deinit(comp.gpa);
6222		if (error_bundle.errorMessageCount() > 0) {
6223		try server.serveErrorBundle(error_bundle);
6224		process.exit(2);
6225		}
6226		} else {
6227		updateModule(comp, options.color, root_prog_node) catch \|err\| switch (err) {
6228		error.CompileErrorsReported => process.exit(2),
6229		else => \|e\| return e,
6230		};
6231		}
6232
6233		const exe_path = try dirs.global_cache.join(arena, &.{
6234		"o",
6235		&Cache.binToHex(comp.digest.?),
6236		comp.emit_bin.?,
6237		});
6238		child_argv.appendAssumeCapacity(exe_path);
6239		}
6240
6241		if (options.prepend_zig_lib_dir_path)
6242		child_argv.appendAssumeCapacity(dirs.zig_lib.path.?);
6243		if (options.prepend_zig_exe_path)
6244		child_argv.appendAssumeCapacity(self_exe_path);
6245		if (options.prepend_global_cache_path)
6246		child_argv.appendAssumeCapacity(dirs.global_cache.path.?);
6247
6248		child_argv.appendSliceAssumeCapacity(args);
6249
6250	5076	if (process.can_replace and options.capture == null) {
6251		if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) {
6252		const cmd = try std.mem.join(arena, " ", child_argv.items);
6253		std.debug.print("{s}\n", .{cmd});
6254		}
6255	5077	const err = process.replace(io, .{ .argv = child_argv.items, .environ_map = environ_map });
6256	5078	const cmd = try std.mem.join(arena, " ", child_argv.items);
6257		fatal("the following command failed to execve with '{t}':\n{s}", .{ err, cmd });
	5079	fatal("the following command failed to execve with {t}:\n{s}", .{ err, cmd });
6258	5080	}
6259	5081
6260	5082	if (!process.can_spawn) {
...	...	@@ -6264,7 +5086,7 @@ fn jitCmdInner(
6264	5086	});
6265	5087	}
6266	5088
6267		switch (t: {
	5089	const term = t: {
6268	5090	_ = try io.lockStderr(&.{}, .no_color);
6269	5091	defer io.unlockStderr();
6270	5092
...	...	@@ -6282,28 +5104,13 @@ fn jitCmdInner(
6282	5104	}
6283	5105
6284	5106	break :t try child.wait(io);
6285		}) {
6286		.exited => \|code\| {
6287		if (code == 0) {
6288		if (options.capture != null) return;
6289		return cleanExit(io);
6290		}
6291		const cmd = try std.mem.join(arena, " ", child_argv.items);
6292		fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
6293		},
6294		.signal => \|sig\| {
6295		const cmd = try std.mem.join(arena, " ", child_argv.items);
6296		fatal("the following build command terminated with signal {t}:\n{s}", .{ sig, cmd });
6297		},
6298		.stopped => \|sig\| {
6299		const cmd = try std.mem.join(arena, " ", child_argv.items);
6300		fatal("the following build command stopped with signal {t}:\n{s}", .{ sig, cmd });
6301		},
6302		.unknown => {
6303		const cmd = try std.mem.join(arena, " ", child_argv.items);
6304		fatal("the following build command crashed:\n{s}", .{cmd});
6305		},
	5107	};
	5108	if (term.success()) {
	5109	if (options.capture != null) return;
	5110	return cleanExit(io);
6306	5111	}
	5112	const cmd = try std.mem.join(arena, " ", child_argv.items);
	5113	fatal("the following build command {f}:\n{s}", .{ term, cmd });
6307	5114	}
6308	5115
6309	5116	const info_zen =
...	...	@@ -7193,667 +6000,6 @@ fn parseRcIncludes(arg: []const u8) std.zig.RcIncludes {
7193	6000	fatal("unsupported rc includes type: {q}", .{arg});
7194	6001	}
7195	6002
7196		const usage_fetch =
7197		\\Usage: zig fetch [options] <url>
7198		\\Usage: zig fetch [options] <path>
7199		\\
7200		\\ Copy a package into the global cache and print its hash.
7201		\\ <url> must point to one of the following:
7202		\\ - A git+http / git+https server for the package
7203		\\ - A tarball file (with or without compression) containing
7204		\\ package source
7205		\\ - A git bundle file containing package source
7206		\\
7207		\\Examples:
7208		\\
7209		\\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git
7210		\\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz
7211		\\
7212		\\Options:
7213		\\ -h, --help Print this help and exit
7214		\\ --global-cache-dir [path] Override path to global Zig cache directory
7215		\\ --cache-dir [path] Override path to local cache directory
7216		\\ --pkg-dir [path] Override path to local package directory
7217		\\ --debug-hash Print verbose hash information to stdout
7218		\\ --debug-log [scope] Enable printing debug/info log messages for scope
7219		\\ --save Add the fetched package to build.zig.zon
7220		\\ --save=[name] Add the fetched package to build.zig.zon as name
7221		\\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim
7222		\\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim
7223		\\
7224		;
7225
7226		fn cmdFetch(
7227		gpa: Allocator,
7228		arena: Allocator,
7229		io: Io,
7230		args: []const []const u8,
7231		environ_map: *process.Environ.Map,
7232		) !void {
7233		dev.check(.fetch_command);
7234
7235		const color: Color = Color.settingFromEnvironment(environ_map);
7236		var opt_path_or_url: ?[]const u8 = null;
7237		var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
7238		var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
7239		var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
7240		var debug_hash: bool = false;
7241		var save: union(enum) {
7242		no,
7243		yes: ?[]const u8,
7244		exact: ?[]const u8,
7245		} = .no;
7246
7247		{
7248		var i: usize = 0;
7249		while (i < args.len) : (i += 1) {
7250		const arg = args[i];
7251		if (mem.startsWith(u8, arg, "-")) {
7252		if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
7253		try Io.File.stdout().writeStreamingAll(io, usage_fetch);
7254		return cleanExit(io);
7255		} else if (mem.eql(u8, arg, "--global-cache-dir")) {
7256		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7257		i += 1;
7258		override_global_cache_dir = args[i];
7259		} else if (mem.eql(u8, arg, "--cache-dir")) {
7260		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7261		i += 1;
7262		override_local_cache_dir = args[i];
7263		} else if (mem.eql(u8, arg, "--pkg-dir")) {
7264		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7265		i += 1;
7266		override_pkg_dir = args[i];
7267		} else if (mem.eql(u8, arg, "--debug-hash")) {
7268		debug_hash = true;
7269		} else if (mem.eql(u8, arg, "--debug-log")) {
7270		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7271		i += 1;
7272		try addDebugLog(arena, args[i]);
7273		} else if (mem.eql(u8, arg, "--save")) {
7274		save = .{ .yes = null };
7275		} else if (mem.cutPrefix(u8, arg, "--save=")) \|rest\| {
7276		save = .{ .yes = rest };
7277		} else if (mem.eql(u8, arg, "--save-exact")) {
7278		save = .{ .exact = null };
7279		} else if (mem.cutPrefix(u8, arg, "--save-exact=")) \|rest\| {
7280		save = .{ .exact = rest };
7281		} else {
7282		fatal("unrecognized parameter: {q}", .{arg});
7283		}
7284		} else if (opt_path_or_url != null) {
7285		fatal("unexpected extra parameter: {q}", .{arg});
7286		} else {
7287		opt_path_or_url = arg;
7288		}
7289		}
7290		}
7291
7292		const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
7293
7294		var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
7295		defer http_client.deinit();
7296
7297		try http_client.initDefaultProxies(arena, environ_map);
7298
7299		var root_prog_node = std.Progress.start(io, .{
7300		.root_name = "Fetch",
7301		});
7302		defer root_prog_node.end();
7303
7304		var global_cache_directory: Directory = l: {
7305		const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena, environ_map);
7306		break :l .{
7307		.handle = try Io.Dir.cwd().createDirPathOpen(io, p, .{}),
7308		.path = p,
7309		};
7310		};
7311		defer global_cache_directory.handle.close(io);
7312
7313		var local_storage: Package.Fetch.LocalStorage = undefined;
7314		var build_root: BuildRoot = undefined;
7315		var build_root_initialized = false;
7316		defer if (build_root_initialized) build_root.deinit(io);
7317
7318		const cwd_path = try introspect.getResolvedCwd(io, arena);
7319
7320		const local_storage_ptr = switch (save) {
7321		.no => null,
7322		.yes, .exact => ls: {
7323		build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
7324		build_root_initialized = true;
7325
7326		local_storage = .{
7327		.cache_root = if (override_local_cache_dir) \|p\| .initCwd(p) else .{
7328		.root_dir = build_root.directory,
7329		.sub_path = ".zig-cache",
7330		},
7331		.pkg_root = if (override_pkg_dir) \|p\| .initCwd(p) else .{
7332		.root_dir = build_root.directory,
7333		.sub_path = "zig-pkg",
7334		},
7335		};
7336
7337		break :ls &local_storage;
7338		},
7339		};
7340
7341		var job_queue: Package.Fetch.JobQueue = .{
7342		.io = io,
7343		.http_client = &http_client,
7344		.global_cache = global_cache_directory,
7345		.local_storage = local_storage_ptr,
7346		.recursive = false,
7347		.read_only = false,
7348		.debug_hash = debug_hash,
7349		.mode = .all,
7350		.prog_node = root_prog_node,
7351		};
7352		defer job_queue.deinit();
7353
7354		var fetch: Package.Fetch = .{
7355		.arena = std.heap.ArenaAllocator.init(gpa),
7356		.location = .{ .path_or_url = path_or_url },
7357		.location_tok = 0,
7358		.hash_tok = .none,
7359		.name_tok = 0,
7360		.lazy_status = .eager,
7361		.remote_package_root = undefined,
7362		.parent_package_root = undefined,
7363		.parent_manifest_ast = null,
7364		.prog_node = root_prog_node,
7365		.job_queue = &job_queue,
7366		.omit_missing_hash_error = true,
7367		.allow_missing_paths_field = false,
7368		.use_latest_commit = true,
7369
7370		.package_root = undefined,
7371		.error_bundle = undefined,
7372		.manifest = undefined,
7373		.manifest_ast = undefined,
7374		.have_manifest = false,
7375		.computed_hash = undefined,
7376		.has_build_zig = false,
7377		.oom_flag = false,
7378		.latest_commit = null,
7379
7380		.module = null,
7381		};
7382		defer fetch.deinit();
7383
7384		fetch.run() catch \|err\| switch (err) {
7385		error.OutOfMemory, error.Canceled => \|e\| return e,
7386		error.FetchFailed => {}, // error bundle checked below
7387		};
7388
7389		try job_queue.group.await(io);
7390
7391		if (fetch.error_bundle.root_list.items.len > 0) {
7392		var errors = try fetch.error_bundle.toOwnedBundle("");
7393		errors.renderToStderr(io, .{}, color) catch {};
7394		process.exit(1);
7395		}
7396
7397		const package_hash = fetch.computedPackageHash();
7398		const package_hash_slice = package_hash.toSlice();
7399
7400		root_prog_node.end();
7401		root_prog_node = .{ .index = .none };
7402
7403		const name = switch (save) {
7404		.no => {
7405		var stdout = Io.File.stdout().writerStreaming(io, &stdout_buffer);
7406		try stdout.interface.print("{s}\n", .{package_hash_slice});
7407		try stdout.interface.flush();
7408		return cleanExit(io);
7409		},
7410		.yes, .exact => \|name\| name: {
7411		if (name) \|n\| break :name n;
7412		if (!fetch.have_manifest)
7413		fatal("unable to determine name; fetched package has no build.zig.zon file", .{});
7414		break :name fetch.manifest.name;
7415		},
7416		};
7417
7418		// The name to use in case the manifest file needs to be created now.
7419		const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);
7420		var manifest, var ast = try loadManifest(gpa, arena, io, .{
7421		.root_name = try sanitizeExampleName(arena, init_root_name),
7422		.dir = build_root.directory.handle,
7423		.color = color,
7424		});
7425		defer {
7426		manifest.deinit(gpa);
7427		ast.deinit(gpa);
7428		}
7429
7430		var fixups: Ast.Render.Fixups = .{};
7431		defer fixups.deinit(gpa);
7432
7433		var saved_path_or_url = path_or_url;
7434
7435		if (fetch.latest_commit) \|latest_commit\| resolved: {
7436		const latest_commit_hex = try std.fmt.allocPrint(arena, "{f}", .{latest_commit});
7437
7438		var uri = try std.Uri.parse(path_or_url);
7439
7440		if (uri.fragment) \|fragment\| {
7441		const target_ref = try fragment.toRawMaybeAlloc(arena);
7442
7443		// the refspec may already be fully resolved
7444		if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved;
7445
7446		std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex });
7447
7448		// include the original refspec in a query parameter, could be used to check for updates
7449		uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={f}", .{
7450		std.fmt.alt(fragment, .formatEscaped),
7451		}) };
7452		} else {
7453		std.log.info("resolved to commit {s}", .{latest_commit_hex});
7454		}
7455
7456		// replace the refspec with the resolved commit SHA
7457		uri.fragment = .{ .raw = latest_commit_hex };
7458
7459		switch (save) {
7460		.yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{f}", .{uri}),
7461		.no, .exact => {}, // keep the original URL
7462		}
7463		}
7464
7465		const new_node_init = try std.fmt.allocPrint(arena,
7466		\\.{{
7467		\\ .url = "{f}",
7468		\\ .hash = "{f}",
7469		\\ }}
7470		, .{
7471		std.zig.fmtString(saved_path_or_url),
7472		std.zig.fmtString(package_hash_slice),
7473		});
7474
7475		const new_node_text = try std.fmt.allocPrint(arena, ".{f} = {s},\n", .{
7476		std.zig.fmtIdPU(name), new_node_init,
7477		});
7478
7479		const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{
7480		new_node_text,
7481		});
7482
7483		const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{
7484		dependencies_init,
7485		});
7486
7487		if (manifest.dependencies.get(name)) \|dep\| {
7488		if (dep.hash) \|h\| {
7489		switch (dep.location) {
7490		.url => \|u\| {
7491		if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
7492		std.log.info("existing dependency named {q} is up-to-date", .{name});
7493		process.exit(0);
7494		}
7495		},
7496		.path => {},
7497		}
7498		}
7499
7500		const location_replace = try std.fmt.allocPrint(
7501		arena,
7502		"\"{f}\"",
7503		.{std.zig.fmtString(saved_path_or_url)},
7504		);
7505		const hash_replace = try std.fmt.allocPrint(
7506		arena,
7507		"\"{f}\"",
7508		.{std.zig.fmtString(package_hash_slice)},
7509		);
7510
7511		warn("overwriting existing dependency named {q}", .{name});
7512		try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace);
7513		if (dep.hash_node.unwrap()) \|hash_node\| {
7514		try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace);
7515		} else {
7516		// https://github.com/ziglang/zig/issues/21690
7517		}
7518		} else if (manifest.dependencies.count() > 0) {
7519		// Add fixup for adding another dependency.
7520		const deps = manifest.dependencies.values();
7521		const last_dep_node = deps[deps.len - 1].node;
7522		try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text);
7523		} else if (manifest.dependencies_node.unwrap()) \|dependencies_node\| {
7524		// Add fixup for replacing the entire dependencies struct.
7525		try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init);
7526		} else {
7527		// Add fixup for adding dependencies struct.
7528		try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text);
7529		}
7530
7531		var aw: Io.Writer.Allocating = .init(gpa);
7532		defer aw.deinit();
7533		try ast.render(gpa, &aw.writer, fixups);
7534		const rendered = aw.written();
7535
7536		build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch \|err\| {
7537		fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err });
7538		};
7539
7540		return cleanExit(io);
7541		}
7542
7543		fn createEmptyDependenciesModule(
7544		arena: Allocator,
7545		io: Io,
7546		main_mod: *Package.Module,
7547		dirs: Compilation.Directories,
7548		global_options: Compilation.Config,
7549		) !void {
7550		var source = std.array_list.Managed(u8).init(arena);
7551		try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source);
7552		_ = try createDependenciesModule(
7553		arena,
7554		io,
7555		source.items,
7556		main_mod,
7557		dirs,
7558		global_options,
7559		);
7560		}
7561
7562		/// Creates the dependencies.zig file and corresponding `Package.Module` for the
7563		/// build runner to obtain via `@import("@dependencies")`.
7564		fn createDependenciesModule(
7565		arena: Allocator,
7566		io: Io,
7567		source: []const u8,
7568		main_mod: *Package.Module,
7569		dirs: Compilation.Directories,
7570		global_options: Compilation.Config,
7571		) !*Package.Module {
7572		// Atomically create the file in a directory named after the hash of its contents.
7573		const basename = "dependencies.zig";
7574		const rand_int = randInt(io, u64);
7575		const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
7576		{
7577		var tmp_dir = try dirs.local_cache.handle.createDirPathOpen(io, tmp_dir_sub_path, .{});
7578		defer tmp_dir.close(io);
7579		try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source });
7580		}
7581		const tmp_dir_path: Path = .{
7582		.root_dir = dirs.local_cache,
7583		.sub_path = tmp_dir_sub_path,
7584		};
7585
7586		var hh: Cache.HashHelper = .{};
7587		hh.addBytes(build_options.version);
7588		hh.addBytes(source);
7589		const hex_digest = hh.final();
7590
7591		const o_dir_path: Path = .{
7592		.root_dir = dirs.local_cache,
7593		.sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest),
7594		};
7595		try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path);
7596
7597		const deps_mod = try Package.Module.create(arena, .{
7598		.paths = .{
7599		.root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path),
7600		.root_src_path = basename,
7601		},
7602		.fully_qualified_name = "root.@dependencies",
7603		.parent = main_mod,
7604		.cc_argv = &.{},
7605		.inherited = .{},
7606		.global = global_options,
7607		});
7608		try main_mod.deps.put(arena, "@dependencies", deps_mod);
7609		return deps_mod;
7610		}
7611
7612		const BuildRoot = struct {
7613		directory: Cache.Directory,
7614		build_zig_basename: []const u8,
7615		cleanup_build_dir: ?Io.Dir,
7616
7617		fn deinit(br: *BuildRoot, io: Io) void {
7618		if (br.cleanup_build_dir) \|*dir\| dir.close(io);
7619		br.* = undefined;
7620		}
7621		};
7622
7623		const FindBuildRootOptions = struct {
7624		build_file: ?[]const u8 = null,
7625		cwd_path: ?[]const u8 = null,
7626		};
7627
7628		fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot {
7629		const cwd_path = options.cwd_path orelse try introspect.getResolvedCwd(io, arena);
7630		const build_zig_basename = if (options.build_file) \|bf\|
7631		fs.path.basename(bf)
7632		else
7633		Package.build_zig_basename;
7634
7635		if (options.build_file) \|bf\| {
7636		if (fs.path.dirname(bf)) \|dirname\| {
7637		const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
7638		fatal("unable to open directory to build file from argument 'build-file', {q}: {t}", .{ dirname, err });
7639		};
7640		return .{
7641		.build_zig_basename = build_zig_basename,
7642		.directory = .{ .path = dirname, .handle = dir },
7643		.cleanup_build_dir = dir,
7644		};
7645		}
7646
7647		return .{
7648		.build_zig_basename = build_zig_basename,
7649		.directory = .{ .path = null, .handle = Io.Dir.cwd() },
7650		.cleanup_build_dir = null,
7651		};
7652		}
7653		// Search up parent directories until we find build.zig.
7654		var dirname: []const u8 = cwd_path;
7655		while (true) {
7656		const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename });
7657		if (Io.Dir.cwd().access(io, joined_path, .{})) \|_\| {
7658		const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
7659		fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err });
7660		};
7661		return .{
7662		.build_zig_basename = build_zig_basename,
7663		.directory = .{
7664		.path = dirname,
7665		.handle = dir,
7666		},
7667		.cleanup_build_dir = dir,
7668		};
7669		} else \|err\| switch (err) {
7670		error.FileNotFound => {
7671		dirname = fs.path.dirname(dirname) orelse {
7672		std.log.info("initialize {s} template file with 'zig init'", .{
7673		Package.build_zig_basename,
7674		});
7675		std.log.info("see 'zig --help' for more options", .{});
7676		fatal("no build.zig file found, in the current directory or any parent directories", .{});
7677		};
7678		continue;
7679		},
7680		else => \|e\| return e,
7681		}
7682		}
7683		}
7684
7685		const LoadManifestOptions = struct {
7686		root_name: []const u8,
7687		dir: Io.Dir,
7688		color: Color,
7689		};
7690
7691		fn loadManifest(
7692		gpa: Allocator,
7693		arena: Allocator,
7694		io: Io,
7695		options: LoadManifestOptions,
7696		) !struct { Package.Manifest, Ast } {
7697		const rng: std.Random.IoSource = .{ .io = io };
7698
7699		const manifest_bytes = while (true) {
7700		break options.dir.readFileAllocOptions(
7701		io,
7702		Package.Manifest.basename,
7703		arena,
7704		.limited(Package.Manifest.max_bytes),
7705		.@"1",
7706		0,
7707		) catch \|err\| switch (err) {
7708		error.FileNotFound => {
7709		writeSimpleTemplateFile(io, Package.Manifest.basename,
7710		\\.{{
7711		\\ .name = .{s},
7712		\\ .version = "{s}",
7713		\\ .paths = .{{""}},
7714		\\ .fingerprint = 0x{x},
7715		\\}}
7716		\\
7717		, .{
7718		options.root_name,
7719		build_options.version,
7720		Package.Fingerprint.generate(rng.interface(), options.root_name).int(),
7721		}) catch \|e\| {
7722		fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e });
7723		};
7724		continue;
7725		},
7726		else => \|e\| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }),
7727		};
7728		};
7729		var ast = try Ast.parse(gpa, manifest_bytes, .zon);
7730		errdefer ast.deinit(gpa);
7731
7732		if (ast.errors.len > 0) {
7733		try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color);
7734		process.exit(2);
7735		}
7736
7737		var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{});
7738		errdefer manifest.deinit(gpa);
7739
7740		if (manifest.errors.len > 0) {
7741		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
7742		try wip_errors.init(gpa);
7743		defer wip_errors.deinit();
7744
7745		const src_path = try wip_errors.addString(Package.Manifest.basename);
7746		try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors);
7747
7748		var error_bundle = try wip_errors.toOwnedBundle("");
7749		defer error_bundle.deinit(gpa);
7750		error_bundle.renderToStderr(io, .{}, options.color) catch {};
7751
7752		process.exit(2);
7753		}
7754		return .{ manifest, ast };
7755		}
7756
7757		const Templates = struct {
7758		zig_lib_directory: Cache.Directory,
7759		dir: Io.Dir,
7760		buffer: std.array_list.Managed(u8),
7761
7762		fn deinit(templates: *Templates, io: Io) void {
7763		templates.zig_lib_directory.handle.close(io);
7764		templates.dir.close(io);
7765		templates.buffer.deinit();
7766		templates.* = undefined;
7767		}
7768
7769		fn write(
7770		templates: *Templates,
7771		arena: Allocator,
7772		io: Io,
7773		out_dir: Io.Dir,
7774		root_name: []const u8,
7775		template_path: []const u8,
7776		fingerprint: Package.Fingerprint,
7777		) !void {
7778		if (fs.path.dirname(template_path)) \|dirname\| {
7779		out_dir.createDirPath(io, dirname) catch \|err\| {
7780		fatal("unable to make path {q}: {t}", .{ dirname, err });
7781		};
7782		}
7783
7784		const max_bytes = 10 * 1024 * 1024;
7785		const contents = templates.dir.readFileAlloc(io, template_path, arena, .limited(max_bytes)) catch \|err\| {
7786		fatal("unable to read template file {q}: {t}", .{ template_path, err });
7787		};
7788		templates.buffer.clearRetainingCapacity();
7789		try templates.buffer.ensureUnusedCapacity(contents.len);
7790		var i: usize = 0;
7791		while (i < contents.len) {
7792		if (contents[i] == '_' or contents[i] == '.') {
7793		// Both '_' and '.' are allowed because depending on the context
7794		// one prefix will be valid, while the other might not.
7795		if (std.mem.startsWith(u8, contents[i + 1 ..], "NAME")) {
7796		try templates.buffer.appendSlice(root_name);
7797		i += "_NAME".len;
7798		continue;
7799		} else if (std.mem.startsWith(u8, contents[i + 1 ..], "FINGERPRINT")) {
7800		try templates.buffer.print("0x{x}", .{fingerprint.int()});
7801		i += "_FINGERPRINT".len;
7802		continue;
7803		} else if (std.mem.startsWith(u8, contents[i + 1 ..], "ZIGVER")) {
7804		try templates.buffer.appendSlice(build_options.version);
7805		i += "_ZIGVER".len;
7806		continue;
7807		}
7808		}
7809
7810		try templates.buffer.append(contents[i]);
7811		i += 1;
7812		}
7813
7814		return out_dir.writeFile(io, .{
7815		.sub_path = template_path,
7816		.data = templates.buffer.items,
7817		.flags = .{ .exclusive = true },
7818		});
7819		}
7820		};
7821		fn writeSimpleTemplateFile(io: Io, file_name: []const u8, comptime fmt: []const u8, args: anytype) !void {
7822		const f = try Io.Dir.cwd().createFile(io, file_name, .{ .exclusive = true });
7823		defer f.close(io);
7824		var buf: [4096]u8 = undefined;
7825		var fw = f.writer(io, &buf);
7826		try fw.interface.print(fmt, args);
7827		try fw.interface.flush();
7828		}
7829
7830		fn findTemplates(gpa: Allocator, arena: Allocator, io: Io) Templates {
7831		const cwd_path = introspect.getResolvedCwd(io, arena) catch \|err\| {
7832		fatal("unable to get cwd: {t}", .{err});
7833		};
7834		const self_exe_path = process.executablePathAlloc(io, arena) catch \|err\| {
7835		fatal("unable to find self exe path: {t}", .{err});
7836		};
7837		var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch \|err\| {
7838		fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err });
7839		};
7840
7841		const s = fs.path.sep_str;
7842		const template_sub_path = "init";
7843		const template_dir = zig_lib_directory.handle.openDir(io, template_sub_path, .{}) catch \|err\| {
7844		const path = zig_lib_directory.path orelse ".";
7845		fatal("unable to open zig project template directory '{s}{s}{s}': {t}", .{
7846		path, s, template_sub_path, err,
7847		});
7848		};
7849
7850		return .{
7851		.zig_lib_directory = zig_lib_directory,
7852		.dir = template_dir,
7853		.buffer = std.array_list.Managed(u8).init(gpa),
7854		};
7855		}
7856
7857	6003	fn parseOptimizeMode(s: []const u8) std.lang.OptimizeMode {
7858	6004	return stringToEnum(std.lang.OptimizeMode, s) orelse
7859	6005	fatal("unrecognized optimization mode: {q}", .{s});
...	...	@@ -7879,7 +6025,7 @@ fn handleModArg(
7879	6025	mod_name: []const u8,
7880	6026	opt_root_src_orig: ?[]const u8,
7881	6027	create_module: *CreateModule,
7882		mod_opts: *Package.Module.CreateOptions.Inherited,
	6028	mod_opts: *Module.CreateOptions.Inherited,
7883	6029	cc_argv: *std.ArrayList([]const u8),
7884	6030	target_arch_os_abi: *?[]const u8,
7885	6031	target_mcpu: *?[]const u8,

src/print_env.zig+2-3

...	...	@@ -8,7 +8,6 @@ const fatal = std.process.fatal;
8	8
9	9	const build_options = @import("build_options");
10	10	const Compilation = @import("Compilation.zig");
11		const introspect = @import("introspect.zig");
12	11
13	12	pub fn cmdEnv(
14	13	arena: Allocator,
...	...	@@ -29,9 +28,9 @@ pub fn cmdEnv(
29	28	},
30	29	};
31	30
32		const cwd_path = try introspect.getResolvedCwd(io, arena);
	31	const cwd_path = try std.zig.getResolvedCwd(io, arena);
33	32
34		var dirs: Compilation.Directories = .init(
	33	var dirs: std.zig.Directories = .init(
35	34	arena,
36	35	io,
37	36	override_lib_dir,

src/print_targets.zig+1-2

...	...	@@ -9,7 +9,6 @@ const Target = std.Target;
9	9	const assert = std.debug.assert;
10	10
11	11	const glibc = @import("libs/glibc.zig");
12		const introspect = @import("introspect.zig");
13	12	const target = @import("target.zig");
14	13
15	14	pub fn cmdTargets(
...	...	@@ -20,7 +19,7 @@ pub fn cmdTargets(
20	19	native_target: *const Target,
21	20	) !void {
22	21	_ = args;
23		var zig_lib_directory = introspect.findZigLibDir(allocator, io) catch \|err\|
	22	var zig_lib_directory = std.zig.findZigLibDir(allocator, io) catch \|err\|
24	23	fatal("unable to find zig installation directory: {t}", .{err});
25	24	defer zig_lib_directory.handle.close(io);
26	25	defer allocator.free(zig_lib_directory.path.?);

test/src/Cases.zig+17-10

...	...	@@ -316,20 +316,19 @@ pub fn addCompile(
316	316	/// Each file should include a test manifest as a contiguous block of comments at
317	317	/// the end of the file. The first line should be the test type, followed by a set of
318	318	/// key-value config values, followed by a blank line, then the expected output.
319		pub fn addFromDir(ctx: Cases, dir: Io.Dir, b: std.Build) void {
	319	pub fn addFromDir(ctx: Cases, dir: Io.Dir, path_from_root: []const u8, b: std.Build) void {
320	320	var current_file: []const u8 = "none";
321		ctx.addFromDirInner(dir, &current_file, b) catch \|err\| {
322		std.debug.panicExtra(
323		@returnAddress(),
324		"test harness failed to process file '{s}': {s}\n",
325		.{ current_file, @errorName(err) },
326		);
	321	ctx.addFromDirInner(dir, path_from_root, &current_file, b) catch \|err\| {
	322	std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}\n", .{
	323	current_file, err,
	324	});
327	325	};
328	326	}
329	327
330	328	fn addFromDirInner(
331	329	ctx: *Cases,
332	330	iterable_dir: Io.Dir,
	331	path_from_root: []const u8,
333	332	/// This is kept up to date with the currently being processed file so
334	333	/// that if any errors occur the caller knows it happened during this file.
335	334	current_file: *[]const u8,
...	...	@@ -340,11 +339,19 @@ fn addFromDirInner(
340	339	var filenames: ArrayList([]const u8) = .empty;
341	340
342	341	while (try it.next(io)) \|entry\| {
343		if (entry.kind != .file) continue;
344
345	342	// Ignore stuff such as .swp files
346	343	if (!knownFileExtension(entry.basename)) continue;
347		try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path));
	344
	345	switch (entry.kind) {
	346	.file => {
	347	b.dependOnFileContents(b.path(b.pathJoin(&.{ path_from_root, entry.path })));
	348	try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path));
	349	},
	350	.directory => {
	351	b.dependOnDirectory(b.path(b.pathJoin(&.{ path_from_root, entry.path })));
	352	},
	353	else => continue,
	354	}
348	355	}
349	356
350	357	for (filenames.items) \|filename\| {

test/tests.zig+12-8

...	...	@@ -3258,14 +3258,12 @@ pub fn addCases(
3258	3258
3259	3259	var cases = @import("src/Cases.zig").init(gpa, arena, io);
3260	3260
3261		// Ensure changes to these files get picked up
3262		// https://codeberg.org/ziglang/zig/issues/35473
3263		b.graph.poisonCache();
	3261	b.dependOnDirectory(b.path("test/cases"));
3264	3262
3265	3263	var dir = try b.root.openDir(io, "test/cases", .{ .iterate = true });
3266	3264	defer dir.close(io);
3267	3265
3268		cases.addFromDir(dir, b);
	3266	cases.addFromDir(dir, "test/cases", b);
3269	3267	try @import("cases.zig").addCases(&cases, build_options, b);
3270	3268
3271	3269	cases.lowerToBuildSteps(
...	...	@@ -3320,22 +3318,28 @@ pub fn addIncrementalTests(b: std.Build, test_step: Step, test_filters: []cons
3320	3318	}),
3321	3319	});
3322	3320
3323		// Ensure changes to these files get picked up
3324		// https://codeberg.org/ziglang/zig/issues/35473
3325		b.graph.poisonCache();
	3321	b.dependOnDirectory(b.path("test/incremental"));
3326	3322
3327	3323	var dir = try b.root.openDir(io, "test/incremental", .{ .iterate = true });
3328	3324	defer dir.close(io);
3329	3325
3330	3326	var it = try dir.walk(b.graph.arena);
3331	3327	while (try it.next(io)) \|entry\| {
3332		if (entry.kind != .file) continue;
3333	3328	if (std.mem.endsWith(u8, entry.basename, ".swp")) continue;
3334	3329
3335	3330	for (test_filters) \|test_filter\| {
3336	3331	if (std.mem.indexOf(u8, entry.path, test_filter)) \|_\| break;
3337	3332	} else if (test_filters.len > 0) continue;
3338	3333
	3334	switch (entry.kind) {
	3335	.file => {},
	3336	.directory => {
	3337	b.dependOnDirectory(b.path(b.pathJoin(&.{ "test", "incremental", entry.path })));
	3338	},
	3339	else => continue,
	3340	}
	3341	b.dependOnFileContents(b.path(b.pathJoin(&.{ "test", "incremental", entry.path })));
	3342
3339	3343	for (incremental_targets) \|target_str\| {
3340	3344	const run = b.addRunArtifact(incr_check);
3341	3345	run.setName(b.fmt("incr-check {s} '{s}'", .{ target_str, entry.basename }));