Commit ce5222d945b4e1c3637f1673716bc224434da19a | zig

author	Andrew Kelley <andrew@ziglang.org> 2021-01-06 23:17:00-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-01-06 23:40:17-07:00
log	ce5222d945b4e1c3637f1673716bc224434da19a
tree	d32ff834b5268df0560fa8f7fcca6b40a21c9fdd
parent	f1ef0a80f1d614bbbb9ad8fae7992d55111c8cd7

ci: rework linux script

 * no longer depend on apt.llvm.org, instead we rely on a zig-bootstrap
   tarball with pre-built Zig, LLVM, LLD, and Clang. Similar to the
   Windows Dev Kit but for Linux. This also makes the script no longer
   depend on Docker, libxml2, or GCC.
 * remove bash retry logic; it was only needed for flaky apt.llvm.org
   and isn't needed for the other resources we download.
 * build and upload the experimental std lib automatically generated
   docs. langref.html is moved to docs/langref.html because the std lib
   docs are in docs/std/.
 * the superfluous "zig" directory in $prefix/lib/zig/std/std.zig is
   removed from the tarball.
 * update_download_page makes a commit in the www.ziglang.org repository
   updating data/releases.json so that repository can manage deploys.

3 files changed, 90 insertions(+), 57 deletions(-)

ci
- azure
  - linux_script
- srht
  - on_master_success
  - update_download_page

ci/azure/linux_script+74-53

...	@@ -3,48 +3,35 @@	...	@@ -3,48 +3,35 @@
3	set -x	3	set -x
4	set -e	4	set -e
5		5
6	# This parameters we wait at most 2mins, it should be enough to sort out any	6	sudo apt-get update -q
7	# transient error.	7	sudo apt-get install -y cmake s3cmd ninja-build tidy
8	CMD_MAX_RETRY=12	8
9	CMD_WAIT_TIME=10s	9	ZIGDIR="$(pwd)"
10		10	ARCH="$(uname -m)"
11	# Execute the given command and, in case of failure, try to execute it again	11	CACHE_BASENAME="zig+llvm+lld+clang-$ARCH-linux-musl-0.8.0-dev.859+f1ef0a80f"
12	# after sleeping for CMD_WAIT_TIME.	12	PREFIX="$HOME/$CACHE_BASENAME"
13	# We give up after retrying CMD_MAX_RETRY times.	13
14	retry() {	14	rm -rf $PREFIX
15	for i in $(seq 1 "$CMD_MAX_RETRY"); do	15	cd $HOME
16	eval "$@" && return	16
17	echo "command \"$@\" failed, retrying..."	17	wget -nv "https://ziglang.org/deps/$CACHE_BASENAME.tar.xz"
18	sleep ${CMD_WAIT_TIME}	18	tar xf "$CACHE_BASENAME.tar.xz"
19	done
20
21	echo "command \"$@\" failed, giving up..."
22	exit 1
23	}
24
25	BUILDDIR="$(pwd)"
26
27	sudo sh -c 'echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" >> /etc/apt/sources.list'
28	retry 'wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key\|sudo apt-key add -'
29	retry sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
30
31	sudo apt-get remove -y llvm-*
32	sudo rm -rf /usr/local/*
33
34	retry sudo apt-get update -q
35	retry sudo apt-get install -y \
36	libxml2-dev libclang-11-dev llvm-11 llvm-11-dev liblld-11-dev cmake s3cmd \
37	gcc-7 g++-7 ninja-build tidy \
38		19
39	QEMUBASE="qemu-linux-x86_64-5.2.0"	20	QEMUBASE="qemu-linux-x86_64-5.2.0"
40	wget -nv https://ziglang.org/deps/$QEMUBASE.tar.xz	21	wget -nv "https://ziglang.org/deps/$QEMUBASE.tar.xz"
41	tar xf $QEMUBASE.tar.xz	22	tar xf "$QEMUBASE.tar.xz"
42	PATH=$PWD/$QEMUBASE/bin:$PATH	23	export PATH="$(pwd)/$QEMUBASE/bin:$PATH"
43		24
44	WASMTIME="wasmtime-v0.20.0-x86_64-linux"	25	WASMTIME="wasmtime-v0.20.0-x86_64-linux"
45	wget -nv https://github.com/bytecodealliance/wasmtime/releases/download/v0.20.0/$WASMTIME.tar.xz	26	wget -nv "https://github.com/bytecodealliance/wasmtime/releases/download/v0.20.0/$WASMTIME.tar.xz"
46	tar xf $WASMTIME.tar.xz	27	tar xf "$WASMTIME.tar.xz"
47	PATH=$PWD/$WASMTIME:$PATH	28	export PATH="$(pwd)/$WASMTIME:$PATH"
		29
		30	ZIG="$PREFIX/bin/zig"
		31	export CC="$ZIG cc"
		32	export CXX="$ZIG c++"
		33
		34	cd $ZIGDIR
48		35
49	# Make the `zig version` number consistent.	36	# Make the `zig version` number consistent.
50	# This will affect the cmake command below.	37	# This will affect the cmake command below.
...	@@ -52,29 +39,63 @@ git config core.abbrev 9	...	@@ -52,29 +39,63 @@ git config core.abbrev 9
52	git fetch --unshallow \|\| true	39	git fetch --unshallow \|\| true
53	git fetch --tags	40	git fetch --tags
54		41
55	export CC=gcc-7
56	export CXX=g++-7
57	mkdir build	42	mkdir build
58	cd build	43	cd build
59	cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja	44	cmake .. \
		45	-DCMAKE_INSTALL_PREFIX="$(pwd)/release" \
		46	-DCMAKE_PREFIX_PATH="$PREFIX" \
		47	-DCMAKE_BUILD_TYPE=Release \
		48	-DZIG_TARGET_TRIPLE="$ARCH-linux-musl" \
		49	-DZIG_TARGET_MCPU="baseline" \
		50	-DZIG_STATIC=ON \
		51	-GNinja
		52
		53	# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
		54	# so that installation and testing do not get affected by them.
		55	unset CC
		56	unset CXX
		57
60	ninja install	58	ninja install
61	./zig build test -Denable-qemu -Denable-wasmtime
62		59
63	# look for HTML errors	60	# Here we rebuild zig but this time using the Zig binary we just now produced to
64	tidy -qe ../zig-cache/langref.html	61	# build zig1.o rather than relying on the one built with stage0. See
		62	# https://github.com/ziglang/zig/issues/6830 for more details.
		63	cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig"
		64	ninja install
65		65
66	VERSION="$(./zig version)"	66	release/bin/zig build test -Denable-qemu -Denable-wasmtime
		67
		68	# Look for HTML errors.
		69	tidy -qe ../zig-cache/langref.html
67		70
68	if [ "${BUILD_REASON}" != "PullRequest" ]; then	71	if [ "${BUILD_REASON}" != "PullRequest" ]; then
69	ARTIFACTSDIR="$BUILDDIR/artifacts"	72	# Produce the experimental std lib documentation.
70	mkdir "$ARTIFACTSDIR"	73	mkdir -p release/docs/std
71	docker run -i --mount type=bind,source="$ARTIFACTSDIR",target=/z ziglang/static-base:llvm11-x86_64-1 -j2 $BUILD_SOURCEVERSION	74	release/bin/zig test ../lib/std/std.zig -femit-docs=release/docs/std -fno-emit-bin
72	TARBALL="$(ls $ARTIFACTSDIR)"	75
		76	mv ../LICENSE release/
		77	mv ../zig-cache/langref.html release/docs/
		78
		79	# Remove the unnecessary bin dir in $prefix/bin/zig
		80	mv release/bin/zig release/
		81	rmdir release/bin
		82
		83	# Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig
		84	mv release/lib/zig release/lib2
		85	rmdir release/lib
		86	mv release/lib2 release/lib
		87
		88	VERSION=$(release/zig version)
		89	DIRNAME="zig-linux-$ARCH-$VERSION"
		90	TARBALL="$DIRNAME.tar.xz"
		91	mv release "$DIRNAME"
		92	tar cfJ "$TARBALL" "$DIRNAME"
		93
73	mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"	94	mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
74	s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$ARTIFACTSDIR/$TARBALL" s3://ziglang.org/builds/	95	s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/
75		96
76	SHASUM=$(sha256sum $ARTIFACTSDIR/$TARBALL \| cut '-d ' -f1)	97	SHASUM=$(sha256sum $TARBALL \| cut '-d ' -f1)
77	BYTESIZE=$(wc -c < $ARTIFACTSDIR/$TARBALL)	98	BYTESIZE=$(wc -c < $TARBALL)
78		99
79	JSONFILE="linux-$GITBRANCH.json"	100	JSONFILE="linux-$GITBRANCH.json"
80	touch $JSONFILE	101	touch $JSONFILE
...	@@ -83,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then	...	@@ -83,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
83	echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE	104	echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
84		105
85	s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"	106	s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
86	s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"	107	s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json"
87		108
88	# `set -x` causes these variables to be mangled.	109	# `set -x` causes these variables to be mangled.
89	# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html	110	# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html

ci/srht/on_master_success+1

...	@@ -24,6 +24,7 @@ packages:	...	@@ -24,6 +24,7 @@ packages:
24	- xz	24	- xz
25	secrets:	25	secrets:
26	- 51bfddf5-86a6-4e01-8576-358c72a4a0a4	26	- 51bfddf5-86a6-4e01-8576-358c72a4a0a4
		27	- 8d5f230b-78d9-4e7c-b583-8cbb1b15807c
27	sources:	28	sources:
28	- https://github.com/ziglang/zig	29	- https://github.com/ziglang/zig
29	tasks:	30	tasks:

ci/srht/update_download_page+15-4

 @@ -35,7 +35,7 @@ wget "https://ziglang.org/builds/$NATIVE_TARBALL"
 tar xf "$NATIVE_TARBALL"
 ZIGDIR=$(basename $NATIVE_TARBALL .tar.xz)
 ZIG="$ZIGDIR/zig"
-LANGREF="$ZIGDIR/langref.html"
+LANGREF="$ZIGDIR/docs/langref.html"
 SRCTARBALLDIR="zig-$VERSION"
 export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
 mv "$SRCDIR" "$SRCTARBALLDIR"
 @@ -68,13 +68,24 @@ export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)"
 export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)"
 export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)"
-git clone https://github.com/ziglang/www.ziglang.org --depth 1
+git clone git@github.com:ziglang/www.ziglang.org.git
 cd www.ziglang.org
+git checkout hugo-redesign
 export MASTER_DATE="$(date +%Y-%m-%d)"
 export MASTER_VERSION="$VERSION"
 "../$ZIG" run update-download-page.zig
 $S3CMD put -P --no-mime-magic --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
 $S3CMD put -P --no-mime-magic "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-$S3CMD put -P --no-mime-magic www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/index.html" s3://ziglang.org/documentation/master/std/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-$S3CMD put -P --no-mime-magic www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/data.js" s3://ziglang.org/documentation/master/std/data.js --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/main.js" s3://ziglang.org/documentation/master/std/main.js --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic ci/out/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic ci/out/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
+cp ci/out/index.json data/releases.json
+git add data/releases.json
+git commit -m "CI: update releases.json"
+git push origin hugo-redesign