Commit ce5222d945b4e1c3637f1673716bc224434da19a | zig

author	Andrew Kelley <andrew@ziglang.org> 2021-01-06 23:17:00-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-01-06 23:40:17-07:00
log	ce5222d945b4e1c3637f1673716bc224434da19a
tree	d32ff834b5268df0560fa8f7fcca6b40a21c9fdd
parent	f1ef0a80f1d614bbbb9ad8fae7992d55111c8cd7

ci: rework linux script

 * no longer depend on apt.llvm.org, instead we rely on a zig-bootstrap
   tarball with pre-built Zig, LLVM, LLD, and Clang. Similar to the
   Windows Dev Kit but for Linux. This also makes the script no longer
   depend on Docker, libxml2, or GCC.
 * remove bash retry logic; it was only needed for flaky apt.llvm.org
   and isn't needed for the other resources we download.
 * build and upload the experimental std lib automatically generated
   docs. langref.html is moved to docs/langref.html because the std lib
   docs are in docs/std/.
 * the superfluous "zig" directory in $prefix/lib/zig/std/std.zig is
   removed from the tarball.
 * update_download_page makes a commit in the www.ziglang.org repository
   updating data/releases.json so that repository can manage deploys.

3 files changed, 90 insertions(+), 57 deletions(-)

ci
- azure
  - linux_script
- srht
  - on_master_success
  - update_download_page

ci/azure/linux_script+74-53

...	...	@@ -3,48 +3,35 @@
3	3	set -x
4	4	set -e
5	5
6		# This parameters we wait at most 2mins, it should be enough to sort out any
7		# transient error.
8		CMD_MAX_RETRY=12
9		CMD_WAIT_TIME=10s
10
11		# Execute the given command and, in case of failure, try to execute it again
12		# after sleeping for CMD_WAIT_TIME.
13		# We give up after retrying CMD_MAX_RETRY times.
14		retry() {
15		for i in $(seq 1 "$CMD_MAX_RETRY"); do
16		eval "$@" && return
17		echo "command \"$@\" failed, retrying..."
18		sleep ${CMD_WAIT_TIME}
19		done
20
21		echo "command \"$@\" failed, giving up..."
22		exit 1
23		}
24
25		BUILDDIR="$(pwd)"
26
27		sudo sh -c 'echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" >> /etc/apt/sources.list'
28		retry 'wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key\|sudo apt-key add -'
29		retry sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
30
31		sudo apt-get remove -y llvm-*
32		sudo rm -rf /usr/local/*
33
34		retry sudo apt-get update -q
35		retry sudo apt-get install -y \
36		libxml2-dev libclang-11-dev llvm-11 llvm-11-dev liblld-11-dev cmake s3cmd \
37		gcc-7 g++-7 ninja-build tidy \
	6	sudo apt-get update -q
	7	sudo apt-get install -y cmake s3cmd ninja-build tidy
	8
	9	ZIGDIR="$(pwd)"
	10	ARCH="$(uname -m)"
	11	CACHE_BASENAME="zig+llvm+lld+clang-$ARCH-linux-musl-0.8.0-dev.859+f1ef0a80f"
	12	PREFIX="$HOME/$CACHE_BASENAME"
	13
	14	rm -rf $PREFIX
	15	cd $HOME
	16
	17	wget -nv "https://ziglang.org/deps/$CACHE_BASENAME.tar.xz"
	18	tar xf "$CACHE_BASENAME.tar.xz"
38	19
39	20	QEMUBASE="qemu-linux-x86_64-5.2.0"
40		wget -nv https://ziglang.org/deps/$QEMUBASE.tar.xz
41		tar xf $QEMUBASE.tar.xz
42		PATH=$PWD/$QEMUBASE/bin:$PATH
	21	wget -nv "https://ziglang.org/deps/$QEMUBASE.tar.xz"
	22	tar xf "$QEMUBASE.tar.xz"
	23	export PATH="$(pwd)/$QEMUBASE/bin:$PATH"
43	24
44	25	WASMTIME="wasmtime-v0.20.0-x86_64-linux"
45		wget -nv https://github.com/bytecodealliance/wasmtime/releases/download/v0.20.0/$WASMTIME.tar.xz
46		tar xf $WASMTIME.tar.xz
47		PATH=$PWD/$WASMTIME:$PATH
	26	wget -nv "https://github.com/bytecodealliance/wasmtime/releases/download/v0.20.0/$WASMTIME.tar.xz"
	27	tar xf "$WASMTIME.tar.xz"
	28	export PATH="$(pwd)/$WASMTIME:$PATH"
	29
	30	ZIG="$PREFIX/bin/zig"
	31	export CC="$ZIG cc"
	32	export CXX="$ZIG c++"
	33
	34	cd $ZIGDIR
48	35
49	36	# Make the `zig version` number consistent.
50	37	# This will affect the cmake command below.
...	...	@@ -52,29 +39,63 @@ git config core.abbrev 9
52	39	git fetch --unshallow \|\| true
53	40	git fetch --tags
54	41
55		export CC=gcc-7
56		export CXX=g++-7
57	42	mkdir build
58	43	cd build
59		cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja
	44	cmake .. \
	45	-DCMAKE_INSTALL_PREFIX="$(pwd)/release" \
	46	-DCMAKE_PREFIX_PATH="$PREFIX" \
	47	-DCMAKE_BUILD_TYPE=Release \
	48	-DZIG_TARGET_TRIPLE="$ARCH-linux-musl" \
	49	-DZIG_TARGET_MCPU="baseline" \
	50	-DZIG_STATIC=ON \
	51	-GNinja
	52
	53	# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
	54	# so that installation and testing do not get affected by them.
	55	unset CC
	56	unset CXX
	57
60	58	ninja install
61		./zig build test -Denable-qemu -Denable-wasmtime
62	59
63		# look for HTML errors
64		tidy -qe ../zig-cache/langref.html
	60	# Here we rebuild zig but this time using the Zig binary we just now produced to
	61	# build zig1.o rather than relying on the one built with stage0. See
	62	# https://github.com/ziglang/zig/issues/6830 for more details.
	63	cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig"
	64	ninja install
65	65
66		VERSION="$(./zig version)"
	66	release/bin/zig build test -Denable-qemu -Denable-wasmtime
	67
	68	# Look for HTML errors.
	69	tidy -qe ../zig-cache/langref.html
67	70
68	71	if [ "${BUILD_REASON}" != "PullRequest" ]; then
69		ARTIFACTSDIR="$BUILDDIR/artifacts"
70		mkdir "$ARTIFACTSDIR"
71		docker run -i --mount type=bind,source="$ARTIFACTSDIR",target=/z ziglang/static-base:llvm11-x86_64-1 -j2 $BUILD_SOURCEVERSION
72		TARBALL="$(ls $ARTIFACTSDIR)"
	72	# Produce the experimental std lib documentation.
	73	mkdir -p release/docs/std
	74	release/bin/zig test ../lib/std/std.zig -femit-docs=release/docs/std -fno-emit-bin
	75
	76	mv ../LICENSE release/
	77	mv ../zig-cache/langref.html release/docs/
	78
	79	# Remove the unnecessary bin dir in $prefix/bin/zig
	80	mv release/bin/zig release/
	81	rmdir release/bin
	82
	83	# Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig
	84	mv release/lib/zig release/lib2
	85	rmdir release/lib
	86	mv release/lib2 release/lib
	87
	88	VERSION=$(release/zig version)
	89	DIRNAME="zig-linux-$ARCH-$VERSION"
	90	TARBALL="$DIRNAME.tar.xz"
	91	mv release "$DIRNAME"
	92	tar cfJ "$TARBALL" "$DIRNAME"
	93
73	94	mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
74		s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$ARTIFACTSDIR/$TARBALL" s3://ziglang.org/builds/
	95	s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/
75	96
76		SHASUM=$(sha256sum $ARTIFACTSDIR/$TARBALL \| cut '-d ' -f1)
77		BYTESIZE=$(wc -c < $ARTIFACTSDIR/$TARBALL)
	97	SHASUM=$(sha256sum $TARBALL \| cut '-d ' -f1)
	98	BYTESIZE=$(wc -c < $TARBALL)
78	99
79	100	JSONFILE="linux-$GITBRANCH.json"
80	101	touch $JSONFILE
...	...	@@ -83,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
83	104	echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
84	105
85	106	s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
86		s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"
	107	s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json"
87	108
88	109	# `set -x` causes these variables to be mangled.
89	110	# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html

ci/srht/on_master_success+1

...	...	@@ -24,6 +24,7 @@ packages:
24	24	- xz
25	25	secrets:
26	26	- 51bfddf5-86a6-4e01-8576-358c72a4a0a4
	27	- 8d5f230b-78d9-4e7c-b583-8cbb1b15807c
27	28	sources:
28	29	- https://github.com/ziglang/zig
29	30	tasks:

ci/srht/update_download_page+15-4

...	...	@@ -35,7 +35,7 @@ wget "https://ziglang.org/builds/$NATIVE_TARBALL"
35	35	tar xf "$NATIVE_TARBALL"
36	36	ZIGDIR=$(basename $NATIVE_TARBALL .tar.xz)
37	37	ZIG="$ZIGDIR/zig"
38		LANGREF="$ZIGDIR/langref.html"
	38	LANGREF="$ZIGDIR/docs/langref.html"
39	39	SRCTARBALLDIR="zig-$VERSION"
40	40	export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
41	41	mv "$SRCDIR" "$SRCTARBALLDIR"
...	...	@@ -68,13 +68,24 @@ export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" \| jq .tarball -r)"
68	68	export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" \| jq .size -r)"
69	69	export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" \| jq .shasum -r)"
70	70
71		git clone https://github.com/ziglang/www.ziglang.org --depth 1
	71	git clone git@github.com:ziglang/www.ziglang.org.git
72	72	cd www.ziglang.org
	73	git checkout hugo-redesign
73	74	export MASTER_DATE="$(date +%Y-%m-%d)"
74	75	export MASTER_VERSION="$VERSION"
75	76	"../$ZIG" run update-download-page.zig
76	77
77	78	$S3CMD put -P --no-mime-magic --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
	79
78	80	$S3CMD put -P --no-mime-magic "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
79		$S3CMD put -P --no-mime-magic www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
80		$S3CMD put -P --no-mime-magic www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
	81	$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/index.html" s3://ziglang.org/documentation/master/std/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
	82	$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/data.js" s3://ziglang.org/documentation/master/std/data.js --add-header="Cache-Control: max-age=0, must-revalidate"
	83	$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/main.js" s3://ziglang.org/documentation/master/std/main.js --add-header="Cache-Control: max-age=0, must-revalidate"
	84
	85	$S3CMD put -P --no-mime-magic ci/out/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
	86	$S3CMD put -P --no-mime-magic ci/out/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
	87
	88	cp ci/out/index.json data/releases.json
	89	git add data/releases.json
	90	git commit -m "CI: update releases.json"
	91	git push origin hugo-redesign