authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-03 14:00:28-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-03 14:00:28-05:00
log5001faa690ce12374260334beb5f26dfa3f61613
tree80f40305c95346fd51325e66097b8871bbf8022e
parent10046f9a52c089c656bfd4578e4ef17671ff1f75
signature Commit is signed but in an unrecognized format.

ci: stop assuming that azure will complete last

Each of the 3 CI services now trigger a sr.ht build via the on_master_success script. The sr.ht build checks if all builds have completed successfully by trying to download the JSON file for the particular version. If all builds have completed successfully then this sr.ht job will update the download page.

11 files changed, 142 insertions(+), 64 deletions(-)

.builds/freebsd.yml+3
......@@ -3,8 +3,11 @@ packages:
33 - cmake
44 - py27-s3cmd
55 - wget
6 - curl
7 - jq
68secrets:
79 - 6c60aaee-92e7-4e7d-812c-114817689b4d
10 - 2cd8c203-ba06-4504-be49-1e2b9b71efdd
811sources:
912 - https://github.com/ziglang/zig
1013tasks:
ci/azure/linux_script+3-1
......@@ -25,6 +25,7 @@ cd build
2525cmake .. -DCMAKE_BUILD_TYPE=Release
2626make -j2 install
2727./zig build test -Denable-qemu
28VERSION="$(./zig version)"
2829
2930if [ "${BUILD_REASON}" != "PullRequest" ]; then
3031 ARTIFACTSDIR="$BUILDDIR/artifacts"
......@@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
4445 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
4546
4647 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
47 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/linux-$VERSION.json"
48 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"
4849
4950 # `set -x` causes these variables to be mangled.
5051 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
......@@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
5253 echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
5354 echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
5455 echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
56 echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
5557fi
ci/azure/macos_script+1-1
......@@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
104104 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
105105
106106 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
107 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/macos-$VERSION.json"
107 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-macos-$VERSION.json"
108108
109109 # `set -x` causes these variables to be mangled.
110110 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
ci/azure/on_master_success created+10
......@@ -0,0 +1,10 @@
1#!/bin/sh
2
3set -x
4set -e
5
6sudo apt-get update -y
7sudo apt-get install -y curl jq
8
9OAUTH_TOKEN="$(cat "$DOWNLOADSECUREFILE_SECUREFILEPATH")"
10./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
ci/azure/pipelines.yml+4-12
......@@ -55,18 +55,10 @@ jobs:
5555 pool:
5656 vmImage: 'ubuntu-16.04'
5757 variables:
58 macos_tarball: $[ dependencies.BuildMacOS.outputs['main.tarball'] ]
59 macos_shasum: $[ dependencies.BuildMacOS.outputs['main.shasum'] ]
60 macos_bytesize: $[ dependencies.BuildMacOS.outputs['main.bytesize'] ]
61 linux_tarball: $[ dependencies.BuildLinux.outputs['main.tarball'] ]
62 linux_shasum: $[ dependencies.BuildLinux.outputs['main.shasum'] ]
63 linux_bytesize: $[ dependencies.BuildLinux.outputs['main.bytesize'] ]
64 windows_tarball: $[ dependencies.BuildWindows.outputs['main.tarball'] ]
65 windows_shasum: $[ dependencies.BuildWindows.outputs['main.shasum'] ]
66 windows_bytesize: $[ dependencies.BuildWindows.outputs['main.bytesize'] ]
58 version: $[ dependencies.BuildLinux.outputs['main.version'] ]
6759 steps:
6860 - task: DownloadSecureFile@1
6961 inputs:
70 secureFile: s3cfg
71 - script: ci/azure/update_download_page
72 displayName: 'Update download page'
62 secureFile: oauth_token
63 - script: ci/azure/on_master_success
64 displayName: 'master branch success hook'
ci/azure/update_download_page deleted-47
......@@ -1,47 +0,0 @@
1#!/bin/sh
2
3set -x
4set -e
5
6SRCDIR=$(pwd)
7rm -rf .git
8sudo apt-get update -y
9sudo apt-get install -y s3cmd curl jq
10
11cd "$HOME"
12wget "https://ziglang.org/builds/$LINUX_TARBALL"
13tar xf $LINUX_TARBALL
14ZIGDIR=$(basename -s .tar.xz $LINUX_TARBALL)
15ZIG="$ZIGDIR/zig"
16LANGREF="$ZIGDIR/langref.html"
17VERSION=$($ZIG version)
18SRCTARBALLDIR="zig-$VERSION"
19export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
20mv "$SRCDIR" "$SRCTARBALLDIR"
21tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
22export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
23export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
24
25# the aarch64 build has to be there too
26AARCH64_LINUX_JSON=$(curl --fail "https://ziglang.org/builds/aarch64-linux-$VERSION.json" || exit 1)
27export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
28export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
29export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
30
31# the freebsd build has to be there too
32FREEBSD_JSON=$(curl --fail "https://ziglang.org/builds/freebsd-$VERSION.json" || exit 1)
33export FREEBSD_TARBALL="$(echo "$FREEBSD_JSON" | jq .tarball -r)"
34export FREEBSD_BYTESIZE="$(echo "$FREEBSD_JSON" | jq .size -r)"
35export FREEBSD_SHASUM="$(echo "$FREEBSD_JSON" | jq .shasum -r)"
36
37git clone https://github.com/ziglang/www.ziglang.org --depth 1
38cd www.ziglang.org
39export MASTER_DATE=$(date +%Y-%m-%d)
40env
41"../$ZIG" run update-download-page.zig
42
43mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
44s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
45s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
46s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
47s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
ci/azure/windows_upload+1-1
......@@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
3030 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
3131
3232 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
33 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/windows-$VERSION.json"
33 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json"
3434
3535 # `set -x` causes these variables to be mangled.
3636 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
ci/drone/linux_script+5-1
......@@ -8,7 +8,7 @@ BUILDDIR="$(pwd)"
88DISTDIR="$(pwd)/dist"
99
1010apk update
11apk add py3-pip xz perl-utils
11apk add py3-pip xz perl-utils jq curl
1212pip3 install s3cmd
1313
1414# Make the `zig version` number consistent.
......@@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then
6060
6161 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
6262 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json"
63 if [ "$GITBRANCH" = "master" ]; then
64 cd "$BUILDDIR"
65 ./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN"
66 fi
6367fi
ci/srht/freebsd_script+7-1
......@@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then
8585 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
8686
8787 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
88 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/freebsd-$VERSION.json"
88 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
89
90 if [ "$GITBRANCH" = "master" ]; then
91 OAUTH_TOKEN="$(cat ~/.oauth_token)"
92 cd "$ZIGDIR"
93 ./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
94 fi
8995fi
ci/srht/on_master_success created+33
......@@ -0,0 +1,33 @@
1#!/bin/sh
2
3# This script must run on a lot of different platforms.
4# It assumes that the following things are installed:
5# * curl
6# * jq
7
8set -x
9set -e
10
11VERSION="$1"
12OAUTH_TOKEN="$2"
13YML_FILE="tmp.yml"
14
15echo "image: ubuntu/xenial" >> "$YML_FILE"
16echo "packages:" >> "$YML_FILE"
17echo " - s3cmd" >> "$YML_FILE"
18echo " - curl" >> "$YML_FILE"
19echo " - jq" >> "$YML_FILE"
20echo "secrets:" >> "$YML_FILE"
21echo " - 6c60aaee-92e7-4e7d-812c-114817689b4d" >> "$YML_FILE"
22echo "sources:" >> "$YML_FILE"
23echo " - https://github.com/ziglang/zig" >> "$YML_FILE"
24echo "tasks:" >> "$YML_FILE"
25echo " - build: cd zig && ./ci/srht/update_download_page $VERSION" >> "$YML_FILE"
26
27jq <$YML_FILE -sR '{
28 "manifest": .,
29 }' | curl \
30 -H Authorization:"token $OAUTH_TOKEN" \
31 -H Content-Type:application/json \
32 -X POST \
33 -d @- "https://builds.sr.ht/api/jobs"
ci/srht/update_download_page created+75
......@@ -0,0 +1,75 @@
1#!/bin/sh
2
3set -x
4set -e
5
6VERSION="$1"
7SRCDIR="$(pwd)"
8NATIVE_TARBALL="zig-linux-$(uname -m)-$VERSION.tar.xz"
9
10# Check for all the builds being completed. It's expected that this script is run several times
11# before they are all complete.
12AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json"
13X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json"
14X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json"
15X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json"
16X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
17
18# If any of these fail, it's not really this job failing; rather we have detected
19# that this job will be called again later when other jobs have completed.
20curl --fail -I "$AARCH64_LINUX_JSON_URL" || exit 0
21curl --fail -I "$X86_64_LINUX_JSON_URL" || exit 0
22curl --fail -I "$X86_64_WINDOWS_JSON_URL" || exit 0
23curl --fail -I "$X86_64_MACOS_JSON_URL" || exit 0
24curl --fail -I "$X86_64_FREEBSD_JSON_URL" || exit 0
25
26rm -rf .git
27
28cd "$HOME"
29wget "https://ziglang.org/builds/$NATIVE_TARBALL"
30tar xf "$NATIVE_TARBALL"
31ZIGDIR=$(basename -s .tar.xz $NATIVE_TARBALL)
32ZIG="$ZIGDIR/zig"
33LANGREF="$ZIGDIR/langref.html"
34SRCTARBALLDIR="zig-$VERSION"
35export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
36mv "$SRCDIR" "$SRCTARBALLDIR"
37tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
38export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
39export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
40
41X86_64_WINDOWS_JSON=$(curl --fail "X86_64_WINDOWS_JSON_URL" || exit 1)
42export X86_64_WINDOWS_TARBALL="$(echo "$X86_64_WINDOWS_JSON" | jq .tarball -r)"
43export X86_64_WINDOWS_BYTESIZE="$(echo "$X86_64_WINDOWS_JSON" | jq .size -r)"
44export X86_64_WINDOWS_SHASUM="$(echo "$X86_64_WINDOWS_JSON" | jq .shasum -r)"
45
46X86_64_MACOS_JSON=$(curl --fail "X86_64_MACOS_JSON_URL" || exit 1)
47export X86_64_MACOS_TARBALL="$(echo "$X86_64_MACOS_JSON" | jq .tarball -r)"
48export X86_64_MACOS_BYTESIZE="$(echo "$X86_64_MACOS_JSON" | jq .size -r)"
49export X86_64_MACOS_SHASUM="$(echo "$X86_64_MACOS_JSON" | jq .shasum -r)"
50
51X86_64_LINUX_JSON=$(curl --fail "X86_64_LINUX_JSON_URL" || exit 1)
52export X86_64_LINUX_TARBALL="$(echo "$X86_64_LINUX_JSON" | jq .tarball -r)"
53export X86_64_LINUX_BYTESIZE="$(echo "$X86_64_LINUX_JSON" | jq .size -r)"
54export X86_64_LINUX_SHASUM="$(echo "$X86_64_LINUX_JSON" | jq .shasum -r)"
55
56AARCH64_LINUX_JSON=$(curl --fail "AARCH64_LINUX_JSON_URL" || exit 1)
57export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
58export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
59export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
60
61X86_64_FREEBSD_JSON=$(curl --fail "X86_64_FREEBSD_JSON_URL" || exit 1)
62export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)"
63export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)"
64export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)"
65
66git clone https://github.com/ziglang/www.ziglang.org --depth 1
67cd www.ziglang.org
68export MASTER_DATE="$(date +%Y-%m-%d)"
69env
70"../$ZIG" run update-download-page.zig
71
72s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
73s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
74s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
75s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"