CI: restore old way for building & uploading packages
Do not use pipes for building & uploading packages because: * They spawn more jobs on Bintray for metadata generation - each upload is done in a separate session. * Bintray doesn't handle well concurrent uploads and metadata can be corrupted. This happened in all recent builds of platform independent packages - in each upload task file had a same name and that triggered problem with metadata. * Using delays for workaround for a problem described above works, but isn't safe.
This commit is contained in:
Leonid Pliushch
parent
2b21492262
commit
9fac976ab1
86
.cirrus.yml
86
.cirrus.yml
@ -1,72 +1,14 @@
|
||||
container:
|
||||
image: termux/package-builder:latest
|
||||
cpu: 8
|
||||
memory: 16
|
||||
|
||||
##
|
||||
## * Maximal timeout is 120 minutes.
|
||||
## * Each architecture require a separate pipe configuration.
|
||||
## * Test builds (i.e. non-uploadable) are done via separate task.
|
||||
##
|
||||
|
||||
build_pipe_template: &BUILD_PIPE_TEMPLATE
|
||||
only_if: $CIRRUS_BRANCH == 'master'
|
||||
|
||||
timeout_in: 120m
|
||||
|
||||
environment:
|
||||
BINTRAY_USERNAME: xeffyr
|
||||
BINTRAY_API_KEY: ENCRYPTED[c056d345d7a15f90b818dd9382129c27ce4ce2b802831ec890786414fc214c4203714c353a8fef3a6b9b1b1d68e4685e]
|
||||
BINTRAY_GPG_SUBJECT: termux
|
||||
BINTRAY_GPG_PASSPHRASE: ENCRYPTED[854c09c78b3318a30d3c9094b1010dfebd4c71ad9173b64479eaafaa7e941b050afaa936ad635af3709382492828316f]
|
||||
|
||||
steps:
|
||||
- image: termux/package-builder:latest
|
||||
build_script: bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
|
||||
- image: termux/package-builder:latest
|
||||
upload_script: bash ./scripts/build/ci/cirrus-ci_dispatcher.sh --upload
|
||||
output_artifacts:
|
||||
path: "./debs/*.deb"
|
||||
|
||||
|
||||
pipe:
|
||||
<< : *BUILD_PIPE_TEMPLATE
|
||||
name: Build and Upload - aarch64
|
||||
environment:
|
||||
TERMUX_ARCH: aarch64
|
||||
UPLOAD_DELAY: 0
|
||||
|
||||
pipe:
|
||||
<< : *BUILD_PIPE_TEMPLATE
|
||||
name: Build and Upload - arm
|
||||
environment:
|
||||
TERMUX_ARCH: arm
|
||||
UPLOAD_DELAY: 20
|
||||
|
||||
pipe:
|
||||
<< : *BUILD_PIPE_TEMPLATE
|
||||
name: Build and Upload - i686
|
||||
environment:
|
||||
TERMUX_ARCH: i686
|
||||
UPLOAD_DELAY: 40
|
||||
|
||||
pipe:
|
||||
<< : *BUILD_PIPE_TEMPLATE
|
||||
name: Build and Upload - x86_64
|
||||
environment:
|
||||
TERMUX_ARCH: x86_64
|
||||
UPLOAD_DELAY: 60
|
||||
|
||||
|
||||
task:
|
||||
name: Test build
|
||||
|
||||
only_if: $CIRRUS_BRANCH != 'master'
|
||||
name: Build
|
||||
|
||||
# 2 hours is a maximal timeout available for free use.
|
||||
timeout_in: 120m
|
||||
|
||||
container:
|
||||
image: termux/package-builder:latest
|
||||
|
||||
environment:
|
||||
matrix:
|
||||
TERMUX_ARCH: aarch64
|
||||
@ -74,7 +16,27 @@ task:
|
||||
TERMUX_ARCH: i686
|
||||
TERMUX_ARCH: x86_64
|
||||
|
||||
build_script: bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
|
||||
build_script: |
|
||||
bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
|
||||
|
||||
output_artifacts:
|
||||
path: "./debs/*.deb"
|
||||
|
||||
task:
|
||||
name: Publish
|
||||
|
||||
only_if: $CIRRUS_BRANCH == 'master'
|
||||
|
||||
timeout_in: 60m
|
||||
|
||||
depends_on:
|
||||
- Build
|
||||
|
||||
environment:
|
||||
BINTRAY_USERNAME: xeffyr
|
||||
BINTRAY_API_KEY: ENCRYPTED[c056d345d7a15f90b818dd9382129c27ce4ce2b802831ec890786414fc214c4203714c353a8fef3a6b9b1b1d68e4685e]
|
||||
BINTRAY_GPG_SUBJECT: termux
|
||||
BINTRAY_GPG_PASSPHRASE: ENCRYPTED[854c09c78b3318a30d3c9094b1010dfebd4c71ad9173b64479eaafaa7e941b050afaa936ad635af3709382492828316f]
|
||||
|
||||
upload_script: |
|
||||
bash ./scripts/build/ci/cirrus-ci_dispatcher.sh --upload
|
||||
|
||||
@ -5,6 +5,9 @@
|
||||
|
||||
set -e
|
||||
|
||||
## Some packages should be excluded from auto builds.
|
||||
EXCLUDED_PACKAGES="rust texlive"
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## Preparation.
|
||||
@ -60,42 +63,42 @@ if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_C
|
||||
fi
|
||||
|
||||
if [ -z "$CIRRUS_PR" ]; then
|
||||
# Changes determined from the last commit where CI finished with status
|
||||
# 'passed' (green) and the top commit.
|
||||
if [ -z "$CIRRUS_LAST_GREEN_CHANGE" ]; then
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CIRRUS_CHANGE_IN_REPO" 2>/dev/null | grep -P "packages/")
|
||||
GIT_CHANGES="$CIRRUS_CHANGE_IN_REPO"
|
||||
else
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
|
||||
GIT_CHANGES="${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}"
|
||||
fi
|
||||
echo "[*] Changes: $GIT_CHANGES"
|
||||
else
|
||||
# Pull requests are handled in a bit different way.
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
|
||||
# Changes in pull request are determined from commits between the
|
||||
# top commit of base branch and latest commit of PR's branch.
|
||||
GIT_CHANGES="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}"
|
||||
echo "[*] Pull request: https://github.com/termux/termux-packages/pull/${CIRRUS_PR}"
|
||||
fi
|
||||
|
||||
## Determine modified packages.
|
||||
existing_dirs=""
|
||||
for dir in $(echo "$UPDATED_FILES" | grep -oP "packages/[a-z0-9+._-]+" | sort | uniq); do
|
||||
if [ -d "${REPO_DIR}/${dir}" ]; then
|
||||
existing_dirs+=" $dir"
|
||||
# Determine changes from commit range.
|
||||
PACKAGE_NAMES=$(git diff-tree --no-commit-id --name-only -r "$GIT_CHANGES" packages/ 2>/dev/null | sed -E 's@^packages/([^/]*)/build.sh@\1@')
|
||||
|
||||
## Filter deleted packages.
|
||||
for pkg in $PACKAGE_NAMES; do
|
||||
if [ ! -d "${REPO_DIR}/packages/${pkg}" ]; then
|
||||
PACKAGE_NAMES=$(sed "s/\<${pkg}\>//g" <<< "$PACKAGE_NAMES")
|
||||
fi
|
||||
done
|
||||
PACKAGE_DIRS="$existing_dirs"
|
||||
unset dir existing_dirs
|
||||
|
||||
## Get names of modified packages.
|
||||
PACKAGE_NAMES=$(echo "$PACKAGE_DIRS" | sed 's/packages\///g')
|
||||
## Filter excluded packages.
|
||||
for pkg in $EXCLUDED_PACKAGES; do
|
||||
PACKAGE_NAMES=$(sed "s/\<${pkg}\>//g" <<< "$PACKAGE_NAMES")
|
||||
done
|
||||
unset pkg
|
||||
|
||||
if [ -z "$PACKAGE_NAMES" ]; then
|
||||
echo "[*] No modified packages found." >&2
|
||||
echo "[*] No modified packages found."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
## Some packages should be excluded from auto builds.
|
||||
EXCLUDED_PACKAGES="rust texlive"
|
||||
|
||||
for excluded_pkg in $EXCLUDED_PACKAGES; do
|
||||
PACKAGE_NAMES=$(echo "$PACKAGE_NAMES" | sed "s/\<${excluded_pkg}\>//g")
|
||||
done
|
||||
unset excluded_pkg
|
||||
|
||||
set -e
|
||||
|
||||
###############################################################################
|
||||
@ -106,47 +109,61 @@ set -e
|
||||
|
||||
if ! $DO_UPLOAD; then
|
||||
echo "[*] Building packages: $PACKAGE_NAMES"
|
||||
if [ -n "$CIRRUS_PR" ]; then
|
||||
echo "[*] Pull request: https://github.com/termux/termux-packages/pull/${CIRRUS_PR}"
|
||||
else
|
||||
if [ -n "$CIRRUS_LAST_GREEN_CHANGE" ]; then
|
||||
echo "[*] Changes: ${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}"
|
||||
else
|
||||
echo "[*] Changes: ${CIRRUS_CHANGE_IN_REPO}"
|
||||
fi
|
||||
fi
|
||||
|
||||
./build-package.sh -a "$TERMUX_ARCH" -I $PACKAGE_NAMES
|
||||
else
|
||||
if [ -z "$BINTRAY_API_KEY" ]; then
|
||||
echo "[!] Can't upload without Bintray API key."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$BINTRAY_GPG_PASSPHRASE" ]; then
|
||||
echo "[!] Can't upload without GPG passphrase."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Workaround for concurrent uploads.
|
||||
if [ "$UPLOAD_DELAY" != "0" ]; then
|
||||
echo "[!] Using workaround for Bintray issue with concurrent uploads."
|
||||
echo "[!] Delaying upload by ${UPLOAD_DELAY} seconds."
|
||||
sleep "$UPLOAD_DELAY"
|
||||
fi
|
||||
|
||||
for attempt in 1 2 3; do
|
||||
echo "[*] Uploading packages to Bintray:"
|
||||
if ./scripts/package_uploader.sh -p "${PWD}/debs" $PACKAGE_NAMES; then
|
||||
break
|
||||
else
|
||||
if [ "$attempt" = "3" ]; then
|
||||
echo "[!] Went through 3 attempts of upload, giving up..."
|
||||
exit 1
|
||||
else
|
||||
echo "[!] Failure, retrying in 30 seconds..."
|
||||
sleep 30
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## Storing packages in cache // retrieving and uploading to Bintray.
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
if [ "$CIRRUS_BRANCH" = "master" ]; then
|
||||
if ! $DO_UPLOAD; then
|
||||
ARCHIVE_NAME="debs-${TERMUX_ARCH}-${CIRRUS_CHANGE_IN_REPO}.tar.gz"
|
||||
|
||||
if [ -d "${REPO_DIR}/debs" ]; then
|
||||
echo "[*] Archiving packages into '${ARCHIVE_NAME}'."
|
||||
tar zcf "$ARCHIVE_NAME" debs
|
||||
|
||||
echo "[*] Uploading '${ARCHIVE_NAME}' to cache:"
|
||||
echo
|
||||
curl --upload-file "$ARCHIVE_NAME" \
|
||||
"http://${CIRRUS_HTTP_CACHE_HOST}/${ARCHIVE_NAME}"
|
||||
echo
|
||||
fi
|
||||
else
|
||||
if [ -z "$BINTRAY_API_KEY" ]; then
|
||||
echo "[!] Can't upload without Bintray API key."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$BINTRAY_GPG_PASSPHRASE" ]; then
|
||||
echo "[!] Can't upload without GPG passphrase."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for arch in aarch64 arm i686 x86_64; do
|
||||
ARCHIVE_NAME="debs-${arch}-${CIRRUS_CHANGE_IN_REPO}.tar.gz"
|
||||
|
||||
echo "[*] Downloading '${ARCHIVE_NAME}' from cache:"
|
||||
echo
|
||||
curl --output "/tmp/${ARCHIVE_NAME}" \
|
||||
"http://${CIRRUS_HTTP_CACHE_HOST}/${ARCHIVE_NAME}"
|
||||
echo
|
||||
|
||||
if [ -s "/tmp/${ARCHIVE_NAME}" ]; then
|
||||
echo "[*] Unpacking '/tmp/${ARCHIVE_NAME}':"
|
||||
echo
|
||||
tar xvf "/tmp/${ARCHIVE_NAME}"
|
||||
echo
|
||||
else
|
||||
echo "[!] Empty archive '/tmp/${ARCHIVE_NAME}'."
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[*] Uploading packages to Bintray:"
|
||||
echo
|
||||
"${REPO_DIR}/scripts/package_uploader.sh" -p "${PWD}/debs" $PACKAGE_NAMES
|
||||
fi
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user