update CI setup
* Move package building/uploading implementation from config to a separate script. * Implement new tag '%ci:no-build' to allow immediately stop build with status 'passed'. %ci:no-build
This commit is contained in:
Leonid Pliushch
parent
f55fba5e67
commit
2563fd812c
51
.cirrus.yml
51
.cirrus.yml
@ -15,8 +15,8 @@ build_task:
|
||||
TERMUX_ARCH: i686
|
||||
TERMUX_ARCH: x86_64
|
||||
|
||||
# Do not use built-in git client provided by Cirrus as it
|
||||
# causes problems when determining changed files.
|
||||
# Do not use built-in git client provided by Cirrus as may
|
||||
# cause problems when determining changed files.
|
||||
clone_script: |
|
||||
if [[ -z "$CIRRUS_PR" ]]; then
|
||||
git clone --recursive --branch="$CIRRUS_BRANCH" "https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git" "$CIRRUS_WORKING_DIR"
|
||||
@ -27,34 +27,18 @@ build_task:
|
||||
git reset --hard "$CIRRUS_CHANGE_IN_REPO"
|
||||
fi
|
||||
|
||||
# Determine changes in repository and build modified packages.
|
||||
# Following script will determine changes via Git and build
|
||||
# modified packages.
|
||||
build_script: |
|
||||
if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then
|
||||
# If commit message contains line '%ci:reset-backlog', builds will be done
|
||||
# only for current commit.
|
||||
unset CIRRUS_LAST_GREEN_CHANGE
|
||||
unset CIRRUS_BASE_SHA
|
||||
fi
|
||||
MODIFIED_PACKAGES=$(./scripts/build/ci/determine_git_changes.sh)
|
||||
for package in $MODIFIED_PACKAGES; do
|
||||
if [ -n "$CIRRUS_PR" ]; then
|
||||
# Perform full builds for PR.
|
||||
./build-package.sh -a "$TERMUX_ARCH" "$package"
|
||||
else
|
||||
./build-package.sh -I -a "$TERMUX_ARCH" "$package"
|
||||
fi
|
||||
done
|
||||
if [ -d ./debs ]; then
|
||||
# Upload packages to cache.
|
||||
tar zcf "debs-${TERMUX_ARCH}.tar.gz" debs
|
||||
curl -s --upload-file "debs-${TERMUX_ARCH}.tar.gz" "http://$CIRRUS_HTTP_CACHE_HOST/debs-${TERMUX_ARCH}-${CIRRUS_CHANGE_IN_REPO}"
|
||||
fi
|
||||
bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
|
||||
|
||||
# Also make them downloadable from the UI.
|
||||
# Make built packages downloadable from web UI.
|
||||
output_artifacts:
|
||||
path: "./debs/*.deb"
|
||||
|
||||
# Publish packages to Bintray.
|
||||
|
||||
# Upload packages to Bintray.
|
||||
# Should be done only on branch 'master'.
|
||||
publish_task:
|
||||
only_if: $CIRRUS_BRANCH == 'master'
|
||||
|
||||
@ -69,18 +53,5 @@ publish_task:
|
||||
BINTRAY_GPG_SUBJECT: termux
|
||||
BINTRAY_GPG_PASSPHRASE: ENCRYPTED[854c09c78b3318a30d3c9094b1010dfebd4c71ad9173b64479eaafaa7e941b050afaa936ad635af3709382492828316f]
|
||||
|
||||
# Publish.
|
||||
upload_to_bintray_script: |
|
||||
if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then
|
||||
unset CIRRUS_LAST_GREEN_CHANGE
|
||||
unset CIRRUS_BASE_SHA
|
||||
fi
|
||||
MODIFIED_PACKAGES=$(./scripts/build/ci/determine_git_changes.sh)
|
||||
if [ -n "$MODIFIED_PACKAGES" ]; then
|
||||
for arch in aarch64 arm i686 x86_64; do
|
||||
curl -s -o "/tmp/debs-${arch}.tar.gz" "http://$CIRRUS_HTTP_CACHE_HOST/debs-${arch}-${CIRRUS_CHANGE_IN_REPO}"
|
||||
tar xvf "/tmp/debs-${arch}.tar.gz"
|
||||
rm -f "/tmp/debs-${arch}.tar.gz"
|
||||
done
|
||||
./scripts/package_uploader.sh $MODIFIED_PACKAGES
|
||||
fi
|
||||
upload_script: |
|
||||
bash ./scripts/build/ci/cirrus-ci_dispatcher.sh --upload
|
||||
|
||||
159
scripts/build/ci/cirrus-ci_dispatcher.sh
Executable file
159
scripts/build/ci/cirrus-ci_dispatcher.sh
Executable file
@ -0,0 +1,159 @@
|
||||
#!/bin/bash
|
||||
##
|
||||
## Determine modified packages and build/upload them.
|
||||
##
|
||||
|
||||
set -e
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## Preparation.
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
REPO_DIR=$(realpath "$(dirname "$(realpath "$0")")/../../../")
|
||||
cd "$REPO_DIR" || {
|
||||
echo "[!] Failed to cd into '$REPO_DIR'."
|
||||
exit 1
|
||||
}
|
||||
|
||||
DO_UPLOAD=false
|
||||
if [ $# -ge 1 ]; then
|
||||
if [ "$1" = "--upload" ]; then
|
||||
DO_UPLOAD=true
|
||||
fi
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## Determining changes.
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
set +e
|
||||
|
||||
# Process tag '%ci:no-build' that may be added as line to commit message.
|
||||
# Will force CI to exit with status 'passed' without performing build.
|
||||
if grep -qiP '^\s*%ci:no-build\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then
|
||||
echo "[*] Exiting with status 'passed' (tag '%ci:no-build' applied)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Process tag '%ci:reset-backlog' that may be added as line to commit message.
|
||||
# Will force CI to build changes only for the current commit.
|
||||
if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then
|
||||
echo "[*] Building only last pushed commit (tag '%ci:reset-backlog' applied)."
|
||||
unset CIRRUS_LAST_GREEN_CHANGE
|
||||
unset CIRRUS_BASE_SHA
|
||||
fi
|
||||
|
||||
if [ -z "$CIRRUS_PR" ]; then
|
||||
if [ -z "$CIRRUS_LAST_GREEN_CHANGE" ]; then
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CIRRUS_CHANGE_IN_REPO" 2>/dev/null | grep -P "packages/")
|
||||
else
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
|
||||
fi
|
||||
else
|
||||
# Pull requests are handled in a bit different way.
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
|
||||
fi
|
||||
|
||||
## Determine modified packages.
|
||||
existing_dirs=""
|
||||
for dir in $(echo "$UPDATED_FILES" | grep -oP "packages/[a-z0-9+._-]+" | sort | uniq); do
|
||||
if [ -d "${REPO_DIR}/${dir}" ]; then
|
||||
existing_dirs+=" $dir"
|
||||
fi
|
||||
done
|
||||
PACKAGE_DIRS="$existing_dirs"
|
||||
unset dir existing_dirs
|
||||
|
||||
## Get names of modified packages.
|
||||
PACKAGE_NAMES=$(echo "$PACKAGE_DIRS" | sed 's/packages\///g')
|
||||
if [ -z "$PACKAGE_NAMES" ]; then
|
||||
echo "[*] No modified packages found." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
## Some packages should be excluded from auto builds.
|
||||
EXCLUDED_PACKAGES="rust texlive"
|
||||
|
||||
for excluded_pkg in $EXCLUDED_PACKAGES; do
|
||||
PACKAGE_NAMES=$(echo "$PACKAGE_NAMES" | sed "s/\<${excluded_pkg}\>//g")
|
||||
done
|
||||
unset excluded_pkg
|
||||
|
||||
set -e
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## Building packages.
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
if ! $DO_UPLOAD; then
|
||||
echo "[*] Building packages: $PACKAGE_NAMES"
|
||||
echo
|
||||
if [ -n "$CIRRUS_PR" ]; then
|
||||
echo " Pull request: https://github.com/termux/unstable-packages/pull/${CIRRUS_PR}"
|
||||
else
|
||||
if [ -n "$CIRRUS_LAST_GREEN_CHANGE" ]; then
|
||||
echo " Changes: ${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}"
|
||||
else
|
||||
echo " Changes: ${CIRRUS_CHANGE_IN_REPO}"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
for pkg in $PACKAGE_NAMES; do
|
||||
./build-package.sh -a "$TERMUX_ARCH" -I "$pkg"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
##
|
||||
## Storing packages in cache.
|
||||
##
|
||||
###############################################################################
|
||||
|
||||
if [ "$CIRRUS_BRANCH" = "master" ]; then
|
||||
if ! $DO_UPLOAD; then
|
||||
ARCHIVE_NAME="debs-${TERMUX_ARCH}-${CIRRUS_CHANGE_IN_REPO}.tar.gz"
|
||||
|
||||
if [ -d "${REPO_DIR}/debs" ]; then
|
||||
echo "[*] Archiving packages into '${ARCHIVE_NAME}'."
|
||||
tar zcf "$ARCHIVE_NAME" debs
|
||||
|
||||
echo "[*] Uploading '${ARCHIVE_NAME}' to cache:"
|
||||
echo
|
||||
curl --upload-file "$ARCHIVE_NAME" \
|
||||
"http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
|
||||
echo
|
||||
fi
|
||||
else
|
||||
for arch in aarch64 arm i686 x86_64; do
|
||||
ARCHIVE_NAME="debs-${arch}-${CIRRUS_CHANGE_IN_REPO}.tar.gz"
|
||||
|
||||
echo "[*] Downloading '$ARCHIVE_NAME' from cache:"
|
||||
echo
|
||||
curl --output "/tmp/${ARCHIVE_NAME}" \
|
||||
"http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
|
||||
echo
|
||||
|
||||
if [ -s "/tmp/${ARCHIVE_NAME}" ]; then
|
||||
echo "[*] Unpacking '/tmp/${ARCHIVE_NAME}':"
|
||||
echo
|
||||
tar xvf "/tmp/${ARCHIVE_NAME}"
|
||||
echo
|
||||
else
|
||||
echo "[!] Empty archive '/tmp/${ARCHIVE_NAME}'."
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[*] Uploading packages to Bintray:"
|
||||
echo
|
||||
"${REPO_DIR}/scripts/package_uploader.sh" -p "${PWD}/debs" $PACKAGE_NAMES
|
||||
fi
|
||||
fi
|
||||
@ -1,98 +0,0 @@
|
||||
#!/bin/bash
|
||||
##
|
||||
## Script for detecting modified packages.
|
||||
## Designed for use with Cirrus, Gitlab or Travis CI.
|
||||
##
|
||||
## Leonid Plyushch <leonid.plyushch@gmail.com> (C) 2019
|
||||
##
|
||||
## This program is free software: you can redistribute it and/or modify
|
||||
## it under the terms of the GNU General Public License as published by
|
||||
## the Free Software Foundation, either version 3 of the License, or
|
||||
## (at your option) any later version.
|
||||
##
|
||||
## This program is distributed in the hope that it will be useful,
|
||||
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
## GNU General Public License for more details.
|
||||
##
|
||||
## You should have received a copy of the GNU General Public License
|
||||
## along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
##
|
||||
|
||||
REPO_DIR=$(realpath "$(dirname "$(realpath "$0")")/../../../")
|
||||
cd "$REPO_DIR" || {
|
||||
echo "[!] Failed to cd into '$REPO_DIR'." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ -n "$TRAVIS_COMMIT_RANGE" ]; then
|
||||
# We are on Travis CI.
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${TRAVIS_COMMIT_RANGE//.../..}" 2>/dev/null | grep -P "packages/")
|
||||
elif [ -n "$CI_COMMIT_SHA" ]; then
|
||||
# We are on Gitlab CI.
|
||||
|
||||
# Make sure that we can use commit range.
|
||||
if [ -z "$CI_COMMIT_BEFORE_SHA" ]; then
|
||||
echo "[!] CI_COMMIT_BEFORE_SHA is not set." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$CI_COMMIT_BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CI_COMMIT_SHA" 2>/dev/null | grep -P "packages/")
|
||||
else
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA}" 2>/dev/null | grep -P "packages/")
|
||||
fi
|
||||
elif [ -n "$CIRRUS_CI" ]; then
|
||||
# We are on Cirrus CI.
|
||||
if [ -z "$CIRRUS_PR" ]; then
|
||||
if [ -z "$CIRRUS_LAST_GREEN_CHANGE" ]; then
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CIRRUS_CHANGE_IN_REPO" 2>/dev/null | grep -P "packages/")
|
||||
else
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
|
||||
fi
|
||||
else
|
||||
# Pull requests are handled in a bit different way.
|
||||
UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
|
||||
fi
|
||||
else
|
||||
# Something wrong.
|
||||
echo "[!] Cannot determine git commit range." >&2
|
||||
echo " Did you executed this script under CI ?" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
## Determine modified packages.
|
||||
existing_dirs=""
|
||||
for dir in $(echo "$UPDATED_FILES" | grep -oP "packages/[a-z0-9+._-]+" | sort | uniq); do
|
||||
if [ -d "$REPO_DIR/$dir" ]; then
|
||||
existing_dirs+=" $dir"
|
||||
fi
|
||||
done
|
||||
PACKAGE_DIRS="$existing_dirs"
|
||||
unset dir existing_dirs
|
||||
|
||||
## Get names of modified packages.
|
||||
PACKAGE_NAMES=$(echo "$PACKAGE_DIRS" | sed 's/packages\///g')
|
||||
if [ -z "$PACKAGE_NAMES" ]; then
|
||||
echo "[*] No modified packages found." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
## Print names of modified packages.
|
||||
for pkg in $PACKAGE_NAMES; do
|
||||
case "$pkg" in
|
||||
# Skip packages that known to have long build time.
|
||||
rust|texlive)
|
||||
{
|
||||
echo
|
||||
echo "Package '$pkg' cannot be built via CI because it has"
|
||||
echo "long build time."
|
||||
echo
|
||||
} >&2
|
||||
continue
|
||||
;;
|
||||
*)
|
||||
echo "$pkg"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
Loading…
Reference in New Issue
Block a user