7aa03f39d8
Aptly may process request for long time enough to make CloudFlare or server drop connection. Treat this as false failure and don't stop GitHub Actions with error status.
308 lines
13 KiB
YAML
308 lines
13 KiB
YAML
name: Packages
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
paths:
|
|
- 'packages/**'
|
|
pull_request:
|
|
paths:
|
|
- 'packages/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
packages:
|
|
description: "A space-separated names of packages selected for rebuilding"
|
|
required: true
|
|
|
|
jobs:
|
|
build:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
ANDROID_HOME: "/opt/termux/android-sdk"
|
|
NDK: "/opt/termux/android-ndk"
|
|
strategy:
|
|
matrix:
|
|
target_arch: [aarch64, arm, i686, x86_64]
|
|
fail-fast: false
|
|
steps:
|
|
- name: Clone repository
|
|
uses: actions/checkout@v2
|
|
with:
|
|
fetch-depth: 1000
|
|
- name: Build
|
|
run: |
|
|
if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
|
|
BASE_COMMIT=$(jq --raw-output .pull_request.base.sha "$GITHUB_EVENT_PATH")
|
|
OLD_COMMIT=$(jq --raw-output .commits[0].id "$GITHUB_EVENT_PATH")
|
|
HEAD_COMMIT=$(jq --raw-output .commits[-1].id "$GITHUB_EVENT_PATH")
|
|
if [ "$BASE_COMMIT" = "null" ]; then
|
|
if [ "$OLD_COMMIT" = "$HEAD_COMMIT" ]; then
|
|
# Single-commit push.
|
|
echo "Processing commit: ${HEAD_COMMIT}"
|
|
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${HEAD_COMMIT}")
|
|
else
|
|
# Multi-commit push.
|
|
OLD_COMMIT="${OLD_COMMIT}~1"
|
|
echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}"
|
|
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}")
|
|
fi
|
|
else
|
|
# Pull requests.
|
|
echo "Processing pull request #$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"): ${BASE_COMMIT}..HEAD"
|
|
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${BASE_COMMIT}" "HEAD")
|
|
fi
|
|
fi
|
|
mkdir -p ./artifacts ./debs
|
|
touch ./debs/.placeholder
|
|
if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
|
|
# Process tag '%ci:no-build' that may be added as line to commit message.
|
|
# Forces CI to cancel current build with status 'passed'.
|
|
if grep -qiP '^\s*%ci:no-build\s*$' <(git log --format="%B" -n 1 "HEAD"); then
|
|
tar cf artifacts/debs-${{ matrix.target_arch }}.tar debs
|
|
echo "[!] Force exiting as tag '%ci:no-build' was applied to HEAD commit message."
|
|
exit 0
|
|
fi
|
|
# Build local Docker image if setup scripts were changed.
|
|
# Useful for pull requests submitting changes for both build environment and packages.
|
|
if grep -qP '^scripts/(Dockerfile|setup-ubuntu\.sh)$' <<< "$CHANGED_FILES"; then
|
|
echo "Detected changes for environment setup scripts. Building custom Docker image now."
|
|
cd ./scripts
|
|
docker build -t termux/package-builder:latest .
|
|
cd ..
|
|
fi
|
|
# Parse changed files and identify new packages and deleted packages.
|
|
# Create lists of those packages that will be passed to upload job for
|
|
# further processing.
|
|
while read -r file; do
|
|
if ! [[ $file == packages/* ]]; then
|
|
# This file does not belong to a package, so ignore it
|
|
continue
|
|
fi
|
|
if [[ $file =~ ^packages/([a-z0-9+-]*)/([a-z0-9+-]*).subpackage.sh$ ]]; then
|
|
# A subpackage was modified, check if it was deleted or just updated
|
|
pkg=${BASH_REMATCH[1]}
|
|
subpkg=${BASH_REMATCH[2]}
|
|
if [ ! -f "packages/${pkg}/${subpkg}.subpackage.sh" ]; then
|
|
echo "$subpkg" >> ./deleted_packages.txt
|
|
fi
|
|
elif [[ $file =~ ^packages/([a-z0-9+-]*)/.*$ ]]; then
|
|
# package, check if it was deleted or updated
|
|
pkg=${BASH_REMATCH[1]}
|
|
if [ ! -d "packages/${pkg}" ]; then
|
|
echo "$pkg" >> ./deleted_packages.txt
|
|
else
|
|
echo "$pkg" >> ./built_packages.txt
|
|
# If there are subpackages we want to create a list of those
|
|
# as well
|
|
for file in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do
|
|
echo "$(basename "${file%%.subpackage.sh}")" >> ./built_subpackages.txt
|
|
done
|
|
fi
|
|
fi
|
|
done<<<${CHANGED_FILES}
|
|
else
|
|
for pkg in ${{ github.event.inputs.packages }}; do
|
|
echo "$pkg" >> ./built_packages.txt
|
|
for subpkg in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do
|
|
echo "$(basename "${subpkg%%.subpackage.sh}")" >> ./built_subpackages.txt
|
|
done
|
|
done
|
|
fi
|
|
|
|
# Fix so that lists do not contain duplicates
|
|
if [ -f ./built_packages.txt ]; then
|
|
uniq ./built_packages.txt > ./built_packages.txt.tmp
|
|
mv ./built_packages.txt.tmp ./built_packages.txt
|
|
fi
|
|
if [ -f ./built_subpackages.txt ]; then
|
|
uniq ./built_subpackages.txt > ./built_subpackages.txt.tmp
|
|
mv ./built_subpackages.txt.tmp ./built_subpackages.txt
|
|
fi
|
|
if [ -f ./deleted_packages.txt ]; then
|
|
uniq ./deleted_packages.txt > ./deleted_packages.txt.tmp
|
|
mv ./deleted_packages.txt.tmp ./deleted_packages.txt
|
|
fi
|
|
|
|
if grep -qP '^rust$' ./built_packages.txt ; then
|
|
echo "Free additional disk space on host"
|
|
sudo apt purge -yq $(dpkg -l | grep '^ii' | awk '{ print $2 }' | grep -P '(cabal-|dotnet-|ghc-|libmono|php)') \
|
|
liblldb-6.0 libllvm6.0:amd64 mono-runtime-common monodoc-manual powershell ruby
|
|
sudo apt autoremove -yq
|
|
sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift
|
|
fi
|
|
|
|
if [ -f ./built_packages.txt ]; then
|
|
./scripts/lint-packages.sh $(cat ./built_packages.txt | awk '{print "packages/"$1"/build.sh"}')
|
|
./scripts/run-docker.sh ./build-package.sh -o ./debs -I -a ${{ matrix.target_arch }} $(cat ./built_packages.txt)
|
|
fi
|
|
|
|
test -d ./termux-packages/debs && mv ./termux-packages/debs/* ./debs/
|
|
# Put package lists into directory with *.deb files so they will be transferred to
|
|
# upload job.
|
|
test -f ./built_packages.txt && mv ./built_packages.txt ./debs/
|
|
test -f ./built_subpackages.txt && cat ./built_subpackages.txt >> ./debs/built_packages.txt \
|
|
&& rm ./built_subpackages.txt
|
|
test -f ./deleted_packages.txt && mv ./deleted_packages.txt ./debs/
|
|
# Files containing certain symbols (e.g. ":") will cause failure in actions/upload-artifact.
|
|
# Archiving *.deb files in a tarball to avoid issues with uploading.
|
|
tar cf artifacts/debs-${{ matrix.target_arch }}-${{ github.sha }}.tar debs
|
|
- name: Checksums for built *.deb files
|
|
run: |
|
|
find debs -type f -name "*.deb" -exec sha256sum "{}" \; | sort -k2
|
|
- name: Store *.deb files
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: termux-packages-${{ matrix.target_arch }}-${{ github.sha }}
|
|
path: ./artifacts
|
|
|
|
upload:
|
|
if: github.event_name != 'pull_request'
|
|
needs: build
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone repository
|
|
uses: actions/checkout@v2
|
|
- name: Get *.deb files
|
|
uses: actions/download-artifact@v2
|
|
with:
|
|
path: ./
|
|
- name: Upload to packages.termux.org
|
|
env:
|
|
REPOSITORY_NAME: termux-main
|
|
REPOSITORY_DISTRIBUTION: stable
|
|
run: |
|
|
for archive in termux-packages-*/*.tar; do
|
|
tar xf "$archive"
|
|
done
|
|
|
|
# Function for deleting temporary directory with uploaded files from
|
|
# the server.
|
|
aptly_delete_dir() {
|
|
echo "[$(date +%H:%M:%S)] Deleting uploads temporary directory..."
|
|
|
|
curl_response=$(
|
|
curl \
|
|
--silent \
|
|
--retry 2 \
|
|
--retry-delay 3 \
|
|
--user "${{ secrets.APTLY_API_AUTH }}" \
|
|
--user-agent "Termux-Packages/1.0 (https://github.com/termux/termux-packages)" \
|
|
--request DELETE \
|
|
--write-out "|%{http_code}" \
|
|
https://packages.termux.org/aptly-api/files/${REPOSITORY_NAME}-${{ github.sha }}
|
|
)
|
|
|
|
http_status_code=$(echo "$curl_response" | cut -d'|' -f2)
|
|
|
|
if [ "$http_status_code" != "200" ]; then
|
|
echo "[$(date +%H:%M:%S)] Warning: server returned $http_status_code code while deleting temporary directory."
|
|
fi
|
|
}
|
|
|
|
# Upload file to temporary directory.
|
|
uploaded_files=false
|
|
shopt -s nullglob
|
|
for filename in $(cat debs/built_packages.txt | sed -E 's/(..*)/debs\/\1_\*.deb debs\/\1-static_\*.deb/g'); do
|
|
curl_response=$(
|
|
curl \
|
|
--silent \
|
|
--retry 2 \
|
|
--retry-delay 3 \
|
|
--user "${{ secrets.APTLY_API_AUTH }}" \
|
|
--user-agent "Termux-Packages/1.0 (https://github.com/termux/termux-packages)" \
|
|
--request POST \
|
|
--form file=@${filename} \
|
|
--write-out "|%{http_code}" \
|
|
https://packages.termux.org/aptly-api/files/${REPOSITORY_NAME}-${{ github.sha }} || true
|
|
)
|
|
|
|
http_status_code=$(echo "$curl_response" | cut -d'|' -f2)
|
|
|
|
if [ "$http_status_code" = "200" ]; then
|
|
echo "[$(date +%H:%M:%S)] Uploaded: $(echo "$curl_response" | cut -d'|' -f1 | jq -r '.[]' | cut -d'/' -f2)"
|
|
elif [ "$http_status_code" = "000" ]; then
|
|
echo "[$(date +%H:%M:%S)]: Failed to upload '$filename'. Server/proxy dropped connection during upload."
|
|
echo "[$(date +%H:%M:%S)]: Aborting any further uploads."
|
|
aptly_delete_dir
|
|
exit 1
|
|
else
|
|
# Manually cleaning up the temporary directory to reclaim disk space.
|
|
# Don't rely on scheduled server-side scripts.
|
|
echo "[$(date +%H:%M:%S)] Error: failed to upload '$filename'. Server returned $http_status_code code."
|
|
echo "[$(date +%H:%M:%S)] Aborting any further uploads."
|
|
aptly_delete_dir
|
|
exit 1
|
|
fi
|
|
|
|
uploaded_files=true
|
|
done
|
|
shopt -u nullglob
|
|
|
|
# Publishing repository changes.
|
|
if [ "$uploaded_files" = "true" ]; then
|
|
echo "[$(date +%H:%M:%S)] Adding packages to repository '$REPOSITORY_NAME'..."
|
|
http_status_code=""
|
|
curl_response=$(
|
|
curl \
|
|
--silent \
|
|
--retry 2 \
|
|
--retry-delay 3 \
|
|
--user "${{ secrets.APTLY_API_AUTH }}" \
|
|
--user-agent "Termux-Packages/1.0 (https://github.com/termux/termux-packages)" \
|
|
--request POST \
|
|
--write-out "|%{http_code}" \
|
|
https://packages-cf.termux.org/aptly-api/repos/${REPOSITORY_NAME}/file/${REPOSITORY_NAME}-${{ github.sha }} || true
|
|
)
|
|
http_status_code=$(echo "$curl_response" | cut -d'|' -f2)
|
|
|
|
if [ "$http_status_code" = "200" ]; then
|
|
warnings=$(echo "$curl_response" | cut -d'|' -f1 | jq '.Report.Warnings' | jq -r '.[]')
|
|
if [ -n "$warnings" ]; then
|
|
echo "[$(date +%H:%M:%S)] APTLY WARNINGS (NON-CRITICAL):"
|
|
echo
|
|
echo "$warnings"
|
|
echo
|
|
fi
|
|
else
|
|
echo "[$(date +%H:%M:%S)] Error: got http_status_code == '$http_status_code', packages may not appear in repository."
|
|
fi
|
|
|
|
# Usually temporary directory is deleted automatically, but in certain cases it is left.
|
|
aptly_delete_dir
|
|
|
|
# Final part to make changes appear in web root.
|
|
echo "[$(date +%H:%M:%S)] Publishing repository changes..."
|
|
http_status_code=""
|
|
curl_response=$(
|
|
curl \
|
|
--silent \
|
|
--retry 2 \
|
|
--retry-delay 3 \
|
|
--user "${{ secrets.APTLY_API_AUTH }}" \
|
|
--user-agent "Termux-Packages/1.0 (https://github.com/termux/termux-packages)" \
|
|
--header 'Content-Type: application/json' \
|
|
--request PUT \
|
|
--data '{"Signing": {"Passphrase": "${{ secrets.GPG_PASSPHRASE }}"}}' \
|
|
--write-out '|%{http_code}' \
|
|
https://packages-cf.termux.org/aptly-api/publish/${REPOSITORY_NAME}/${REPOSITORY_DISTRIBUTION} || true
|
|
)
|
|
http_status_code=$(echo "$curl_response" | cut -d'|' -f2 | grep -oP '\d{3}$')
|
|
|
|
if [ "$http_status_code" = "200" ]; then
|
|
echo "[$(date +%H:%M:%S)] Repository has been updated successfully."
|
|
elif [ "$http_status_code" = "000" ]; then
|
|
echo "[$(date +%H:%M:%S)] Warning: server/proxy has dropped connection."
|
|
# Ignore - nothing can be done with that unless we change proxy.
|
|
#exit 1
|
|
elif [ "$http_status_code" = "504" ]; then
|
|
echo "[$(date +%H:%M:%S)] Warning: request processing time was too long, connection dropped."
|
|
# Ignore - nothing can be done with that unless we change repository
|
|
# management tool or reduce repository size.
|
|
#exit 1
|
|
else
|
|
echo "[$(date +%H:%M:%S)] Error: got http_status_code == '$http_status_code'"
|
|
exit 1
|
|
fi
|
|
fi
|