diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 083725289..bb1d9e8e2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,3 +50,23 @@ build-x86_64: <<: *build_job variables: TERMUX_ARCH: x86_64 + +## Submit build packages to repository. +#upload-packages: +# stage: deploy +# only: +# - android-7@xeffyr/termux-packages +# - master@xeffyr/termux-packages +# script: +# - MODIFIED_PACKAGES=$(./scripts/get-modified-packages.sh) +# - | +# if [ -n "$MODIFIED_PACKAGES" ]; then +# ./scripts/bintray-add-package.sh --path ./termux-packages/debs $MODIFIED_PACKAGES +# else +# exit 0 +# fi +# retry: +# max: 2 +# when: +# - runner_system_failure +# - unknown_failure diff --git a/scripts/bintray-add-package.sh b/scripts/bintray-add-package.sh new file mode 100755 index 000000000..a1bd92f9d --- /dev/null +++ b/scripts/bintray-add-package.sh @@ -0,0 +1,481 @@ +#!/bin/bash +## +## Package uploader for Bintray. +## +## Leonid Plyushch (C) 2019 +## +## This program is free software: you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation, either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program. If not, see . +## + +set -e + +TERMUX_PACKAGES_BASEDIR=$(realpath "$(dirname "$0")/../") +if [ ! -d "$TERMUX_PACKAGES_BASEDIR/packages" ]; then + echo "[!] Cannot find directory 'packages'." >&2 + exit 1 +fi + +# In this variable a package metadata will be stored. +declare -gA PACKAGE_METADATA + +# Initialize default configuration. +DEBFILES_DIR_PATH="$TERMUX_PACKAGES_BASEDIR/debs" +PACKAGE_DELETE_MODE=false + +# Bintray-specific configuration. +BINTRAY_REPO_NAME="termux-packages-24" +BINTRAY_REPO_GITHUB="termux/termux-packages" +BINTRAY_REPO_DISTRIBUTION="stable" +BINTRAY_REPO_COMPONENT="main" + +# Bintray credentials that should be set as external environment +# variables by user. +: "${BINTRAY_USERNAME:=""}" +: "${BINTRAY_API_KEY:=""}" +: "${BINTRAY_GPG_SUBJECT:=""}" +: "${BINTRAY_GPG_PASSPHRASE:=""}" + +# If BINTRAY_GPG_SUBJECT is not specified, then signing will be +# done with gpg key of subject '$BINTRAY_USERNAME'. +if [ -z "$BINTRAY_GPG_SUBJECT" ]; then + BINTRAY_GPG_SUBJECT="$BINTRAY_USERNAME" +fi + +# Packages are built and uploaded for Termux organisation. +BINTRAY_SUBJECT="termux" + +################################################################### + +json_metadata_dump() { + local pkg_licenses + + SAVEIFS=$IFS; IFS=","; + for license in ${PACKAGE_METADATA['LICENSES']}; do + pkg_licenses+="\"$(echo "${license}" | sed -r 's/^\s*(\S+(\s+\S+)*)\s*$/\1/')\"," + done + pkg_licenses=${pkg_licenses%%,}; IFS=$SAVEIFS; + +cat << EOF +{ + "name": "${PACKAGE_METADATA['NAME']}", + "desc": "${PACKAGE_METADATA['DESCRIPTION']}", + "version": "${PACKAGE_METADATA['VERSION_FULL']}", + "licenses": [${pkg_licenses}], + "vcs_url": "https://github.com/${BINTRAY_REPO_GITHUB}", + "website_url": "${PACKAGE_METADATA['WEBSITE_URL']}", + "issue_tracker_url": "https://github.com/${BINTRAY_REPO_GITHUB}/issues", + "github_repo": "${BINTRAY_REPO_GITHUB}", + "public_download_numbers": "true", + "public_stats": "false" +} +EOF +} + +delete_package() { + local package_name=$1 + local curl_response + local http_status_code + local api_response_message + + echo -n "[@] Deleting published package '$package_name' from remote... " >&2 + curl_response=$( + curl \ + --silent \ + --user "${BINTRAY_USERNAME}:${BINTRAY_API_KEY}" \ + --request DELETE \ + --write-out "|%{http_code}" \ + "https://api.bintray.com/packages/${BINTRAY_SUBJECT}/${BINTRAY_REPO_NAME}/${package_name}" + ) + + http_status_code=$(echo "$curl_response" | cut -d'|' -f2) + api_response_message=$(echo "$curl_response" | cut -d'|' -f1 | jq -r .message) + + case "$http_status_code" in + 200) + echo "done" >&2 + ;; + 404) + echo "no-need" >&2 + ;; + *) + echo "failure" >&2 + echo "[!] $api_response_message" >&2 + exit 1 + ;; + esac +} + +upload_package() { + local package_name=$1 + local http_status_code + local api_response_message + declare -A debfiles_catalog + + for arch in all aarch64 arm i686 x86_64; do + # Regular package. + debfiles_catalog["${package_name}_${PACKAGE_METADATA['VERSION_FULL']}_${arch}.deb"]=${arch} + + # Development package. + debfiles_catalog["${package_name}-dev_${PACKAGE_METADATA['VERSION_FULL']}_${arch}.deb"]=${arch} + + # Discover subpackages. + for file in $(find "$TERMUX_PACKAGES_BASEDIR/packages/$package_name" -maxdepth 1 -type f -iname \*.subpackage.sh | sort); do + file=$(basename "$file") + debfiles_catalog["${file%%.subpackage.sh}_${PACKAGE_METADATA['VERSION_FULL']}_${arch}.deb"]=${arch} + done + + unset debfiles + done + + # Filter out nonexistent files. + for item in "${!debfiles_catalog[@]}"; do + if [ ! -f "$DEBFILES_DIR_PATH/$item" ]; then + unset debfiles_catalog["$item"] + fi + done + + # Verify that our catalog is not empty. + if [ ${#debfiles_catalog[@]} -eq 0 ]; then + echo "[!] No *.deb files to upload." >&2 + exit 1 + fi + + # Delete entry for package (with all related debfiles). + delete_package "$package_name" + + # Create new entry for package. + echo -n "[@] Creating new entry for package '$package_name'... " >&2 + curl_response=$( + curl \ + --silent \ + --user "${BINTRAY_USERNAME}:${BINTRAY_API_KEY}" \ + --request POST \ + --header "Content-Type: application/json" \ + --data "$(json_metadata_dump)" \ + --write-out "|%{http_code}" \ + "https://api.bintray.com/packages/${BINTRAY_SUBJECT}/${BINTRAY_REPO_NAME}" + ) + + http_status_code=$(echo "$curl_response" | cut -d'|' -f2) + api_response_message=$(echo "$curl_response" | cut -d'|' -f1 | jq -r .message) + + case "$http_status_code" in + 201) + echo "done" >&2 + ;; + 409) + echo "unchanged" >&2 + ;; + *) + echo "failure" >&2 + echo "[!] $api_response_message" >&2 + exit 1 + ;; + esac + + for item in "${!debfiles_catalog[@]}"; do + local package_arch=${debfiles_catalog[$item]} + + echo -n "[*] Uploading '$item'... " >&2 + curl_response=$( + curl \ + --silent \ + --user "${BINTRAY_USERNAME}:${BINTRAY_API_KEY}" \ + --request PUT \ + --header "X-Bintray-Debian-Distribution: $BINTRAY_REPO_DISTRIBUTION" \ + --header "X-Bintray-Debian-Component: $BINTRAY_REPO_COMPONENT" \ + --header "X-Bintray-Debian-Architecture: $package_arch" \ + --header "X-Bintray-Package: ${package_name}" \ + --header "X-Bintray-Version: ${PACKAGE_METADATA['VERSION_FULL']}" \ + --upload-file "$DEBFILES_DIR_PATH/$item" \ + --write-out "|%{http_code}" \ + "https://api.bintray.com/content/${BINTRAY_SUBJECT}/${BINTRAY_REPO_NAME}/${package_arch}/${item}" + ) + + http_status_code=$(echo "$curl_response" | cut -d'|' -f2) + api_response_message=$(echo "$curl_response" | cut -d'|' -f1 | jq -r .message) + + case "$http_status_code" in + 201) + echo "done" >&2 + ;; + 409) + echo "unchanged" >&2 + ;; + *) + echo "failure" >&2 + echo "[!] $api_response_message" >&2 + exit 1 + ;; + esac + done + + # Publishing package only after uploading all it's files. This will prevent + # spawning multiple metadata-generation jobs and will allow to sign metadata + # with maintainer's key. + echo -n "[@] Publishing package '$package_name'... " >&2 + curl_response=$( + curl \ + --silent \ + --user "${BINTRAY_USERNAME}:${BINTRAY_API_KEY}" \ + --request POST \ + --header "Content-Type: application/json" \ + --data "{\"subject\":\"${BINTRAY_GPG_SUBJECT}\",\"passphrase\":\"$BINTRAY_GPG_PASSPHRASE\"}" \ + --write-out "|%{http_code}" \ + "https://api.bintray.com/content/${BINTRAY_SUBJECT}/${BINTRAY_REPO_NAME}/${package_name}/${PACKAGE_METADATA['VERSION_FULL']}/publish" + ) + + http_status_code=$(echo "$curl_response" | cut -d'|' -f2) + api_response_message=$(echo "$curl_response" | cut -d'|' -f1 | jq -r .message) + + case "$http_status_code" in + 200) + echo "done" >&2 + ;; + *) + echo "failure" >&2 + echo "[!] $api_response_message" >&2 + exit 1 + ;; + esac +} + +extract_variable_from_buildsh() { + local extracted_value + local variable_name + variable_name=$1 + + extracted_value=$( + set -o noglob + + # When sourcing external code, do not expose variables + # with sensitive information. + unset BINTRAY_API_KEY + unset BINTRAY_GPG_PASSPHRASE + unset BINTRAY_GPG_SUBJECT + unset BINTRAY_SUBJECT + unset BINTRAY_USERNAME + + [ -e "$TERMUX_PACKAGES_BASEDIR/scripts/properties.sh" ] && . "$TERMUX_PACKAGES_BASEDIR/scripts/properties.sh" + . "$TERMUX_PACKAGES_BASEDIR/packages/$package_name/build.sh" + echo "${!variable_name}" + set +o noglob + ) + + echo "$extracted_value" +} + +process_packages() { + local package_name + local buildsh_path + + for package_name in "$@"; do + buildsh_path="$TERMUX_PACKAGES_BASEDIR/packages/$package_name/build.sh" + + if [ -f "$buildsh_path" ]; then + PACKAGE_METADATA["NAME"]="$package_name" + + PACKAGE_METADATA["LICENSES"]=$(extract_variable_from_buildsh "TERMUX_PKG_LICENSE" "$buildsh_path") + if [ -z "${PACKAGE_METADATA['LICENSES']}" ]; then + echo "[!] Mandatory field 'TERMUX_PKG_LICENSE' of package '$package_name' is empty." >&2 + exit 1 + fi + + PACKAGE_METADATA["DESCRIPTION"]=$(extract_variable_from_buildsh "TERMUX_PKG_DESCRIPTION" "$buildsh_path") + if [ -z "${PACKAGE_METADATA['DESCRIPTION']}" ]; then + echo "[!] Mandatory field 'TERMUX_PKG_DESCRIPTION' of package '$package_name' is empty." >&2 + exit 1 + fi + + PACKAGE_METADATA["WEBSITE_URL"]=$(extract_variable_from_buildsh "TERMUX_PKG_HOMEPAGE" "$buildsh_path") + if [ -z "${PACKAGE_METADATA['WEBSITE_URL']}" ]; then + echo "[!] Mandatory field 'TERMUX_PKG_HOMEPAGE' of package '$package_name' is empty." >&2 + exit 1 + fi + + PACKAGE_METADATA["VERSION"]=$(extract_variable_from_buildsh "TERMUX_PKG_VERSION" "$buildsh_path") + if [ -z "${PACKAGE_METADATA['VERSION']}" ]; then + echo "[!] Mandatory field 'TERMUX_PKG_VERSION' of package '$package_name' is empty." >&2 + exit 1 + fi + + PACKAGE_METADATA["REVISION"]=$(extract_variable_from_buildsh "TERMUX_PKG_REVISION" "$buildsh_path") + if [ -n "${PACKAGE_METADATA['REVISION']}" ]; then + PACKAGE_METADATA["VERSION_FULL"]="${PACKAGE_METADATA['VERSION']}-${PACKAGE_METADATA['REVISION']}" + else + if [ "${PACKAGE_METADATA['VERSION']}" != "${PACKAGE_METADATA['VERSION']/-/}" ]; then + PACKAGE_METADATA["VERSION_FULL"]="${PACKAGE_METADATA['VERSION']}-0" + else + PACKAGE_METADATA["VERSION_FULL"]="${PACKAGE_METADATA['VERSION']}" + fi + fi + else + echo "[!] Cannot find 'build.sh' for package '$package_name'." >&2 + exit 1 + fi + + if $PACKAGE_DELETE_MODE; then + delete_package "$package_name" + else + upload_package "$package_name" + fi + done + + # In deletion mode we need to do metadata recalculation separately + # to ensure that it will be signed with maintainer's key. + if $PACKAGE_DELETE_MODE; then + local curl_response + local http_status_code + local api_response_message + + echo -n "[@] Requesting metadata recalculation... " >&2 + curl_response=$( + curl \ + --silent \ + --user "${BINTRAY_USERNAME}:${BINTRAY_API_KEY}" \ + --request POST \ + --header "Content-Type: application/json" \ + --data "{\"subject\":\"${BINTRAY_GPG_SUBJECT}\",\"passphrase\":\"$BINTRAY_GPG_PASSPHRASE\"}" \ + --write-out "|%{http_code}" \ + "https://api.bintray.com/calc_metadata/${BINTRAY_SUBJECT}/${BINTRAY_REPO_NAME}/" + ) + + http_status_code=$(echo "$curl_response" | cut -d'|' -f2) + api_response_message=$(echo "$curl_response" | cut -d'|' -f1 | jq -r .message) + + case "$http_status_code" in + 202) + echo "done" >&2 + ;; + *) + echo "failure" >&2 + echo "[!] $api_response_message" >&2 + ;; + esac + fi +} + +show_usage() { + { + echo + echo "Usage: bintray-add-package.sh [OPTIONS] [package name] ..." + echo + echo "Package uploader script for Bintray." + echo + echo "Options:" + echo + echo " -d, --delete Delete package instead of uploading." + echo + echo " -h, --help Print this help." + echo + echo " -p, --path [path] Override path to directory with" + echo " the *.deb files." + echo + echo "Credentials are specified via environment variables:" + echo + echo " BINTRAY_USERNAME - User name." + echo " BINTRAY_API_KEY - User's API key." + echo " BINTRAY_GPG_SUBJECT - Owner of GPG key." + echo " BINTRAY_GPG_PASSPHRASE - GPG key passphrase." + echo + } >&2 +} + +################################################################### + +while getopts ":-:hdp:" opt; do + case "$opt" in + -) + case "$OPTARG" in + delete) + PACKAGE_DELETE_MODE=true + ;; + help) + show_usage + exit 0 + ;; + path) + DEBFILES_DIR_PATH="${!OPTIND}" + OPTIND=$((OPTIND + 1)) + + if [ -z "$DEBFILES_DIR_PATH" ]; then + echo "[!] Option '--${OPTARG}' requires argument." >&2 + show_usage + exit 1 + fi + + if [ ! -d "$DEBFILES_DIR_PATH" ]; then + echo "[!] Directory '$DEBFILES_DIR_PATH' is not exist." >&2 + show_usage + exit 1 + fi + ;; + *) + echo "[!] Invalid option '$OPTARG'." >&2 + show_usage + exit 1 + ;; + esac + ;; + d) + PACKAGE_DELETE_MODE=true + ;; + h) + show_usage + exit 0 + ;; + p) + DEBFILES_DIR_PATH="${OPTARG}" + if [ ! -d "$DEBFILES_DIR_PATH" ]; then + echo "[!] Directory '$DEBFILES_DIR_PATH' is not exist." >&2 + show_usage + exit 1 + fi + ;; + *) + echo "[!] Invalid option '-${OPTARG}'." >&2 + show_usage + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +if [ $# -gt 0 ]; then + # These variables should never be changed. + readonly DEBFILES_DIR_PATH + readonly PACKAGE_DELETE_MODE + readonly TERMUX_PACKAGES_BASEDIR + + # Without Bintray credentials this script is useless. + if [ -z "$BINTRAY_USERNAME" ]; then + echo "[!] Variable 'BINTRAY_USERNAME' is not set." >&2 + exit 1 + fi + if [ -z "$BINTRAY_API_KEY" ]; then + echo "[!] Variable 'BINTRAY_API_KEY' is not set." >&2 + exit 1 + fi + if [ -z "$BINTRAY_GPG_SUBJECT" ]; then + echo "[!] Variable 'BINTRAY_GPG_SUBJECT' is not set." >&2 + exit 1 + fi + + process_packages "$@" + exit 0 +else + echo "[!] No packages specified." >&2 + show_usage + exit 1 +fi