termux-packages/scripts/bin/update-packages

#!/usr/bin/env bash

set -u

# Following variables should be set in environment outside of this script.
# Build updated packages.
: "${BUILD_PACKAGES:=false}"
# Commit changes to Git.
: "${GIT_COMMIT_PACKAGES:=false}"
# Push changes to remote.
: "${GIT_PUSH_PACKAGES:=false}"

export TERMUX_PKG_UPDATE_METHOD=""         # Which method to use for updating? (repology, github or gitlab)
export TERMUX_PKG_UPDATE_TAG_TYPE=""       # Whether to use latest-release-tag or newest-tag.
export TERMUX_GITLAB_API_HOST="gitlab.com" # Default host for gitlab-ci.
export TERMUX_PKG_AUTO_UPDATE=false        # Whether to auto-update or not. Disabled by default.
export TERMUX_PKG_UPDATE_VERSION_REGEXP="" # Regexp to extract version.
export TERMUX_REPOLOGY_DATA_FILE
TERMUX_REPOLOGY_DATA_FILE="$(mktemp -t termux-repology.XXXXXX)" # File to store repology data.

export TERMUX_SCRIPTDIR
TERMUX_SCRIPTDIR=$(realpath "$(dirname "$0")/../..") # Root of repository.

export TERMUX_PACKAGES_DIRECTORIES
TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json)

# Define few more variables used by scripts.
# shellcheck source=scripts/properties.sh
. "${TERMUX_SCRIPTDIR}/scripts/properties.sh"

# Utility function to write error message to stderr.
# shellcheck source=scripts/updates/utils/termux_error_exit.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_error_exit.sh

# Utility function to write updated version to build.sh.
# shellcheck source=scripts/updates/utils/termux_pkg_upgrade_version.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_upgrade_version.sh

# Utility function to check if package needs to be updated, based on version comparison.
# shellcheck source=scripts/updates/utils/termux_pkg_is_update_needed.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_pkg_is_update_needed.sh

# Wrapper around github api to get latest release or newest tag.
# shellcheck source=scripts/updates/api/termux_github_api_get_tag.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_github_api_get_tag.sh

# Wrapper around gitlab api to get latest release or newest tag.
# shellcheck source=scripts/updates/api/termux_gitlab_api_get_tag.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_gitlab_api_get_tag.sh

# Function to get latest version of a package as per repology.
# shellcheck source=scripts/updates/api/termux_repology_api_get_latest_version.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/api/termux_repology_api_get_latest_version.sh

# Default auto update script for packages hosted on github.com. Should not be overrided by build.sh.
# To use custom algorithm, one should override termux_pkg_auto_update().
# shellcheck source=scripts/updates/internal/termux_github_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_auto_update.sh

# Default auto update script for packages hosted on hosts using gitlab-ci. Should not be overrided by build.sh.
# To use custom algorithm, one should override termux_pkg_auto_update().
# shellcheck source=scripts/updates/internal/termux_gitlab_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_gitlab_auto_update.sh

# Default auto update script for rest packages. Should not be overrided by build.sh.
# To use custom algorithm, one should override termux_pkg_auto_update().
# shellcheck source=scripts/updates/internal/termux_repology_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_repology_auto_update.sh

# Main script to:
# - by default, decide which update method to use,
# - but can be overrided by build.sh to use custom update method.
# - For example: see neovim-nightly's build.sh.
# shellcheck source=scripts/updates/termux_pkg_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh

_update() {
	export TERMUX_PKG_NAME
	TERMUX_PKG_NAME="$(basename "$1")"
	export TERMUX_PKG_BUILDER_DIR
	TERMUX_PKG_BUILDER_DIR="$(realpath "$1")" # Directory containing build.sh.

	IFS="," read -r -a BLACKLISTED_ARCH <<<"${TERMUX_PKG_BLACKLISTED_ARCHES:-}"
	export TERMUX_ARCH="" # Arch to test updates.
	for arch in aarch64 arm i686 x86_64; do
		# shellcheck disable=SC2076
		if [[ ! " ${BLACKLISTED_ARCH[*]} " =~ " ${arch} " ]]; then
			TERMUX_ARCH="${arch}"
			break
		fi
	done
	# Set +e +u to avoid:
	# - ending on errors such as $(which prog), where prog is not installed.
	# - error on unbound variable. (All global variables used should be covered by properties.sh and above.)
	set +e +u
	# shellcheck source=/dev/null
	. "${TERMUX_PKG_BUILDER_DIR}"/build.sh 2>/dev/null
	set -e -u

	echo # Newline.
	echo "INFO: Updating ${TERMUX_PKG_NAME} [Current version: ${TERMUX_PKG_VERSION}]"
	termux_pkg_auto_update
}

declare -A _FAILED_UPDATES=()
declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped.

_run_update() {
	local pkg_dir="$1"
	# Run each package update in separate process since we include their environment variables.
	local output=""
	{
		output=$(
			set -euo pipefail
			_update "${pkg_dir}" 2>&1 | tee /dev/fd/3 # fd 3 is used to output to stdout as well.
			exit "${PIPESTATUS[0]}"                   # Return exit code of _update.
		)
	} 3>&1
	# shellcheck disable=SC2181
	if [[ $? -ne 0 ]]; then
		_FAILED_UPDATES["$(basename "${pkg_dir}")"]="${output}"
	else
		_ALREADY_SEEN+=("$(basename "${pkg_dir}")")
	fi
}

declare -a _CACHED_ISSUE_TITLES=()
# Check if an issue with same title already exists and is open.
_gh_check_issue_exists() {
	local pkg_name="$1"
	if [[ -z "${_CACHED_ISSUE_TITLES[*]}" ]]; then
		while read -r title; do
			_CACHED_ISSUE_TITLES+=("'${title}'") # An extra quote ('') is added to avoid false positive matches.
		done <<<"$(
			gh issue list \
				--label "auto update failing" --label "bot" \
				--state open \
				--search "Auto update failing for in:title type:issue" \
				--json title | jq -r '.[] | .title'
		)"
	fi
	# shellcheck disable=SC2076 # We want literal match here, not regex based.
	if [[ "${_CACHED_ISSUE_TITLES[*]}" =~ "'Auto update failing for ${pkg_name}'" ]]; then
		return 0
	fi
	return 1
}

_should_update() {
	local pkg_dir="$1"

	if [[ ! -f "${pkg_dir}/build.sh" ]]; then
		# Fail if detected a non-package directory.
		termux_error_exit "ERROR: directory '${pkg_dir}' is not a package."
	fi

	# shellcheck disable=SC2076
	if ! grep -q '^TERMUX_PKG_AUTO_UPDATE=true$' "${pkg_dir}/build.sh" ||
		[[ " ${_ALREADY_SEEN[*]} ${!_FAILED_UPDATES[*]} " =~ " $(basename "${pkg_dir}") " ]]; then
		return 1 # Skip.
	fi
	if [[ "${GITHUB_ACTIONS:-}" == "true" ]] && _gh_check_issue_exists "$(basename "${pkg_dir}")"; then
		echo "INFO: Skipping '$(basename "${pkg_dir}")', an update issue for it hasn't been resolved yet."
		return 1
	fi

	return 0
}

shopt -s extglob
_update_dependencies() {
	local pkg_dir="$1"

	if ! grep -qE "^(TERMUX_PKG_DEPENDS|TERMUX_PKG_BUILD_DEPENDS|TERMUX_SUBPKG_DEPENDS)=" \
		"${pkg_dir}"/+(build|*.subpackage).sh; then
		return 0
	fi
	# shellcheck disable=SC2086 # Allow splitting of TERMUX_PACKAGES_DIRECTORIES.
	while read -r dep dep_dir; do
		if [[ -z $dep ]]; then
			continue
		elif [[ "${dep}" == "ERROR" ]]; then
			termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")"
		fi
		_should_update "${dep_dir}" && _run_update "${dep_dir}"
	done <<<"$("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")"
}

echo "INFO: Running update for: $*"

if [[ "$1" == "@all" ]]; then
	for repo_dir in $(jq --raw-output 'keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
		for pkg_dir in "${repo_dir}"/*; do
			! _should_update "${pkg_dir}" && continue # Skip if not needed.
			# Update all its dependencies first.
			_update_dependencies "${pkg_dir}"
			# NOTE: I am not cheacking whether dependencies were updated successfully or not.
			# There is no way I could know whether this package will build with current
			# available verions of its dependencies or needs new ones.
			# So, whatever the case may be. We just need packages to be updated in order
			# and not care about anything else in between. If something fails to update,
			# it will be reported by failure handling code, so no worries.
			_run_update "${pkg_dir}"
		done
	done
else
	for pkg in "$@"; do
		if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory.
			for repo_dir in $(jq --raw-output 'keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
				if [ -d "${repo_dir}/${pkg}" ]; then
					pkg="${repo_dir}/${pkg}"
					break
				fi
			done
		fi
		# Here `pkg` is a directory.
		! _should_update "${pkg}" && continue
		_update_dependencies "${pkg}"
		_run_update "${pkg}"
	done
fi

################################################FAILURE HANDLING#################################################

_gh_create_new_issue() {
	local pkg_name="$1"

	local max_body_length=65536 # Max length of the body for one request.
	local assignee="${GITHUB_ACTOR:-}"
	local issue_number
	local body

	if [[ "${assignee:-termuxbot2}" == "termuxbot2" ]]; then
		assignee="MrAdityaAlok" # Assign myself if termuxbot2 is the actor.
	fi

	body="$(
		cat <<-EOF
			Hi, I'm Termux 🤖.

			I'm here to help you update your Termux packages.

			I've tried to update the ${pkg_name} package, but it failed.

			Here's the output of the update script:
			<details>
				<summary>Show log</summary>
				<pre lang="bash">${_FAILED_UPDATES["${pkg_name}"]}</pre>
			</details>

			<hr>
			<i>
			Above error occured when I last tried to update at $(date -u +"%Y-%m-%d %H:%M:%S UTC").<br>
			Run ID: <a href="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}">${GITHUB_RUN_ID}</a><br><br>
			<b>Note:</b> Automatic updates will be disabled until this issue is resolved.<br>
			</i>
		EOF
	)"
	issue_number=$(
		gh issue create \
			--title "Auto update failing for ${pkg_name}" \
			--body "${body:0:${max_body_length}}" \
			--label "auto update failing" --label "bot" \
			--assignee ${assignee} |
			grep -oE "[0-9]+" # Last component of the URL returned is the issue number.
	)
	if [ -z "${issue_number}" ]; then
		echo "ERROR: Failed to create issue."
		return 1
	fi

	echo "INFO: Created issue ${issue_number} for ${pkg_name}."

	if [[ -n "${body:${max_body_length}}" ]]; then
		# The body was too long, so we need to append the rest.
		while true; do
			body="${body:${max_body_length}}"
			if [[ -z "${body}" ]]; then
				break
			fi
			sleep 5 # Otherwise we might get rate limited.
			gh issue edit "$issue_number" \
				--body-file - <<<"$(
					gh issue view "$issue_number" \
						--json body \
						--jq '.body'
				)${body:0:${max_body_length}}" >/dev/null
			# NOTE: we use --body-file instead of --body to avoid shell error 'argument list too long'.
		done
	fi
}

_handle_failure() {
	echo # Newline.
	if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
		echo "INFO: Creating issue for failed updates...(if any)"
		for pkg_name in "${!_FAILED_UPDATES[@]}"; do
			_gh_create_new_issue "${pkg_name}"
		done
	else
		echo "==> Failed updates:"
		local count=0
		for pkg_name in "${!_FAILED_UPDATES[@]}"; do
			count=$((count + 1))
			echo "${count}. ${pkg_name}"
		done
		exit 1
	fi
}

if [[ ${#_FAILED_UPDATES[@]} -gt 0 ]]; then
	_handle_failure
fi