From f0ea9b922bf2b4a642ba78136cd01d46b5e0f624 Mon Sep 17 00:00:00 2001 From: Yaksh Bariya Date: Sun, 17 Apr 2022 17:07:08 +0530 Subject: [PATCH] monorepo: apply changes suggested by buttaface --- .github/workflows/packages.yml | 52 ++++++------ .gitignore | 6 +- build-package.sh | 27 +++--- repo.json | 5 ++ scripts/build/termux_step_setup_variables.sh | 1 - scripts/build/termux_step_start_build.sh | 2 +- scripts/setup-offline-bundle.sh | 86 ++++++++++---------- 7 files changed, 96 insertions(+), 83 deletions(-) create mode 100644 repo.json diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 1996ff4d8..eaf3e63b3 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -5,14 +5,14 @@ on: branches: - master paths: - - 'main/**' - - 'x11/**' - - 'root/**' + - 'packages/**' + - 'root-packages/**' + - 'x11-packages/**' pull_request: paths: - - 'main/**' - - 'x11/**' - - 'root/**' + - 'packages/**' + - 'root-packages/**' + - 'x11-packages/**' workflow_dispatch: inputs: packages: @@ -78,32 +78,33 @@ jobs: cd .. fi - for repo in main x11 root; do + for repo_path in $(jq --raw-output 'keys | .[]' < repo.json); do + local repo=$(jq --raw-output '.["'$repo_path'"]' < repo.json) # Parse changed files and identify new packages and deleted packages. # Create lists of those packages that will be passed to upload job for # further processing. while read -r file; do - if ! [[ $file == $repo/* ]]; then + if ! [[ $file == $repo_path/* ]]; then # This file does not belong to a package, so ignore it continue fi - if [[ $file =~ ^$repo/([.a-z0-9+-]*)/([.a-z0-9+-]*).subpackage.sh$ ]]; then + if [[ $file =~ ^$repo_path/([.a-z0-9+-]*)/([.a-z0-9+-]*).subpackage.sh$ ]]; then # A subpackage was modified, check if it was deleted or just updated pkg=${BASH_REMATCH[1]} subpkg=${BASH_REMATCH[2]} - if [ ! -f "${repo}/${pkg}/${subpkg}.subpackage.sh" ]; then + if [ ! -f "${repo_path}/${pkg}/${subpkg}.subpackage.sh" ]; then echo "$subpkg" >> ./deleted_$repo_packages.txt fi - elif [[ $file =~ ^$repo/([.a-z0-9+-]*)/.*$ ]]; then + elif [[ $file =~ ^$repo_path/([.a-z0-9+-]*)/.*$ ]]; then # package, check if it was deleted or updated pkg=${BASH_REMATCH[1]} - if [ ! -d "${repo}/${pkg}" ]; then + if [ ! -d "${repo_path}/${pkg}" ]; then echo "$pkg" >> ./deleted_$repo_packages.txt else echo "$pkg" >> ./built_$repo_packages.txt # If there are subpackages we want to create a list of those # as well - for file in $(find "${repo}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do + for file in $(find "${repo_path}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do echo "$(basename "${file%%.subpackage.sh}")" >> ./built_$repo_subpackages.txt done fi @@ -112,7 +113,8 @@ jobs: done else for pkg in ${{ github.event.inputs.packages }}; do - for repo in main x11 root; do + for repo_path in $(jq --raw-output 'keys | .[]' < repo.json); do + local repo=$(jq --raw-output '.["'$repo_path'"]' < repo.json) if [ -d "${repo}/${pkg}" ]; then echo "$pkg" >> ./built_$repo_packages.txt for subpkg in $(find "${repo}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do @@ -125,7 +127,7 @@ jobs: done fi - for repo in main x11 root; do + for repo in $(jq --raw-output '.[]' < repo.json); do # Fix so that lists do not contain duplicates if [ -f ./built_$repo_packages.txt ]; then uniq ./built_$repo_packages.txt > ./built_$repo_packages.txt.tmp @@ -154,9 +156,10 @@ jobs: - name: Lint packages run: | local package_recipes= - for repo in main x11 root; do + for repo_path in $(jq --raw-output 'keys | .[]' < repo.json); do + local repo=$(jq --raw-output '.["'$repo_path'"]' < repo.json) if [ -f ./built_$repo_packages.txt ]; then - package_recipes+=$(cat ./built_$repo_packages.txt | repo=$repo awk '{print ENVIRON["repo"]"/"$1"/build.sh"}') + package_recipes+=$(cat ./built_$repo_packages.txt | repo_path=$repo_path awk '{print ENVIRON["repo_path"]"/"$1"/build.sh"}') fi done @@ -165,7 +168,9 @@ jobs: - name: Build packages run: | local packages= - for repo in main x11 root; do + for repo_path in $(jq --raw-output 'keys | .[]' < repo.json); do + local repo=$(jq --raw-output '.["'$repo_path'"]' < repo.json) + if [ -f ./built_$repo_packages.txt ]; then packages+=$(cat ./build_$repo_packages.txt) fi @@ -178,7 +183,7 @@ jobs: run: | test -d termux-packages/output && mv termux-packages/output/* ./output/ - for repo in main x11 root; do + for repo in $(jq --raw-output '.[]' < repo.json); do mkdir debs # Put package lists into directory with *.deb files so they will be transferred to # upload job. @@ -223,7 +228,6 @@ jobs: path: ./ - name: Upload to packages.termux.org env: - REPOSITORY_NAME: termux-main REPOSITORY_DISTRIBUTION: stable REPOSITORY_URL: https://packages.termux.org/aptly-api run: | @@ -233,8 +237,8 @@ jobs: source scripts/aptly_api.sh - for repo in main x11 root; do - export REPOSITORY_NAME=termux-$repo + for repo in $(jq --raw-output '.[]' < repo.json); do + export REPOSITORY_NAME=$repo for archive in debs-*/debs-$repo-{aarch64,arm,i686,x86_64}-${{ github.sha }}.tar; do tar xf "$archive" @@ -284,8 +288,8 @@ jobs: source scripts/aptly_api.sh - for repo in main x11 root; do - export REPOSITORY_NAME=termux-$repo + for repo in $(jq --raw-output '.[]' < repo.json); do + export REPOSITORY_NAME=$repo for archive in debs-*/debs-$repo-{aarch64,arm,i686,x86_64}-${{ github.sha }}.tar; do tar xf "$archive" diff --git a/.gitignore b/.gitignore index fcea288e1..108df79ee 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,6 @@ scripts/.vagrant/ /build-tools/ # Predownloaded packages sources. -/main/*/cache -/root/*/cache -/x11/*/cache +/packages/*/cache +/root-packages/*/cache +/x11-packages/*/cache diff --git a/build-package.sh b/build-package.sh index 815beb305..1fd15b5e4 100755 --- a/build-package.sh +++ b/build-package.sh @@ -39,6 +39,8 @@ if [ ! -e "$TERMUX_BUILD_LOCK_FILE" ]; then touch "$TERMUX_BUILD_LOCK_FILE" fi +export TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'keys | .[]' < ${TERMUX_SCRIPTDIR}/repo.json) + # Special variable for internal use. It forces script to ignore # lock file. : "${TERMUX_BUILD_IGNORE_LOCK:=false}" @@ -451,23 +453,24 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do # Check the package to build: TERMUX_PKG_NAME=$(basename "${PACKAGE_LIST[i]}") + export TERMUX_PKG_BUILDER_DIR= if [[ ${PACKAGE_LIST[i]} == *"/"* ]]; then # Path to directory which may be outside this repo: if [ ! -d "${PACKAGE_LIST[i]}" ]; then termux_error_exit "'${PACKAGE_LIST[i]}' seems to be a path but is not a directory"; fi - export TERMUX_PKG_BUILDER_DIR - TERMUX_PKG_BUILDER_DIR=$(realpath "${PACKAGE_LIST[i]}") + export TERMUX_PKG_BUILDER_DIR=$(realpath "${PACKAGE_LIST[i]}") else # Package name: - if [ -d "${TERMUX_SCRIPTDIR}/main/${TERMUX_PKG_NAME}" ]; then - export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/main/$TERMUX_PKG_NAME - elif [ -d "${TERMUX_SCRIPTDIR}/x11/${TERMUX_PKG_NAME}" ]; then - export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/x11/$TERMUX_PKG_NAME - elif [ -d "${TERMUX_SCRIPTDIR}/root/${TERMUX_PKG_NAME}" ]; then - export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/root/$TERMUX_PKG_NAME - elif [ -n "${TERMUX_IS_DISABLED=""}" ] && [ -d "${TERMUX_SCRIPTDIR}/disabled/${TERMUX_PKG_NAME}"]; then - export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/disabled/$TERMUX_PKG_NAME - else - termux_error_exit "No package $TERMUX_PKG_NAME found in any of the enabled repositories, if you are trying to set up your custom repository give a look at build-package.sh:460-470" + for package_directory in $TERMUX_PACKAGES_DIRECTORIES; do + if [ -d "${TERMUX_SCRIPTDIR}/${package_directory}/${TERMUX_PKG_NAME}" ]; then + export TERMUX_PKG_BUILDER_DIR=${TERMUX_SCRIPTDIR}/$package_directory/$TERMUX_PKG_NAME + break + elif [ -n "${TERMUX_IS_DISABLED=""}" ] && [ -d "${TERMUX_SCRIPTDIR}/disabled-packages/${TERMUX_PKG_NAME}"]; then + export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/disabled-packages/$TERMUX_PKG_NAME + break + fi + done + if [ -z "${TERMUX_PKG_BUILDER_DIR}" ]; then + termux_error_exit "No package $TERMUX_PKG_NAME found in any of the enabled repositories. Are you trying to set up a custom repository?" fi fi TERMUX_PKG_BUILDER_SCRIPT=$TERMUX_PKG_BUILDER_DIR/build.sh diff --git a/repo.json b/repo.json new file mode 100644 index 000000000..6a3fcb319 --- /dev/null +++ b/repo.json @@ -0,0 +1,5 @@ +{ + "packages": "termux-main", + "root-packages": "termux-root", + "x11-packages": "termux-x11" +} diff --git a/scripts/build/termux_step_setup_variables.sh b/scripts/build/termux_step_setup_variables.sh index fc53c9228..863994bf1 100644 --- a/scripts/build/termux_step_setup_variables.sh +++ b/scripts/build/termux_step_setup_variables.sh @@ -7,7 +7,6 @@ termux_step_setup_variables() { : "${TERMUX_INSTALL_DEPS:="false"}" : "${TERMUX_MAKE_PROCESSES:="$(nproc)"}" : "${TERMUX_NO_CLEAN:="false"}" - : "${TERMUX_PACKAGES_DIRECTORIES:="main x11 root"}" : "${TERMUX_PKG_API_LEVEL:="24"}" : "${TERMUX_CONTINUE_BUILD:="false"}" : "${TERMUX_QUIET_BUILD:="false"}" diff --git a/scripts/build/termux_step_start_build.sh b/scripts/build/termux_step_start_build.sh index 2f57fc7e6..80e7c028a 100644 --- a/scripts/build/termux_step_start_build.sh +++ b/scripts/build/termux_step_start_build.sh @@ -86,7 +86,7 @@ termux_step_start_build() { local TERMUX_ELF_CLEANER_SRC=$TERMUX_COMMON_CACHEDIR/termux-elf-cleaner.cpp local TERMUX_ELF_CLEANER_VERSION - TERMUX_ELF_CLEANER_VERSION=$(bash -c ". $TERMUX_SCRIPTDIR/main/termux-elf-cleaner/build.sh; echo \$TERMUX_PKG_VERSION") + TERMUX_ELF_CLEANER_VERSION=$(bash -c ". $TERMUX_SCRIPTDIR/packages/termux-elf-cleaner/build.sh; echo \$TERMUX_PKG_VERSION") termux_download \ "https://raw.githubusercontent.com/termux/termux-elf-cleaner/v$TERMUX_ELF_CLEANER_VERSION/termux-elf-cleaner.cpp" \ "$TERMUX_ELF_CLEANER_SRC" \ diff --git a/scripts/setup-offline-bundle.sh b/scripts/setup-offline-bundle.sh index 2c8d4ce46..ae44c617f 100755 --- a/scripts/setup-offline-bundle.sh +++ b/scripts/setup-offline-bundle.sh @@ -59,48 +59,50 @@ rm -rf "${TERMUX_PKG_TMPDIR}" ) # Package sources. -for p in "$TERMUX_SCRIPTDIR"/{main,x11,root}/*; do - ( - . "$TERMUX_SCRIPTDIR"/scripts/properties.sh - . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_step_get_source.sh - . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_git_clone_src.sh - . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_download_src_archive.sh - . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_unpack_src_archive.sh - - # Disable archive extraction in termux_step_get_source.sh. - termux_extract_src_archive() { - : - } - - TERMUX_PKG_NAME=$(basename "$p") - TERMUX_PKG_BUILDER_DIR="${p}" - TERMUX_PKG_CACHEDIR="${p}/cache" - TERMUX_PKG_METAPACKAGE=false - - # Set some variables to dummy values to avoid errors. - TERMUX_PKG_TMPDIR="${TERMUX_PKG_CACHEDIR}/.tmp" - TERMUX_PKG_SRCDIR="${TERMUX_PKG_CACHEDIR}/.src" - TERMUX_PKG_BUILDDIR="$TERMUX_PKG_SRCDIR" - TERMUX_PKG_HOSTBUILD_DIR=$TERMUX_PKG_TMPDIR - TERMUX_HOST_PLATFORM=aarch64-linux-android - TERMUX_ARCH_BITS=64 - TERMUX_BUILD_TUPLE=x86_64-pc-linux-gnu - TERMUX_PKG_GIT_BRANCH="" - TERMUX_DEBUG_BUILD=false - TERMUX_MAKE_PROCESSES=1 - - mkdir -p "$TERMUX_PKG_CACHEDIR" "$TERMUX_PKG_TMPDIR" "$TERMUX_PKG_SRCDIR" - cd "$TERMUX_PKG_CACHEDIR" - - . "${p}"/build.sh || true - if ! ${TERMUX_PKG_METAPACKAGE}; then - echo "Downloading sources for '$TERMUX_PKG_NAME'..." - termux_step_get_source - - # Delete dummy src and tmp directories. - rm -rf "$TERMUX_PKG_TMPDIR" "$TERMUX_PKG_SRCDIR" - fi - ) +for repo_path in $(jq --raw-output 'keys | .[]' < $TERMUX_SCRIPTDIR/repo.json); do + for p in "$TERMUX_SCRIPTDIR"/$repo_path/*; do + ( + . "$TERMUX_SCRIPTDIR"/scripts/properties.sh + . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_step_get_source.sh + . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_git_clone_src.sh + . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_download_src_archive.sh + . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_unpack_src_archive.sh + + # Disable archive extraction in termux_step_get_source.sh. + termux_extract_src_archive() { + : + } + + TERMUX_PKG_NAME=$(basename "$p") + TERMUX_PKG_BUILDER_DIR="${p}" + TERMUX_PKG_CACHEDIR="${p}/cache" + TERMUX_PKG_METAPACKAGE=false + + # Set some variables to dummy values to avoid errors. + TERMUX_PKG_TMPDIR="${TERMUX_PKG_CACHEDIR}/.tmp" + TERMUX_PKG_SRCDIR="${TERMUX_PKG_CACHEDIR}/.src" + TERMUX_PKG_BUILDDIR="$TERMUX_PKG_SRCDIR" + TERMUX_PKG_HOSTBUILD_DIR=$TERMUX_PKG_TMPDIR + TERMUX_HOST_PLATFORM=aarch64-linux-android + TERMUX_ARCH_BITS=64 + TERMUX_BUILD_TUPLE=x86_64-pc-linux-gnu + TERMUX_PKG_GIT_BRANCH="" + TERMUX_DEBUG_BUILD=false + TERMUX_MAKE_PROCESSES=1 + + mkdir -p "$TERMUX_PKG_CACHEDIR" "$TERMUX_PKG_TMPDIR" "$TERMUX_PKG_SRCDIR" + cd "$TERMUX_PKG_CACHEDIR" + + . "${p}"/build.sh || true + if ! ${TERMUX_PKG_METAPACKAGE}; then + echo "Downloading sources for '$TERMUX_PKG_NAME'..." + termux_step_get_source + + # Delete dummy src and tmp directories. + rm -rf "$TERMUX_PKG_TMPDIR" "$TERMUX_PKG_SRCDIR" + fi + ) + done done # Mark to tell build-package.sh to enable offline mode.