From 52c383b4d11deccd6a24d8ee1a4be8d196578009 Mon Sep 17 00:00:00 2001 From: Yaksh Bariya Date: Sat, 16 Apr 2022 12:20:09 +0530 Subject: [PATCH] chore: switch to monorepo --- .github/workflows/packages.yml | 305 +++++++++++-------- .gitignore | 4 +- build-package.sh | 12 +- scripts/build/termux_step_setup_variables.sh | 2 +- scripts/build/termux_step_start_build.sh | 2 +- scripts/buildorder.py | 5 +- scripts/setup-offline-bundle.sh | 2 +- 7 files changed, 194 insertions(+), 138 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 0ef2de378..1996ff4d8 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -5,10 +5,14 @@ on: branches: - master paths: - - 'packages/**' + - 'main/**' + - 'x11/**' + - 'root/**' pull_request: paths: - - 'packages/**' + - 'main/**' + - 'x11/**' + - 'root/**' workflow_dispatch: inputs: packages: @@ -27,7 +31,7 @@ jobs: fail-fast: false steps: - name: Clone repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 1000 - name: Gather build summary @@ -73,58 +77,69 @@ jobs: docker build -t termux/package-builder:latest . cd .. fi - # Parse changed files and identify new packages and deleted packages. - # Create lists of those packages that will be passed to upload job for - # further processing. - while read -r file; do - if ! [[ $file == packages/* ]]; then - # This file does not belong to a package, so ignore it - continue - fi - if [[ $file =~ ^packages/([.a-z0-9+-]*)/([.a-z0-9+-]*).subpackage.sh$ ]]; then - # A subpackage was modified, check if it was deleted or just updated - pkg=${BASH_REMATCH[1]} - subpkg=${BASH_REMATCH[2]} - if [ ! -f "packages/${pkg}/${subpkg}.subpackage.sh" ]; then - echo "$subpkg" >> ./deleted_packages.txt + + for repo in main x11 root; do + # Parse changed files and identify new packages and deleted packages. + # Create lists of those packages that will be passed to upload job for + # further processing. + while read -r file; do + if ! [[ $file == $repo/* ]]; then + # This file does not belong to a package, so ignore it + continue fi - elif [[ $file =~ ^packages/([.a-z0-9+-]*)/.*$ ]]; then - # package, check if it was deleted or updated - pkg=${BASH_REMATCH[1]} - if [ ! -d "packages/${pkg}" ]; then - echo "$pkg" >> ./deleted_packages.txt - else - echo "$pkg" >> ./built_packages.txt - # If there are subpackages we want to create a list of those - # as well - for file in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do - echo "$(basename "${file%%.subpackage.sh}")" >> ./built_subpackages.txt - done + if [[ $file =~ ^$repo/([.a-z0-9+-]*)/([.a-z0-9+-]*).subpackage.sh$ ]]; then + # A subpackage was modified, check if it was deleted or just updated + pkg=${BASH_REMATCH[1]} + subpkg=${BASH_REMATCH[2]} + if [ ! -f "${repo}/${pkg}/${subpkg}.subpackage.sh" ]; then + echo "$subpkg" >> ./deleted_$repo_packages.txt + fi + elif [[ $file =~ ^$repo/([.a-z0-9+-]*)/.*$ ]]; then + # package, check if it was deleted or updated + pkg=${BASH_REMATCH[1]} + if [ ! -d "${repo}/${pkg}" ]; then + echo "$pkg" >> ./deleted_$repo_packages.txt + else + echo "$pkg" >> ./built_$repo_packages.txt + # If there are subpackages we want to create a list of those + # as well + for file in $(find "${repo}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do + echo "$(basename "${file%%.subpackage.sh}")" >> ./built_$repo_subpackages.txt + done + fi fi - fi - done<<<${CHANGED_FILES} + done<<<${CHANGED_FILES} + done else for pkg in ${{ github.event.inputs.packages }}; do - echo "$pkg" >> ./built_packages.txt - for subpkg in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do - echo "$(basename "${subpkg%%.subpackage.sh}")" >> ./built_subpackages.txt + for repo in main x11 root; do + if [ -d "${repo}/${pkg}" ]; then + echo "$pkg" >> ./built_$repo_packages.txt + for subpkg in $(find "${repo}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do + echo "$(basename "${subpkg%%.subpackage.sh}")" >> ./built_$repo_subpackages.txt + done + else + echo "Package '${pkg}' not found in any of the repo" + fi done done fi - # Fix so that lists do not contain duplicates - if [ -f ./built_packages.txt ]; then - uniq ./built_packages.txt > ./built_packages.txt.tmp - mv ./built_packages.txt.tmp ./built_packages.txt - fi - if [ -f ./built_subpackages.txt ]; then - uniq ./built_subpackages.txt > ./built_subpackages.txt.tmp - mv ./built_subpackages.txt.tmp ./built_subpackages.txt - fi - if [ -f ./deleted_packages.txt ]; then - uniq ./deleted_packages.txt > ./deleted_packages.txt.tmp - mv ./deleted_packages.txt.tmp ./deleted_packages.txt - fi + for repo in main x11 root; do + # Fix so that lists do not contain duplicates + if [ -f ./built_$repo_packages.txt ]; then + uniq ./built_$repo_packages.txt > ./built_$repo_packages.txt.tmp + mv ./built_$repo_packages.txt.tmp ./built_$repo_packages.txt + fi + if [ -f ./built_$repo_subpackages.txt ]; then + uniq ./built_$repo_subpackages.txt > ./built_$repo_subpackages.txt.tmp + mv ./built_$repo_subpackages.txt.tmp ./built_$repo_subpackages.txt + fi + if [ -f ./deleted_$repo_packages.txt ]; then + uniq ./deleted_$repo_packages.txt > ./deleted_$repo_packages.txt.tmp + mv ./deleted_$repo_packages.txt.tmp ./deleted_$repo_packages.txt + fi + done - name: Free additional disk space (if necessary) run: | @@ -136,45 +151,63 @@ jobs: sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift fi - - name: Build + - name: Lint packages run: | - if [ -f ./built_packages.txt ]; then - ./scripts/lint-packages.sh $(cat ./built_packages.txt | awk '{print "packages/"$1"/build.sh"}') - ./scripts/run-docker.sh ./build-package.sh -I -a ${{ matrix.target_arch }} $(cat ./built_packages.txt) - fi + local package_recipes= + for repo in main x11 root; do + if [ -f ./built_$repo_packages.txt ]; then + package_recipes+=$(cat ./built_$repo_packages.txt | repo=$repo awk '{print ENVIRON["repo"]"/"$1"/build.sh"}') + fi + done + + ./scripts/lint-packages.sh $package_recipes + + - name: Build packages + run: | + local packages= + for repo in main x11 root; do + if [ -f ./built_$repo_packages.txt ]; then + packages+=$(cat ./build_$repo_packages.txt) + fi + done + + ./scripts/run-docker.sh ./build-package.sh -I -a ${{ matrix.target_arch }} $packages - name: Generate build artifacts if: always() run: | - mkdir -p debs test -d termux-packages/output && mv termux-packages/output/* ./output/ - # Put package lists into directory with *.deb files so they will be transferred to - # upload job. - test -f ./built_packages.txt && mv ./built_packages.txt ./debs/ - test -f ./built_subpackages.txt && cat ./built_subpackages.txt >> ./debs/built_packages.txt \ - && rm ./built_subpackages.txt - test -f ./deleted_packages.txt && mv ./deleted_packages.txt ./debs/ + for repo in main x11 root; do + mkdir debs + # Put package lists into directory with *.deb files so they will be transferred to + # upload job. + test -f ./built_$repo_packages.txt && mv ./built_repo_packages.txt ./debs/built_packages.txt + test -f ./built_$repo_subpackages.txt && cat ./built_repo_subpackages.txt >> ./debs/built_packages.txt \ + && rm ./built_$repo_subpackages.txt + test -f ./deleted_$repo_packages.txt && mv ./deleted_$repo_packages.txt ./debs/deleted_packages.txt - # Move only debs from built_packages into debs/ folder before - # creating an archive. - while read -r pkg; do - # Match both $pkg.deb and $pkg-static.deb. - find output \( -name "$pkg_*.deb" -o -name "$pkg-static_*.deb" \) -type f -print0 | xargs -0r mv -t debs/ - done < <(cat ./debs/built_packages.txt) + # Move only debs from built_packages into debs/ folder before + # creating an archive. + while read -r pkg; do + # Match both $pkg.deb and $pkg-static.deb. + find output \( -name "$pkg_*.deb" -o -name "$pkg-static_*.deb" \) -type f -print0 | xargs -0r mv -t debs/ + done < <(cat ./debs/built_packages.txt) - # Files containing certain symbols (e.g. ":") will cause failure in actions/upload-artifact. - # Archiving *.deb files in a tarball to avoid issues with uploading. - tar cf artifacts/debs-${{ matrix.target_arch }}-${{ github.sha }}.tar debs + # Files containing certain symbols (e.g. ":") will cause failure in actions/upload-artifact. + # Archiving *.deb files in a tarball to avoid issues with uploading. + tar cf artifacts/debs-${repo}-${{ matrix.target_arch }}-${{ github.sha }}.tar debs + rm -r debs/ + done - name: Checksums for built *.deb files if: always() run: | find debs -type f -name "*.deb" -exec sha256sum "{}" \; | sort -k2 - name: Store *.deb files if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: - name: termux-packages-${{ matrix.target_arch }}-${{ github.sha }} + name: debs-${{ matrix.target_arch }}-${{ github.sha }} path: ./artifacts upload: @@ -183,9 +216,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Get *.deb files - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: path: ./ - name: Upload to packages.termux.org @@ -200,41 +233,48 @@ jobs: source scripts/aptly_api.sh - for archive in termux-packages-*/*.tar; do - tar xf "$archive" + for repo in main x11 root; do + export REPOSITORY_NAME=termux-$repo + + for archive in debs-*/debs-$repo-{aarch64,arm,i686,x86_64}-${{ github.sha }}.tar; do + tar xf "$archive" + done + + # Upload file to temporary directory. + uploaded_files=false + shopt -s nullglob + for filename in $(cat debs/built_packages.txt | sed -E 's/(..*)/debs\/\1_\*.deb debs\/\1-static_\*.deb/g'); do + if ! aptly_upload_file "$filename"; then + exit 1 + fi + + uploaded_files=true + done + shopt -u nullglob + + # Publishing repository changes. + if [ "$uploaded_files" = "true" ]; then + if ! aptly_add_to_repo; then + exit 1 + fi + + # Usually temporary directory is deleted automatically, but in certain cases it is left. + aptly_delete_dir + + # Final part to make changes appear in web root. + if ! aptly_publish_repo; then + exit 1 + fi + fi + + # Delete the debs folder in order to not upload packages from one repo to other + # TODO(@thunder-coding): Perhaps it would make sense to make use of tar's `-C` option in order to prevent extracting the archives twice + rm -rf debs/ done - - # Upload file to temporary directory. - uploaded_files=false - shopt -s nullglob - for filename in $(cat debs/built_packages.txt | sed -E 's/(..*)/debs\/\1_\*.deb debs\/\1-static_\*.deb/g'); do - if ! aptly_upload_file "$filename"; then - exit 1 - fi - - uploaded_files=true - done - shopt -u nullglob - - # Publishing repository changes. - if [ "$uploaded_files" = "true" ]; then - if ! aptly_add_to_repo; then - exit 1 - fi - - # Usually temporary directory is deleted automatically, but in certain cases it is left. - aptly_delete_dir - - # Final part to make changes appear in web root. - if ! aptly_publish_repo; then - exit 1 - fi - fi - name: Upload to grimler.se # Run even if upload to packages.termux.org failed: if: always() env: - REPOSITORY_NAME: termux-main REPOSITORY_DISTRIBUTION: stable REPOSITORY_URL: https://aptly-api.grimler.se run: | @@ -243,30 +283,41 @@ jobs: source scripts/aptly_api.sh - # Upload file to temporary directory. - uploaded_files=false - shopt -s nullglob - for filename in $(cat debs/built_packages.txt | sed -E 's/(..*)/debs\/\1_\*.deb debs\/\1-static_\*.deb/g'); do - if ! aptly_upload_file "$filename"; then - exit 1 - fi - uploaded_files=true + for repo in main x11 root; do + export REPOSITORY_NAME=termux-$repo + + for archive in debs-*/debs-$repo-{aarch64,arm,i686,x86_64}-${{ github.sha }}.tar; do + tar xf "$archive" + done + + # Upload file to temporary directory. + uploaded_files=false + shopt -s nullglob + for filename in $(cat debs/built_packages.txt | sed -E 's/(..*)/debs\/\1_\*.deb debs\/\1-static_\*.deb/g'); do + if ! aptly_upload_file "$filename"; then + exit 1 + fi + + uploaded_files=true + done + shopt -u nullglob + + # Publishing repository changes. + if [ "$uploaded_files" = "true" ]; then + if ! aptly_add_to_repo; then + exit 1 + fi + + # Usually temporary directory is deleted automatically, but in certain cases it is left. + aptly_delete_dir + + # grimler.se mirror is signed manually, can't publish + # through CI + # if ! aptly_publish_repo; then + # exit 1 + # fi + fi + # Delete the debs folder in order to not upload packages from one repo to other + rm -rf debs/ done - shopt -u nullglob - - # Publishing repository changes. - if [ "$uploaded_files" = "true" ]; then - if ! aptly_add_to_repo; then - exit 1 - fi - - # Usually temporary directory is deleted automatically, but in certain cases it is left. - aptly_delete_dir - - # grimler.se mirror is signed manually, can't publish - # through CI - # if ! aptly_publish_repo; then - # exit 1 - # fi - fi diff --git a/.gitignore b/.gitignore index 8b4c93be3..fcea288e1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,6 @@ scripts/.vagrant/ /build-tools/ # Predownloaded packages sources. -/packages/*/cache +/main/*/cache +/root/*/cache +/x11/*/cache diff --git a/build-package.sh b/build-package.sh index faf84cdb2..815beb305 100755 --- a/build-package.sh +++ b/build-package.sh @@ -458,10 +458,16 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do TERMUX_PKG_BUILDER_DIR=$(realpath "${PACKAGE_LIST[i]}") else # Package name: - if [ -n "${TERMUX_IS_DISABLED=""}" ]; then - export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/disabled-packages/$TERMUX_PKG_NAME + if [ -d "${TERMUX_SCRIPTDIR}/main/${TERMUX_PKG_NAME}" ]; then + export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/main/$TERMUX_PKG_NAME + elif [ -d "${TERMUX_SCRIPTDIR}/x11/${TERMUX_PKG_NAME}" ]; then + export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/x11/$TERMUX_PKG_NAME + elif [ -d "${TERMUX_SCRIPTDIR}/root/${TERMUX_PKG_NAME}" ]; then + export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/root/$TERMUX_PKG_NAME + elif [ -n "${TERMUX_IS_DISABLED=""}" ] && [ -d "${TERMUX_SCRIPTDIR}/disabled/${TERMUX_PKG_NAME}"]; then + export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/disabled/$TERMUX_PKG_NAME else - export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/packages/$TERMUX_PKG_NAME + termux_error_exit "No package $TERMUX_PKG_NAME found in any of the enabled repositories, if you are trying to set up your custom repository give a look at build-package.sh:460-470" fi fi TERMUX_PKG_BUILDER_SCRIPT=$TERMUX_PKG_BUILDER_DIR/build.sh diff --git a/scripts/build/termux_step_setup_variables.sh b/scripts/build/termux_step_setup_variables.sh index 91333d12e..fc53c9228 100644 --- a/scripts/build/termux_step_setup_variables.sh +++ b/scripts/build/termux_step_setup_variables.sh @@ -7,7 +7,7 @@ termux_step_setup_variables() { : "${TERMUX_INSTALL_DEPS:="false"}" : "${TERMUX_MAKE_PROCESSES:="$(nproc)"}" : "${TERMUX_NO_CLEAN:="false"}" - : "${TERMUX_PACKAGES_DIRECTORIES:="packages"}" + : "${TERMUX_PACKAGES_DIRECTORIES:="main x11 root"}" : "${TERMUX_PKG_API_LEVEL:="24"}" : "${TERMUX_CONTINUE_BUILD:="false"}" : "${TERMUX_QUIET_BUILD:="false"}" diff --git a/scripts/build/termux_step_start_build.sh b/scripts/build/termux_step_start_build.sh index 80e7c028a..2f57fc7e6 100644 --- a/scripts/build/termux_step_start_build.sh +++ b/scripts/build/termux_step_start_build.sh @@ -86,7 +86,7 @@ termux_step_start_build() { local TERMUX_ELF_CLEANER_SRC=$TERMUX_COMMON_CACHEDIR/termux-elf-cleaner.cpp local TERMUX_ELF_CLEANER_VERSION - TERMUX_ELF_CLEANER_VERSION=$(bash -c ". $TERMUX_SCRIPTDIR/packages/termux-elf-cleaner/build.sh; echo \$TERMUX_PKG_VERSION") + TERMUX_ELF_CLEANER_VERSION=$(bash -c ". $TERMUX_SCRIPTDIR/main/termux-elf-cleaner/build.sh; echo \$TERMUX_PKG_VERSION") termux_download \ "https://raw.githubusercontent.com/termux/termux-elf-cleaner/v$TERMUX_ELF_CLEANER_VERSION/termux-elf-cleaner.cpp" \ "$TERMUX_ELF_CLEANER_SRC" \ diff --git a/scripts/buildorder.py b/scripts/buildorder.py index 83262f2c7..82ca09e87 100755 --- a/scripts/buildorder.py +++ b/scripts/buildorder.py @@ -246,13 +246,11 @@ def main(): parser.add_argument('package', nargs='?', help='Package to generate dependency list for.') parser.add_argument('package_dirs', nargs='*', - help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.') + help='Directories with packages. Can for example point to "../community-packages/packages". Note that the packages suffix is no longer added automatically if not present.') args = parser.parse_args() fast_build_mode = args.i package = args.package packages_directories = args.package_dirs - if 'packages' not in packages_directories: - packages_directories.append('packages') if not package: full_buildorder = True @@ -263,7 +261,6 @@ def main(): die('-i mode does not work when building all packages') if not full_buildorder: - packages_real_path = os.path.realpath('packages') for path in packages_directories: if not os.path.isdir(path): die('Not a directory: ' + path) diff --git a/scripts/setup-offline-bundle.sh b/scripts/setup-offline-bundle.sh index c0946eb97..2c8d4ce46 100755 --- a/scripts/setup-offline-bundle.sh +++ b/scripts/setup-offline-bundle.sh @@ -59,7 +59,7 @@ rm -rf "${TERMUX_PKG_TMPDIR}" ) # Package sources. -for p in "$TERMUX_SCRIPTDIR"/packages/*; do +for p in "$TERMUX_SCRIPTDIR"/{main,x11,root}/*; do ( . "$TERMUX_SCRIPTDIR"/scripts/properties.sh . "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_step_get_source.sh