chore: switch to monorepo

.github/workflows/packages.yml

@@ -5,10 +5,14 @@ on:
     branches:
     - master
     paths:
-    - 'packages/**'
+    - 'main/**'
+    - 'x11/**'
+    - 'root/**'
   pull_request:
     paths:
-    - 'packages/**'
+    - 'main/**'
+    - 'x11/**'
+    - 'root/**'
   workflow_dispatch:
     inputs:
       packages:
@@ -27,7 +31,7 @@ jobs:
       fail-fast: false
     steps:
     - name: Clone repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 1000
     - name: Gather build summary
@@ -73,58 +77,69 @@ jobs:
             docker build -t termux/package-builder:latest .
             cd ..
           fi
+
+          for repo in main x11 root; do
             # Parse changed files and identify new packages and deleted packages.
             # Create lists of those packages that will be passed to upload job for
             # further processing.
             while read -r file; do
-            if ! [[ $file == packages/* ]]; then
+              if ! [[ $file == $repo/* ]]; then
                 # This file does not belong to a package, so ignore it
                 continue
               fi
-            if [[ $file =~ ^packages/([.a-z0-9+-]*)/([.a-z0-9+-]*).subpackage.sh$ ]]; then
+              if [[ $file =~ ^$repo/([.a-z0-9+-]*)/([.a-z0-9+-]*).subpackage.sh$ ]]; then
                 # A subpackage was modified, check if it was deleted or just updated
                 pkg=${BASH_REMATCH[1]}
                 subpkg=${BASH_REMATCH[2]}
-              if [ ! -f "packages/${pkg}/${subpkg}.subpackage.sh" ]; then
+                if [ ! -f "${repo}/${pkg}/${subpkg}.subpackage.sh" ]; then
-                echo "$subpkg" >> ./deleted_packages.txt
+                  echo "$subpkg" >> ./deleted_$repo_packages.txt
                 fi
-            elif [[ $file =~ ^packages/([.a-z0-9+-]*)/.*$ ]]; then
+              elif [[ $file =~ ^$repo/([.a-z0-9+-]*)/.*$ ]]; then
                 # package, check if it was deleted or updated
                 pkg=${BASH_REMATCH[1]}
-              if [ ! -d "packages/${pkg}" ]; then
+                if [ ! -d "${repo}/${pkg}" ]; then
-                echo "$pkg" >> ./deleted_packages.txt
+                  echo "$pkg" >> ./deleted_$repo_packages.txt
                 else
-                echo "$pkg" >> ./built_packages.txt
+                  echo "$pkg" >> ./built_$repo_packages.txt
                   # If there are subpackages we want to create a list of those
                   # as well
-                for file in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do
+                  for file in $(find "${repo}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do
-                  echo "$(basename "${file%%.subpackage.sh}")" >> ./built_subpackages.txt
+                    echo "$(basename "${file%%.subpackage.sh}")" >> ./built_$repo_subpackages.txt
                   done
                 fi
               fi
             done<<<${CHANGED_FILES}
+          done
         else
           for pkg in ${{ github.event.inputs.packages }}; do
-            echo "$pkg" >> ./built_packages.txt
+            for repo in main x11 root; do
-            for subpkg in $(find "packages/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do
+              if [ -d "${repo}/${pkg}" ]; then
-              echo "$(basename "${subpkg%%.subpackage.sh}")" >> ./built_subpackages.txt
+                echo "$pkg" >> ./built_$repo_packages.txt
+                for subpkg in $(find "${repo}/${pkg}/" -maxdepth 1 -type f -name \*.subpackage.sh | sort); do
+                  echo "$(basename "${subpkg%%.subpackage.sh}")" >> ./built_$repo_subpackages.txt
+                done
+              else
+                echo "Package '${pkg}' not found in any of the repo"
+              fi
             done
           done
         fi
 
+        for repo in main x11 root; do
           # Fix so that lists do not contain duplicates
-        if [ -f ./built_packages.txt ]; then
+          if [ -f ./built_$repo_packages.txt ]; then
-          uniq ./built_packages.txt > ./built_packages.txt.tmp
+            uniq ./built_$repo_packages.txt > ./built_$repo_packages.txt.tmp
-          mv ./built_packages.txt.tmp ./built_packages.txt
+            mv ./built_$repo_packages.txt.tmp ./built_$repo_packages.txt
           fi
-        if [ -f ./built_subpackages.txt ]; then
+          if [ -f ./built_$repo_subpackages.txt ]; then
-          uniq ./built_subpackages.txt > ./built_subpackages.txt.tmp
+            uniq ./built_$repo_subpackages.txt > ./built_$repo_subpackages.txt.tmp
-          mv ./built_subpackages.txt.tmp ./built_subpackages.txt
+            mv ./built_$repo_subpackages.txt.tmp ./built_$repo_subpackages.txt
           fi
-        if [ -f ./deleted_packages.txt ]; then
+          if [ -f ./deleted_$repo_packages.txt ]; then
-          uniq ./deleted_packages.txt > ./deleted_packages.txt.tmp
+            uniq ./deleted_$repo_packages.txt > ./deleted_$repo_packages.txt.tmp
-          mv ./deleted_packages.txt.tmp ./deleted_packages.txt
+            mv ./deleted_$repo_packages.txt.tmp ./deleted_$repo_packages.txt
           fi
+        done
 
     - name: Free additional disk space (if necessary)
       run: |
@@ -136,25 +151,41 @@ jobs:
           sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift
         fi
 
-    - name: Build
+    - name: Lint packages
       run: |
-        if [ -f ./built_packages.txt ]; then
+        local package_recipes=
-          ./scripts/lint-packages.sh $(cat ./built_packages.txt | awk '{print "packages/"$1"/build.sh"}')
+        for repo in main x11 root; do
-          ./scripts/run-docker.sh ./build-package.sh -I -a ${{ matrix.target_arch }} $(cat ./built_packages.txt)
+          if [ -f ./built_$repo_packages.txt ]; then
+            package_recipes+=$(cat ./built_$repo_packages.txt | repo=$repo awk '{print ENVIRON["repo"]"/"$1"/build.sh"}')
           fi
+        done
+
+        ./scripts/lint-packages.sh $package_recipes
+
+    - name: Build packages
+      run: |
+        local packages=
+        for repo in main x11 root; do
+          if [ -f ./built_$repo_packages.txt ]; then
+            packages+=$(cat ./build_$repo_packages.txt)
+          fi
+        done
+
+        ./scripts/run-docker.sh ./build-package.sh -I -a ${{ matrix.target_arch }} $packages
 
     - name: Generate build artifacts
       if: always()
       run: |
-        mkdir -p debs
         test -d termux-packages/output && mv termux-packages/output/* ./output/
 
+        for repo in main x11 root; do
+          mkdir debs
           # Put package lists into directory with *.deb files so they will be transferred to
           # upload job.
-        test -f ./built_packages.txt && mv ./built_packages.txt ./debs/
+          test -f ./built_$repo_packages.txt && mv ./built_repo_packages.txt ./debs/built_packages.txt
-        test -f ./built_subpackages.txt && cat ./built_subpackages.txt >> ./debs/built_packages.txt \
+          test -f ./built_$repo_subpackages.txt && cat ./built_repo_subpackages.txt >> ./debs/built_packages.txt \
-          && rm ./built_subpackages.txt
+            && rm ./built_$repo_subpackages.txt
-        test -f ./deleted_packages.txt && mv ./deleted_packages.txt ./debs/
+          test -f ./deleted_$repo_packages.txt && mv ./deleted_$repo_packages.txt ./debs/deleted_packages.txt
 
           # Move only debs from built_packages into debs/ folder before
           # creating an archive.
@@ -165,16 +196,18 @@ jobs:
 
           # Files containing certain symbols (e.g. ":") will cause failure in actions/upload-artifact.
           # Archiving *.deb files in a tarball to avoid issues with uploading.
-        tar cf artifacts/debs-${{ matrix.target_arch }}-${{ github.sha }}.tar debs
+          tar cf artifacts/debs-${repo}-${{ matrix.target_arch }}-${{ github.sha }}.tar debs
+          rm -r debs/
+        done
     - name: Checksums for built *.deb files
       if: always()
       run: |
         find debs -type f -name "*.deb" -exec sha256sum "{}" \; | sort -k2
     - name: Store *.deb files
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
-        name: termux-packages-${{ matrix.target_arch }}-${{ github.sha }}
+        name: debs-${{ matrix.target_arch }}-${{ github.sha }}
         path: ./artifacts
 
   upload:
@@ -183,9 +216,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Clone repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Get *.deb files
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3
       with:
         path: ./
     - name: Upload to packages.termux.org
@@ -200,7 +233,10 @@ jobs:
 
         source scripts/aptly_api.sh
 
-        for archive in termux-packages-*/*.tar; do
+        for repo in main x11 root; do
+          export REPOSITORY_NAME=termux-$repo
+
+          for archive in debs-*/debs-$repo-{aarch64,arm,i686,x86_64}-${{ github.sha }}.tar; do
             tar xf "$archive"
           done
 
@@ -230,11 +266,15 @@ jobs:
               exit 1
             fi
           fi
+
+          # Delete the debs folder in order to not upload packages from one repo to other
+          # TODO(@thunder-coding): Perhaps it would make sense to make use of tar's `-C` option in order to prevent extracting the archives twice
+          rm -rf debs/
+        done
     - name: Upload to grimler.se
       # Run even if upload to packages.termux.org failed:
       if: always()
       env:
-        REPOSITORY_NAME: termux-main
         REPOSITORY_DISTRIBUTION: stable
         REPOSITORY_URL: https://aptly-api.grimler.se
       run: |
@@ -243,6 +283,14 @@ jobs:
 
         source scripts/aptly_api.sh
 
+
+        for repo in main x11 root; do
+          export REPOSITORY_NAME=termux-$repo
+
+          for archive in debs-*/debs-$repo-{aarch64,arm,i686,x86_64}-${{ github.sha }}.tar; do
+            tar xf "$archive"
+          done
+
           # Upload file to temporary directory.
           uploaded_files=false
           shopt -s nullglob
@@ -270,3 +318,6 @@ jobs:
             #   exit 1
             # fi
           fi
+          # Delete the debs folder in order to not upload packages from one repo to other
+          rm -rf debs/
+        done

.gitignore

@@ -18,4 +18,6 @@ scripts/.vagrant/
 /build-tools/
 
 # Predownloaded packages sources.
-/packages/*/cache
+/main/*/cache
+/root/*/cache
+/x11/*/cache

build-package.sh

@@ -458,10 +458,16 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do
 			TERMUX_PKG_BUILDER_DIR=$(realpath "${PACKAGE_LIST[i]}")
 		else
 			# Package name:
-			if [ -n "${TERMUX_IS_DISABLED=""}" ]; then
+			if [ -d "${TERMUX_SCRIPTDIR}/main/${TERMUX_PKG_NAME}" ]; then
-				export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/disabled-packages/$TERMUX_PKG_NAME
+				export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/main/$TERMUX_PKG_NAME
+			elif [ -d "${TERMUX_SCRIPTDIR}/x11/${TERMUX_PKG_NAME}" ]; then
+				export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/x11/$TERMUX_PKG_NAME
+			elif [ -d "${TERMUX_SCRIPTDIR}/root/${TERMUX_PKG_NAME}" ]; then
+				export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/root/$TERMUX_PKG_NAME
+			elif [ -n "${TERMUX_IS_DISABLED=""}" ] && [ -d "${TERMUX_SCRIPTDIR}/disabled/${TERMUX_PKG_NAME}"]; then
+				export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/disabled/$TERMUX_PKG_NAME
 			else
-				export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/packages/$TERMUX_PKG_NAME
+				termux_error_exit "No package $TERMUX_PKG_NAME found in any of the enabled repositories, if you are trying to set up your custom repository give a look at build-package.sh:460-470"
 			fi
 		fi
 		TERMUX_PKG_BUILDER_SCRIPT=$TERMUX_PKG_BUILDER_DIR/build.sh

scripts/build/termux_step_setup_variables.sh

@@ -7,7 +7,7 @@ termux_step_setup_variables() {
 	: "${TERMUX_INSTALL_DEPS:="false"}"
 	: "${TERMUX_MAKE_PROCESSES:="$(nproc)"}"
 	: "${TERMUX_NO_CLEAN:="false"}"
-	: "${TERMUX_PACKAGES_DIRECTORIES:="packages"}"
+	: "${TERMUX_PACKAGES_DIRECTORIES:="main x11 root"}"
 	: "${TERMUX_PKG_API_LEVEL:="24"}"
 	: "${TERMUX_CONTINUE_BUILD:="false"}"
 	: "${TERMUX_QUIET_BUILD:="false"}"

scripts/build/termux_step_start_build.sh

@@ -86,7 +86,7 @@ termux_step_start_build() {
 
 	local TERMUX_ELF_CLEANER_SRC=$TERMUX_COMMON_CACHEDIR/termux-elf-cleaner.cpp
 	local TERMUX_ELF_CLEANER_VERSION
-	TERMUX_ELF_CLEANER_VERSION=$(bash -c ". $TERMUX_SCRIPTDIR/packages/termux-elf-cleaner/build.sh; echo \$TERMUX_PKG_VERSION")
+	TERMUX_ELF_CLEANER_VERSION=$(bash -c ". $TERMUX_SCRIPTDIR/main/termux-elf-cleaner/build.sh; echo \$TERMUX_PKG_VERSION")
 	termux_download \
 		"https://raw.githubusercontent.com/termux/termux-elf-cleaner/v$TERMUX_ELF_CLEANER_VERSION/termux-elf-cleaner.cpp" \
 		"$TERMUX_ELF_CLEANER_SRC" \

scripts/buildorder.py

@@ -246,13 +246,11 @@ def main():
     parser.add_argument('package', nargs='?',
                         help='Package to generate dependency list for.')
     parser.add_argument('package_dirs', nargs='*',
-                        help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.')
+                        help='Directories with packages. Can for example point to "../community-packages/packages". Note that the packages suffix is no longer added automatically if not present.')
     args = parser.parse_args()
     fast_build_mode = args.i
     package = args.package
     packages_directories = args.package_dirs
-    if 'packages' not in packages_directories:
-        packages_directories.append('packages')
 
     if not package:
         full_buildorder = True
@@ -263,7 +261,6 @@ def main():
         die('-i mode does not work when building all packages')
 
     if not full_buildorder:
-        packages_real_path = os.path.realpath('packages')
         for path in packages_directories:
             if not os.path.isdir(path):
                 die('Not a directory: ' + path)

scripts/setup-offline-bundle.sh

@@ -59,7 +59,7 @@ rm -rf "${TERMUX_PKG_TMPDIR}"
 )
 
 # Package sources.
-for p in "$TERMUX_SCRIPTDIR"/packages/*; do
+for p in "$TERMUX_SCRIPTDIR"/{main,x11,root}/*; do
 	(
 		. "$TERMUX_SCRIPTDIR"/scripts/properties.sh
 		. "$TERMUX_SCRIPTDIR"/scripts/build/get_source/termux_step_get_source.sh