build-package: download dependencies recursively when fast-building
Use scripts/buildorder.py with a new -i flag to get all dependencies (including subpackages). The script now also spits out both package name and package dir, to make it easier to build packages from another repo.
This commit is contained in:
Henrik Grimler
Leonid Pliushch
parent
379f2452f3
commit
10fdf172d5
19
build-all.sh
19
build-all.sh
@ -55,28 +55,27 @@ fi
|
||||
|
||||
exec > >(tee -a $BUILDALL_DIR/ALL.out)
|
||||
exec 2> >(tee -a $BUILDALL_DIR/ALL.err >&2)
|
||||
trap "echo ERROR: See $BUILDALL_DIR/\${package}.err" ERR
|
||||
trap "echo ERROR: See $BUILDALL_DIR/\${PKG}.err" ERR
|
||||
|
||||
for package_path in $(cat $BUILDORDER_FILE); do
|
||||
package=$(basename $package_path)
|
||||
while read PKG PKG_DIR; do
|
||||
# Check build status (grepping is a bit crude, but it works)
|
||||
if [ -e $BUILDSTATUS_FILE ] && grep "^$package\$" $BUILDSTATUS_FILE >/dev/null; then
|
||||
echo "Skipping $package"
|
||||
if [ -e $BUILDSTATUS_FILE ] && grep "^$PKG\$" $BUILDSTATUS_FILE >/dev/null; then
|
||||
echo "Skipping $PKG"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo -n "Building $package... "
|
||||
echo -n "Building $PKG... "
|
||||
BUILD_START=$(date "+%s")
|
||||
bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \
|
||||
${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \
|
||||
> $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err
|
||||
${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $PKG_DIR \
|
||||
> $BUILDALL_DIR/${PKG}.out 2> $BUILDALL_DIR/${PKG}.err
|
||||
BUILD_END=$(date "+%s")
|
||||
BUILD_SECONDS=$(( $BUILD_END - $BUILD_START ))
|
||||
echo "done in $BUILD_SECONDS"
|
||||
|
||||
# Update build status
|
||||
echo "$package" >> $BUILDSTATUS_FILE
|
||||
done
|
||||
echo "$PKG" >> $BUILDSTATUS_FILE
|
||||
done<${BUILDORDER_FILE}
|
||||
|
||||
# Update build status
|
||||
rm -f $BUILDSTATUS_FILE
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
termux_extract_dep_info() {
|
||||
package=$1
|
||||
if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then
|
||||
PKG=$1
|
||||
PKG_DIR=$2
|
||||
if [ "$PKG" != "$(basename ${PKG_DIR})" ]; then
|
||||
# We are dealing with a subpackage
|
||||
TERMUX_ARCH=$(
|
||||
# set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed
|
||||
# set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to parent package's value and override if needed
|
||||
TERMUX_PKG_PLATFORM_INDEPENDENT=""
|
||||
source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh
|
||||
source ${PKG_DIR}/build.sh
|
||||
TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT
|
||||
source $(find packages/ -name "$package.subpackage.sh")
|
||||
source ${PKG_DIR}/${PKG}.subpackage.sh
|
||||
if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then
|
||||
echo all
|
||||
else
|
||||
@ -15,10 +16,9 @@ termux_extract_dep_info() {
|
||||
fi
|
||||
)
|
||||
|
||||
package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh")))
|
||||
elif [ "${package/-dev/}-dev" == "${package}" ]; then
|
||||
elif [ "${PKG/-dev/}-dev" == "${PKG}" ]; then
|
||||
# dev package
|
||||
package=${package/-dev/}
|
||||
PKG=${PKG/-dev/}
|
||||
fi
|
||||
(
|
||||
# Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't
|
||||
@ -26,7 +26,7 @@ termux_extract_dep_info() {
|
||||
# deps that should have the default values
|
||||
TERMUX_PKG_PLATFORM_INDEPENDENT=""
|
||||
TERMUX_PKG_REVISION="0"
|
||||
source packages/$package/build.sh
|
||||
source ${PKG_DIR}/build.sh
|
||||
if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi
|
||||
if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then
|
||||
TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION"
|
||||
|
||||
@ -14,36 +14,21 @@ termux_step_start_build() {
|
||||
|
||||
if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then
|
||||
# Download dependencies
|
||||
local PKG DEP_ARCH DEP_VERSION DEB_FILE _PKG_DEPENDS _PKG_BUILD_DEPENDS _SUBPKG_DEPENDS=""
|
||||
# remove (>= 1.0) and similar version tags:
|
||||
_PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS// /} | sed "s/[(][^)]*[)]//g")
|
||||
_PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS// /}
|
||||
# Also download subpackages dependencies (except the parent package):
|
||||
for SUBPKG in packages/$TERMUX_PKG_NAME/*.subpackage.sh; do
|
||||
test -e $SUBPKG || continue
|
||||
_SUBPKG_DEPENDS+=" $(. $SUBPKG; echo $TERMUX_SUBPKG_DEPENDS | sed s%$TERMUX_PKG_NAME%%g)"
|
||||
done
|
||||
for PKG in $(echo ${_PKG_DEPENDS//,/ } ${_SUBPKG_DEPENDS//,/ } ${_PKG_BUILD_DEPENDS//,/ } | tr ' ' '\n' | sort -u); do
|
||||
# handle "or" in dependencies (use first one):
|
||||
if [ ! "$PKG" = "${PKG/|/}" ]; then PKG=$(echo "$PKG" | sed "s%|.*%%"); fi
|
||||
while read PKG PKG_DIR; do
|
||||
if [ -z $PKG ]; then
|
||||
continue
|
||||
fi
|
||||
# llvm doesn't build if ndk-sysroot is installed:
|
||||
if [ "$PKG" = "ndk-sysroot" ]; then continue; fi
|
||||
read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info "$PKG")
|
||||
read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info $PKG "${PKG_DIR}")
|
||||
|
||||
if [ ! "$TERMUX_QUIET_BUILD" = true ]; then
|
||||
echo "Downloading dependency $PKG@$DEP_VERSION if necessary..."
|
||||
fi
|
||||
if ! termux_download_deb $PKG $DEP_ARCH $DEP_VERSION; then
|
||||
if find packages/ -type f -name ${PKG}.subpackage.sh -exec false {} +; then
|
||||
echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
|
||||
./build-package.sh -a $TERMUX_ARCH -I "$PKG"
|
||||
continue
|
||||
else
|
||||
# subpackage, so we need to build parent package
|
||||
PARENT=$(dirname $(find packages/ -name "${PKG}.subpackage.sh"))
|
||||
echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building parent $PARENT instead"
|
||||
./build-package.sh -a $TERMUX_ARCH -I $PARENT
|
||||
fi
|
||||
echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
|
||||
./build-package.sh -a $TERMUX_ARCH -I "${PKG_DIR}"
|
||||
continue
|
||||
else
|
||||
if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "extracting $PKG..."; fi
|
||||
(
|
||||
@ -64,14 +49,17 @@ termux_step_start_build() {
|
||||
fi
|
||||
mkdir -p /data/data/.built-packages
|
||||
echo "$DEP_VERSION" > "/data/data/.built-packages/$PKG"
|
||||
done
|
||||
done<<<$(./scripts/buildorder.py -i "$TERMUX_PKG_BUILDER_DIR")
|
||||
elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then
|
||||
# Build dependencies
|
||||
for PKG in $(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR"); do
|
||||
while read PKG PKG_DIR; do
|
||||
if [ -z $PKG ]; then
|
||||
continue
|
||||
fi
|
||||
echo "Building dependency $PKG if necessary..."
|
||||
# Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one
|
||||
./build-package.sh -a $TERMUX_ARCH -s "$PKG"
|
||||
done
|
||||
./build-package.sh -a $TERMUX_ARCH -s "${PKG_DIR}"
|
||||
done<<<$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR")
|
||||
fi
|
||||
|
||||
TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION
|
||||
|
||||
@ -107,13 +107,26 @@ class TermuxSubPackage:
|
||||
self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
|
||||
self.parent = parent
|
||||
self.deps = parse_build_file_dependencies(subpackage_file_path)
|
||||
self.dir = parent.dir
|
||||
|
||||
self.needed_by = set() # Populated outside constructor, reverse of deps.
|
||||
|
||||
def __repr__(self):
|
||||
return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
|
||||
|
||||
def read_packages_from_directories(directories):
|
||||
def recursive_dependencies(self, pkgs_map):
|
||||
"""All the dependencies of the subpackage, both direct and indirect.
|
||||
Only relevant when building in fast-build mode"""
|
||||
result = []
|
||||
for dependency_name in sorted(self.deps):
|
||||
dependency_package = pkgs_map[dependency_name]
|
||||
result += dependency_package.recursive_dependencies(pkgs_map)
|
||||
result += [dependency_package]
|
||||
return unique_everseen(result)
|
||||
|
||||
def read_packages_from_directories(directories, fast_build_mode):
|
||||
"""Construct a map from package name to TermuxPackage.
|
||||
For subpackages this maps from the subpackage name to the parent package."""
|
||||
Subpackages are mapped to the parent package if fast_build_mode is false."""
|
||||
pkgs_map = {}
|
||||
all_packages = []
|
||||
|
||||
@ -132,6 +145,8 @@ def read_packages_from_directories(directories):
|
||||
for subpkg in new_package.subpkgs:
|
||||
if subpkg.name in pkgs_map:
|
||||
die('Duplicated package: ' + subpkg.name)
|
||||
elif fast_build_mode:
|
||||
pkgs_map[subpkg.name] = subpkg
|
||||
else:
|
||||
pkgs_map[subpkg.name] = new_package
|
||||
all_packages.append(subpkg)
|
||||
@ -141,7 +156,7 @@ def read_packages_from_directories(directories):
|
||||
if dependency_name not in pkgs_map:
|
||||
die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
|
||||
dep_pkg = pkgs_map[dependency_name]
|
||||
if not isinstance(pkg, TermuxSubPackage):
|
||||
if fast_build_mode or not isinstance(pkg, TermuxSubPackage):
|
||||
dep_pkg.needed_by.add(pkg)
|
||||
return pkgs_map
|
||||
|
||||
@ -198,7 +213,7 @@ def generate_full_buildorder(pkgs_map):
|
||||
|
||||
return build_order
|
||||
|
||||
def generate_target_buildorder(target_path, pkgs_map):
|
||||
def generate_target_buildorder(target_path, pkgs_map, fast_build_mode):
|
||||
"Generate a build order for building the dependencies of the specified package."
|
||||
if target_path.endswith('/'):
|
||||
target_path = target_path[:-1]
|
||||
@ -209,28 +224,50 @@ def generate_target_buildorder(target_path, pkgs_map):
|
||||
|
||||
def main():
|
||||
"Generate the build order either for all packages or a specific one."
|
||||
packages_directories = ['packages']
|
||||
full_buildorder = len(sys.argv) == 1
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Generate order in which to build dependencies for a package. Generates')
|
||||
parser.add_argument('-i', default=False, action='store_true',
|
||||
help='Generate dependency list for fast-build mode. This includes subpackages in output since these can be downloaded.')
|
||||
parser.add_argument('package', nargs='?',
|
||||
help='Package to generate dependency list for.')
|
||||
parser.add_argument('package_dirs', nargs='*',
|
||||
help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.')
|
||||
args = parser.parse_args()
|
||||
fast_build_mode = args.i
|
||||
package = args.package
|
||||
packages_directories = args.package_dirs + ['packages']
|
||||
|
||||
if not package:
|
||||
full_buildorder = True
|
||||
else:
|
||||
full_buildorder = False
|
||||
|
||||
if fast_build_mode and full_buildorder:
|
||||
die('-i mode does not work when building all packages')
|
||||
|
||||
if not full_buildorder:
|
||||
packages_real_path = os.path.realpath('packages')
|
||||
for path in sys.argv[1:]:
|
||||
for path in packages_directories:
|
||||
if not os.path.isdir(path):
|
||||
die('Not a directory: ' + path)
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
parent_path = os.path.dirname(path)
|
||||
if packages_real_path != os.path.realpath(parent_path):
|
||||
packages_directories.append(parent_path)
|
||||
|
||||
pkgs_map = read_packages_from_directories(packages_directories)
|
||||
if package:
|
||||
if package[-1] == "/":
|
||||
package = package[:-1]
|
||||
if not os.path.isdir(package):
|
||||
die('Not a directory: ' + package)
|
||||
if not os.path.relpath(os.path.dirname(package), '.') in packages_directories:
|
||||
packages_directories.insert(0, os.path.dirname(package))
|
||||
pkgs_map = read_packages_from_directories(packages_directories, fast_build_mode)
|
||||
|
||||
if full_buildorder:
|
||||
build_order = generate_full_buildorder(pkgs_map)
|
||||
else:
|
||||
build_order = generate_target_buildorder(sys.argv[1], pkgs_map)
|
||||
build_order = generate_target_buildorder(package, pkgs_map, fast_build_mode)
|
||||
|
||||
for pkg in build_order:
|
||||
print(pkg.dir)
|
||||
print("%-30s %s" % (pkg.name, pkg.dir))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user