termux-packages/scripts/buildorder.py

298 lines
11 KiB
Python
Raw Normal View History

2015-06-13 01:03:31 +02:00
#!/usr/bin/env python3
2017-11-19 14:29:49 +01:00
"Script to generate a build order respecting package dependencies."
2015-06-13 01:03:31 +02:00
2015-12-24 04:43:35 +01:00
import os
import re
2015-12-24 04:43:35 +01:00
import sys
2015-06-13 01:03:31 +02:00
from itertools import filterfalse
def unique_everseen(iterable, key=None):
2017-11-19 14:29:49 +01:00
"""List unique elements, preserving order. Remember all elements ever seen.
See https://docs.python.org/3/library/itertools.html#itertools-recipes
Examples:
unique_everseen('AAAABBBCCDAABBB') --> A B C D
unique_everseen('ABBCcAD', str.lower) --> A B C D"""
seen = set()
seen_add = seen.add
if key is None:
for element in filterfalse(seen.__contains__, iterable):
seen_add(element)
yield element
else:
for element in iterable:
k = key(element)
if k not in seen:
seen_add(k)
yield element
2015-12-24 04:43:35 +01:00
def die(msg):
2017-11-19 14:29:49 +01:00
"Exit the process with an error message."
2015-12-24 04:43:35 +01:00
sys.exit('ERROR: ' + msg)
2015-06-13 01:03:31 +02:00
2017-11-19 14:29:49 +01:00
def parse_build_file_dependencies(path):
"Extract the dependencies of a build.sh or *.subpackage.sh file."
dependencies = []
with open(path, encoding="utf-8") as build_script:
for line in build_script:
if line.startswith( ('TERMUX_PKG_DEPENDS', 'TERMUX_PKG_BUILD_DEPENDS', 'TERMUX_SUBPKG_DEPENDS', 'TERMUX_PKG_DEVPACKAGE_DEPENDS') ):
2019-01-13 18:45:40 +01:00
dependencies_string = line.split('DEPENDS=')[1]
for char in "\"'\n":
dependencies_string = dependencies_string.replace(char, '')
2017-11-19 14:29:49 +01:00
2019-01-13 18:45:40 +01:00
# Split also on '|' to dependencies with '|', as in 'nodejs | nodejs-current':
for dependency_value in re.split(',|\\|', dependencies_string):
# Replace parenthesis to ignore version qualifiers as in "gcc (>= 5.0)":
dependency_value = re.sub(r'\(.*?\)', '', dependency_value).strip()
2015-12-24 07:20:47 +01:00
2019-01-13 18:45:40 +01:00
dependencies.append(dependency_value)
2015-12-24 07:20:47 +01:00
2017-11-19 14:29:49 +01:00
return set(dependencies)
2015-12-24 07:20:47 +01:00
def develsplit(path):
with open(path, encoding="utf-8") as build_script:
for line in build_script:
if line.startswith('TERMUX_PKG_NO_DEVELSPLIT'):
return False
return True
class TermuxPackage(object):
2017-11-19 14:29:49 +01:00
"A main package definition represented by a directory with a build.sh file."
def __init__(self, dir_path, fast_build_mode):
self.dir = dir_path
self.name = os.path.basename(self.dir)
2015-12-24 07:20:47 +01:00
# search package build.sh
build_sh_path = os.path.join(self.dir, 'build.sh')
if not os.path.isfile(build_sh_path):
raise Exception("build.sh not found for package '" + self.name + "'")
2015-12-24 07:20:47 +01:00
2017-11-19 14:29:49 +01:00
self.deps = parse_build_file_dependencies(build_sh_path)
always_deps = ['libandroid-support', 'libc++']
for dependency_name in always_deps:
if dependency_name not in self.deps and self.name not in always_deps:
self.deps.add(dependency_name)
2015-12-24 07:20:47 +01:00
# search subpackages
self.subpkgs = []
for filename in os.listdir(self.dir):
2017-11-19 14:29:49 +01:00
if not filename.endswith('.subpackage.sh'):
continue
subpkg = TermuxSubPackage(self.dir + '/' + filename, self)
2015-12-24 07:20:47 +01:00
self.subpkgs.append(subpkg)
self.deps.add(subpkg.name)
2015-12-24 07:20:47 +01:00
self.deps |= subpkg.deps
if develsplit(build_sh_path):
subpkg = TermuxSubPackage(self.dir + '/' + self.name + '-dev' + '.subpackage.sh', self, virtual=True)
self.subpkgs.append(subpkg)
self.deps.add(subpkg.name)
2015-12-24 07:20:47 +01:00
# Do not depend on itself
self.deps.discard(self.name)
# Do not depend on any sub package
if not fast_build_mode:
self.deps.difference_update([subpkg.name for subpkg in self.subpkgs])
2015-12-24 07:20:47 +01:00
2017-11-19 14:29:49 +01:00
self.needed_by = set() # Populated outside constructor, reverse of deps.
2015-12-24 07:20:47 +01:00
def __repr__(self):
return "<{} '{}'>".format(self.__class__.__name__, self.name)
2017-11-19 14:29:49 +01:00
def recursive_dependencies(self, pkgs_map):
"All the dependencies of the package, both direct and indirect."
result = []
for dependency_name in sorted(self.deps):
dependency_package = pkgs_map[dependency_name]
result += dependency_package.recursive_dependencies(pkgs_map)
result += [dependency_package]
return unique_everseen(result)
2015-12-24 07:20:47 +01:00
2017-11-19 14:29:49 +01:00
class TermuxSubPackage:
"A sub-package represented by a ${PACKAGE_NAME}.subpackage.sh file."
def __init__(self, subpackage_file_path, parent, virtual=False):
2015-12-24 07:20:47 +01:00
if parent is None:
raise Exception("SubPackages should have a parent")
self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
2015-12-24 07:20:47 +01:00
self.parent = parent
self.deps = set([parent.name])
if not virtual:
self.deps |= parse_build_file_dependencies(subpackage_file_path)
self.dir = parent.dir
self.needed_by = set() # Populated outside constructor, reverse of deps.
2015-12-24 07:20:47 +01:00
def __repr__(self):
return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
2015-12-24 04:43:35 +01:00
def recursive_dependencies(self, pkgs_map):
"""All the dependencies of the subpackage, both direct and indirect.
Only relevant when building in fast-build mode"""
result = []
for dependency_name in sorted(self.deps):
if dependency_name == self.parent.name:
self.parent.deps.discard(self.name)
dependency_package = pkgs_map[dependency_name]
if dependency_package not in self.parent.subpkgs:
result += dependency_package.recursive_dependencies(pkgs_map)
result += [dependency_package]
return unique_everseen(result)
def read_packages_from_directories(directories, fast_build_mode):
2017-11-19 14:29:49 +01:00
"""Construct a map from package name to TermuxPackage.
Subpackages are mapped to the parent package if fast_build_mode is false."""
2017-11-19 14:29:49 +01:00
pkgs_map = {}
all_packages = []
2017-11-19 14:29:49 +01:00
for package_dir in directories:
for pkgdir_name in sorted(os.listdir(package_dir)):
dir_path = package_dir + '/' + pkgdir_name
if os.path.isfile(dir_path + '/build.sh'):
new_package = TermuxPackage(package_dir + '/' + pkgdir_name, fast_build_mode)
2015-12-24 04:47:38 +01:00
2017-11-19 14:29:49 +01:00
if new_package.name in pkgs_map:
die('Duplicated package: ' + new_package.name)
else:
pkgs_map[new_package.name] = new_package
all_packages.append(new_package)
2015-12-24 07:20:47 +01:00
2017-11-19 14:29:49 +01:00
for subpkg in new_package.subpkgs:
if subpkg.name in pkgs_map:
die('Duplicated package: ' + subpkg.name)
elif fast_build_mode:
pkgs_map[subpkg.name] = subpkg
2017-11-19 14:29:49 +01:00
else:
pkgs_map[subpkg.name] = new_package
all_packages.append(subpkg)
2015-12-24 04:47:38 +01:00
2017-11-19 14:29:49 +01:00
for pkg in all_packages:
for dependency_name in pkg.deps:
if dependency_name not in pkgs_map:
die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
dep_pkg = pkgs_map[dependency_name]
if fast_build_mode or not isinstance(pkg, TermuxSubPackage):
2017-11-19 14:29:49 +01:00
dep_pkg.needed_by.add(pkg)
return pkgs_map
2015-12-24 07:20:47 +01:00
2017-11-19 14:29:49 +01:00
def generate_full_buildorder(pkgs_map):
"Generate a build order for building all packages."
build_order = []
# List of all TermuxPackages without dependencies
2015-12-24 07:20:47 +01:00
leaf_pkgs = [pkg for name, pkg in pkgs_map.items() if not pkg.deps]
if not leaf_pkgs:
die('No package without dependencies - where to start?')
2015-12-24 04:47:38 +01:00
# Sort alphabetically:
pkg_queue = sorted(leaf_pkgs, key=lambda p: p.name)
2015-12-24 04:47:38 +01:00
# Topological sorting
2015-12-24 07:20:47 +01:00
visited = set()
2017-11-19 14:29:49 +01:00
# Tracks non-visited deps for each package
remaining_deps = {}
for name, pkg in pkgs_map.items():
remaining_deps[name] = set(pkg.deps)
for subpkg in pkg.subpkgs:
remaining_deps[subpkg.name] = set(subpkg.deps)
2015-12-24 07:20:47 +01:00
while pkg_queue:
pkg = pkg_queue.pop(0)
if pkg.name in visited:
continue
# print("Processing {}:".format(pkg.name), pkg.needed_by)
visited.add(pkg.name)
2015-12-24 04:47:38 +01:00
build_order.append(pkg)
2015-12-24 07:20:47 +01:00
for other_pkg in sorted(pkg.needed_by, key=lambda p: p.name):
# Remove this pkg from deps
2015-12-24 07:20:47 +01:00
remaining_deps[other_pkg.name].discard(pkg.name)
# ... and all its subpackages
remaining_deps[other_pkg.name].difference_update(
[subpkg.name for subpkg in pkg.subpkgs]
)
if not remaining_deps[other_pkg.name]: # all deps were already appended?
pkg_queue.append(other_pkg) # should be processed
2015-12-24 07:20:47 +01:00
if set(pkgs_map.values()) != set(build_order):
2015-12-24 04:47:38 +01:00
print("ERROR: Cycle exists. Remaining: ")
2015-12-24 07:20:47 +01:00
for name, pkg in pkgs_map.items():
2015-12-24 04:47:38 +01:00
if pkg not in build_order:
2015-12-24 07:20:47 +01:00
print(name, remaining_deps[name])
2015-12-24 04:47:38 +01:00
sys.exit(1)
return build_order
2015-12-24 04:47:38 +01:00
def generate_target_buildorder(target_path, pkgs_map, fast_build_mode):
2017-11-19 14:29:49 +01:00
"Generate a build order for building the dependencies of the specified package."
if target_path.endswith('/'):
target_path = target_path[:-1]
2017-11-19 14:29:49 +01:00
package_name = os.path.basename(target_path)
package = pkgs_map[package_name]
# Do not depend on any sub package
if fast_build_mode:
package.deps.difference_update([subpkg.name for subpkg in package.subpkgs])
2017-11-19 14:29:49 +01:00
return package.recursive_dependencies(pkgs_map)
2017-11-19 14:29:49 +01:00
def main():
"Generate the build order either for all packages or a specific one."
import argparse
parser = argparse.ArgumentParser(description='Generate order in which to build dependencies for a package. Generates')
parser.add_argument('-i', default=False, action='store_true',
help='Generate dependency list for fast-build mode. This includes subpackages in output since these can be downloaded.')
parser.add_argument('package', nargs='?',
help='Package to generate dependency list for.')
parser.add_argument('package_dirs', nargs='*',
help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.')
args = parser.parse_args()
fast_build_mode = args.i
package = args.package
packages_directories = args.package_dirs
if 'packages' not in packages_directories:
packages_directories.append('packages')
if not package:
full_buildorder = True
else:
full_buildorder = False
if fast_build_mode and full_buildorder:
die('-i mode does not work when building all packages')
if not full_buildorder:
packages_real_path = os.path.realpath('packages')
for path in packages_directories:
if not os.path.isdir(path):
die('Not a directory: ' + path)
if package:
if package[-1] == "/":
package = package[:-1]
if not os.path.isdir(package):
die('Not a directory: ' + package)
if not os.path.relpath(os.path.dirname(package), '.') in packages_directories:
packages_directories.insert(0, os.path.dirname(package))
pkgs_map = read_packages_from_directories(packages_directories, fast_build_mode)
if full_buildorder:
2017-11-19 14:29:49 +01:00
build_order = generate_full_buildorder(pkgs_map)
else:
build_order = generate_target_buildorder(package, pkgs_map, fast_build_mode)
2017-11-19 14:29:49 +01:00
for pkg in build_order:
print("%-30s %s" % (pkg.name, pkg.dir))
2017-11-19 14:29:49 +01:00
if __name__ == '__main__':
main()