Cleanup scripts/buildorder.py
This commit is contained in:
Fredrik Fornwall
parent
545a24028c
commit
b39b311462
@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
# buildorder.py - script to generate a build order respecting package dependencies
|
||||
"Script to generate a build order respecting package dependencies."
|
||||
|
||||
import os
|
||||
import re
|
||||
@ -8,12 +8,12 @@ import sys
|
||||
from itertools import filterfalse
|
||||
|
||||
|
||||
# https://docs.python.org/3/library/itertools.html#itertools-recipes
|
||||
|
||||
def unique_everseen(iterable, key=None):
|
||||
"List unique elements, preserving order. Remember all elements ever seen."
|
||||
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
|
||||
# unique_everseen('ABBCcAD', str.lower) --> A B C D
|
||||
"""List unique elements, preserving order. Remember all elements ever seen.
|
||||
See https://docs.python.org/3/library/itertools.html#itertools-recipes
|
||||
Examples:
|
||||
unique_everseen('AAAABBBCCDAABBB') --> A B C D
|
||||
unique_everseen('ABBCcAD', str.lower) --> A B C D"""
|
||||
seen = set()
|
||||
seen_add = seen.add
|
||||
if key is None:
|
||||
@ -29,26 +29,20 @@ def unique_everseen(iterable, key=None):
|
||||
|
||||
|
||||
def die(msg):
|
||||
"Exit the process with an error message."
|
||||
sys.exit('ERROR: ' + msg)
|
||||
|
||||
def rchop(thestring, ending):
|
||||
if thestring.endswith(ending):
|
||||
return thestring[:-len(ending)]
|
||||
return thestring
|
||||
|
||||
class TermuxBuildFile(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def _get_dependencies(self):
|
||||
def parse_build_file_dependencies(path):
|
||||
"Extract the dependencies of a build.sh or *.subpackage.sh file."
|
||||
pkg_dep_prefix = 'TERMUX_PKG_DEPENDS='
|
||||
pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS='
|
||||
subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS='
|
||||
comma_deps = ''
|
||||
dependencies = []
|
||||
|
||||
with open(self.path, encoding="utf-8") as f:
|
||||
with open(path, encoding="utf-8") as build_script:
|
||||
prefix = None
|
||||
for line in f:
|
||||
for line in build_script:
|
||||
if line.startswith(pkg_dep_prefix):
|
||||
prefix = pkg_dep_prefix
|
||||
elif line.startswith(pkg_build_dep_prefix):
|
||||
@ -58,21 +52,21 @@ class TermuxBuildFile(object):
|
||||
else:
|
||||
continue
|
||||
|
||||
comma_deps += line[len(prefix):].replace('"', '').replace("'", '').replace("\n", ",")
|
||||
dependencies_string = line[len(prefix):]
|
||||
for char in "\"'\n":
|
||||
dependencies_string = dependencies_string.replace(char, '')
|
||||
|
||||
# Remove trailing ',' that is otherwise replacing the final newline
|
||||
comma_deps = comma_deps[:-1]
|
||||
if not comma_deps:
|
||||
# no deps found
|
||||
return set()
|
||||
for dependency_value in dependencies_string.split(','):
|
||||
# Replace parenthesis to ignore version qualifiers as in "gcc (>= 5.0)":
|
||||
dependency_value = re.sub(r'\(.*?\)', '', dependency_value).strip()
|
||||
dependency_value = re.sub('-dev$', '', dependency_value)
|
||||
dependencies.append(dependency_value)
|
||||
|
||||
return set([
|
||||
# Replace parenthesis to handle version qualifiers, as in "gcc (>= 5.0)":
|
||||
rchop(re.sub(r'\(.*?\)', '', dep).strip(), '-dev') for dep in comma_deps.split(',')
|
||||
])
|
||||
return set(dependencies)
|
||||
|
||||
|
||||
class TermuxPackage(object):
|
||||
"A main package definition represented by a directory with a build.sh file."
|
||||
def __init__(self, dir_path):
|
||||
self.dir = dir_path
|
||||
self.name = os.path.basename(self.dir)
|
||||
@ -82,8 +76,7 @@ class TermuxPackage(object):
|
||||
if not os.path.isfile(build_sh_path):
|
||||
raise Exception("build.sh not found for package '" + self.name + "'")
|
||||
|
||||
self.buildfile = TermuxBuildFile(build_sh_path)
|
||||
self.deps = self.buildfile._get_dependencies()
|
||||
self.deps = parse_build_file_dependencies(build_sh_path)
|
||||
if 'libandroid-support' not in self.deps and self.name != 'libandroid-support':
|
||||
# Every package may depend on libandroid-support without declaring it:
|
||||
self.deps.add('libandroid-support')
|
||||
@ -92,7 +85,8 @@ class TermuxPackage(object):
|
||||
self.subpkgs = []
|
||||
|
||||
for filename in os.listdir(self.dir):
|
||||
if not filename.endswith('.subpackage.sh'): continue
|
||||
if not filename.endswith('.subpackage.sh'):
|
||||
continue
|
||||
subpkg = TermuxSubPackage(self.dir + '/' + filename, self)
|
||||
|
||||
self.subpkgs.append(subpkg)
|
||||
@ -103,71 +97,72 @@ class TermuxPackage(object):
|
||||
# Do not depend on any sub package
|
||||
self.deps.difference_update([subpkg.name for subpkg in self.subpkgs])
|
||||
|
||||
self.needed_by = set() # to be completed outside, reverse of deps
|
||||
self.needed_by = set() # Populated outside constructor, reverse of deps.
|
||||
|
||||
def __repr__(self):
|
||||
return "<{} '{}'>".format(self.__class__.__name__, self.name)
|
||||
|
||||
def recursive_dependencies(self, pkgs_map):
|
||||
"All the dependencies of the package, both direct and indirect."
|
||||
result = []
|
||||
for dependency_name in sorted(self.deps):
|
||||
dependency_package = pkgs_map[dependency_name]
|
||||
result += dependency_package.recursive_dependencies(pkgs_map)
|
||||
result += [dependency_package]
|
||||
return unique_everseen(result)
|
||||
|
||||
class TermuxSubPackage(TermuxPackage):
|
||||
|
||||
class TermuxSubPackage:
|
||||
"A sub-package represented by a ${PACKAGE_NAME}.subpackage.sh file."
|
||||
def __init__(self, subpackage_file_path, parent):
|
||||
if parent is None:
|
||||
raise Exception("SubPackages should have a parent")
|
||||
|
||||
self.buildfile = TermuxBuildFile(subpackage_file_path)
|
||||
self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
|
||||
self.parent = parent
|
||||
self.deps = self.buildfile._get_dependencies()
|
||||
self.deps = parse_build_file_dependencies(subpackage_file_path)
|
||||
|
||||
def __repr__(self):
|
||||
return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
|
||||
|
||||
|
||||
# Tracks non-visited deps for each package
|
||||
remaining_deps = {}
|
||||
|
||||
# Mapping from package name to TermuxPackage
|
||||
# (if subpackage, mapping from subpackage name to parent package)
|
||||
pkgs_map = {}
|
||||
|
||||
# Reverse dependencies
|
||||
pkg_depends_on = {}
|
||||
|
||||
PACKAGES_DIRS = ['packages']
|
||||
|
||||
|
||||
def populate():
|
||||
def read_packages_from_directories(directories):
|
||||
"""Construct a map from package name to TermuxPackage.
|
||||
For subpackages this maps from the subpackage name to the parent package."""
|
||||
pkgs_map = {}
|
||||
all_packages = []
|
||||
for package_dir in PACKAGES_DIRS:
|
||||
|
||||
for package_dir in directories:
|
||||
for pkgdir_name in sorted(os.listdir(package_dir)):
|
||||
dir_path = package_dir + '/' + pkgdir_name
|
||||
if os.path.isfile(dir_path + '/build.sh'):
|
||||
all_packages.append(TermuxPackage(package_dir + '/' + pkgdir_name))
|
||||
new_package = TermuxPackage(package_dir + '/' + pkgdir_name)
|
||||
|
||||
if new_package.name in pkgs_map:
|
||||
die('Duplicated package: ' + new_package.name)
|
||||
else:
|
||||
pkgs_map[new_package.name] = new_package
|
||||
all_packages.append(new_package)
|
||||
|
||||
for subpkg in new_package.subpkgs:
|
||||
if subpkg.name in pkgs_map:
|
||||
die('Duplicated package: ' + subpkg.name)
|
||||
else:
|
||||
pkgs_map[subpkg.name] = new_package
|
||||
all_packages.append(subpkg)
|
||||
|
||||
for pkg in all_packages:
|
||||
if pkg.name in pkgs_map: die('Duplicated package: ' + pkg.name)
|
||||
else: pkgs_map[pkg.name] = pkg
|
||||
|
||||
for subpkg in pkg.subpkgs:
|
||||
pkgs_map[subpkg.name] = pkg
|
||||
remaining_deps[subpkg.name] = set(subpkg.deps)
|
||||
|
||||
remaining_deps[pkg.name] = set(pkg.deps)
|
||||
|
||||
all_pkg_names = set(pkgs_map.keys())
|
||||
|
||||
for name, pkg in pkgs_map.items():
|
||||
for dep_name in remaining_deps[name]:
|
||||
if dep_name not in all_pkg_names:
|
||||
die('Package %s depends on non-existing package "%s"' % (
|
||||
name, dep_name
|
||||
))
|
||||
|
||||
dep_pkg = pkgs_map[dep_name]
|
||||
for dependency_name in pkg.deps:
|
||||
if dependency_name not in pkgs_map:
|
||||
die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
|
||||
dep_pkg = pkgs_map[dependency_name]
|
||||
if not isinstance(pkg, TermuxSubPackage):
|
||||
dep_pkg.needed_by.add(pkg)
|
||||
return pkgs_map
|
||||
|
||||
|
||||
def generate_full_buildorder():
|
||||
def generate_full_buildorder(pkgs_map):
|
||||
"Generate a build order for building all packages."
|
||||
build_order = []
|
||||
|
||||
# List of all TermuxPackages without dependencies
|
||||
@ -182,6 +177,13 @@ def generate_full_buildorder():
|
||||
# Topological sorting
|
||||
visited = set()
|
||||
|
||||
# Tracks non-visited deps for each package
|
||||
remaining_deps = {}
|
||||
for name, pkg in pkgs_map.items():
|
||||
remaining_deps[name] = set(pkg.deps)
|
||||
for subpkg in pkg.subpkgs:
|
||||
remaining_deps[subpkg.name] = set(subpkg.deps)
|
||||
|
||||
while pkg_queue:
|
||||
pkg = pkg_queue.pop(0)
|
||||
if pkg.name in visited:
|
||||
@ -212,50 +214,40 @@ def generate_full_buildorder():
|
||||
|
||||
return build_order
|
||||
|
||||
def deps(pkg):
|
||||
l = []
|
||||
for dep in sorted(pkg.deps):
|
||||
l += deps_then_me(pkgs_map[dep])
|
||||
return l
|
||||
|
||||
def deps_then_me(pkg):
|
||||
l = []
|
||||
for dep in sorted(pkg.deps):
|
||||
l += deps_then_me(pkgs_map[dep])
|
||||
l += [pkg]
|
||||
return l
|
||||
def generate_target_buildorder(target_path, pkgs_map):
|
||||
"Generate a build order for building the dependencies of the specified package."
|
||||
if target_path.endswith('/'):
|
||||
target_path = target_path[:-1]
|
||||
|
||||
package_name = os.path.basename(target_path)
|
||||
package = pkgs_map[package_name]
|
||||
return package.recursive_dependencies(pkgs_map)
|
||||
|
||||
def generate_targets_buildorder(target_paths):
|
||||
buildorder = []
|
||||
|
||||
for target_path in target_paths:
|
||||
if target_path.endswith('/'): target_path = target_path[:-1]
|
||||
pkgname = os.path.basename(target_path)
|
||||
if not pkgname in pkgs_map:
|
||||
die('Dependencies for ' + pkgname + ' could not be calculated (skip dependency check with -s)')
|
||||
buildorder += deps(pkgs_map[pkgname])
|
||||
|
||||
return unique_everseen(buildorder)
|
||||
|
||||
if __name__ == '__main__':
|
||||
def main():
|
||||
"Generate the build order either for all packages or a specific one."
|
||||
packages_directories = ['packages']
|
||||
full_buildorder = len(sys.argv) == 1
|
||||
if not full_buildorder:
|
||||
packages_real_path = os.path.realpath('packages')
|
||||
for path in sys.argv[1:]:
|
||||
if not os.path.isdir(path):
|
||||
die('Not a directory: ' + path)
|
||||
if path.endswith('/'): path = path[:-1]
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
parent_path = os.path.dirname(path)
|
||||
if packages_real_path != os.path.realpath(parent_path):
|
||||
PACKAGES_DIRS.append(parent_path)
|
||||
packages_directories.append(parent_path)
|
||||
|
||||
populate()
|
||||
pkgs_map = read_packages_from_directories(packages_directories)
|
||||
|
||||
if full_buildorder:
|
||||
bo = generate_full_buildorder()
|
||||
build_order = generate_full_buildorder(pkgs_map)
|
||||
else:
|
||||
bo = generate_targets_buildorder(sys.argv[1:])
|
||||
build_order = generate_target_buildorder(sys.argv[1], pkgs_map)
|
||||
|
||||
for pkg in bo:
|
||||
for pkg in build_order:
|
||||
print(pkg.dir)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user