Merge pull request #112 from franciscod/buildorder.py
Huge refactor of buildorder.py!
This commit is contained in:
Fredrik Fornwall
commit
ba1982c779
282
buildorder.py
282
buildorder.py
@ -1,85 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
# buildorder.py - script to generate a build order respecting package dependencies
|
||||
|
||||
import os, sys
|
||||
import os
|
||||
import sys
|
||||
|
||||
def die(msg): sys.exit('ERROR: ' + msg)
|
||||
from itertools import filterfalse
|
||||
|
||||
if len(sys.argv) != 1: die('buildorder.py takes no arguments')
|
||||
|
||||
class DebianPackage:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.remaining_dependencies = set() # String
|
||||
self.sub_packages = set() # String
|
||||
self.prerequisite_for = set() # Packages that needs this package
|
||||
# https://docs.python.org/3/library/itertools.html#itertools-recipes
|
||||
|
||||
all_packages = [] # List of all DebianPackage:s
|
||||
packages_map = {} # Mapping from package name to DebianPackage (if subpackage, mapping from subpackage name to parent package)
|
||||
def unique_everseen(iterable, key=None):
|
||||
"List unique elements, preserving order. Remember all elements ever seen."
|
||||
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
|
||||
# unique_everseen('ABBCcAD', str.lower) --> A B C D
|
||||
seen = set()
|
||||
seen_add = seen.add
|
||||
if key is None:
|
||||
for element in filterfalse(seen.__contains__, iterable):
|
||||
seen_add(element)
|
||||
yield element
|
||||
else:
|
||||
for element in iterable:
|
||||
k = key(element)
|
||||
if k not in seen:
|
||||
seen_add(k)
|
||||
yield element
|
||||
|
||||
packages_dir = 'packages'
|
||||
for subdir_name in sorted(os.listdir(packages_dir)):
|
||||
subdir_path = packages_dir + '/' + subdir_name
|
||||
if os.path.exists(subdir_path + '/BROKEN.txt'): continue
|
||||
build_sh_path = subdir_path + '/build.sh'
|
||||
|
||||
this_package = DebianPackage(subdir_name)
|
||||
all_packages.append(this_package)
|
||||
packages_map[this_package.name] = this_package
|
||||
def die(msg):
|
||||
sys.exit('ERROR: ' + msg)
|
||||
|
||||
if not os.path.isfile(build_sh_path): die('The directory ' + subdir_name + ' does not contain build.sh')
|
||||
with open(build_sh_path) as build_sh_file:
|
||||
for line in build_sh_file:
|
||||
if line.startswith('TERMUX_PKG_DEPENDS='):
|
||||
deps_comma_separated = line[(line.index('=')+2):(len(line)-2)]
|
||||
for dep in deps_comma_separated.split(','):
|
||||
dep = dep.strip()
|
||||
if not dep.endswith('libandroid-support-dev'): this_package.remaining_dependencies.add(dep)
|
||||
for file_in_subdir_name in sorted(os.listdir(subdir_path)):
|
||||
if file_in_subdir_name.endswith('.subpackage.sh'):
|
||||
subpackage_name = file_in_subdir_name[0:-len(".subpackage.sh"):]
|
||||
this_package.sub_packages.add(subpackage_name)
|
||||
packages_map[subpackage_name] = this_package
|
||||
with open(subdir_path + '/' + file_in_subdir_name) as subpackage_sh_file:
|
||||
for line in subpackage_sh_file:
|
||||
if line.startswith('TERMUX_SUBPKG_DEPENDS='):
|
||||
deps_comma_separated = line[(line.index('=')+2):(len(line)-2)]
|
||||
for dep in deps_comma_separated.split(','):
|
||||
dep = dep.strip()
|
||||
this_package.remaining_dependencies.add(dep)
|
||||
this_package.remaining_dependencies.discard(this_package.name) # Do not depend on itself
|
||||
this_package.remaining_dependencies.difference_update(this_package.sub_packages) # Do not depend on any sub package
|
||||
|
||||
for package in all_packages:
|
||||
for remaining in package.remaining_dependencies:
|
||||
if not remaining in packages_map: die('Package ' + package.name + ' depends on non-existing package "' + remaining + '"')
|
||||
packages_map[remaining].prerequisite_for.add(package)
|
||||
class TermuxBuildFile(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
# List of all DebianPackage:s without dependencies
|
||||
packages_without_deps = [p for p in all_packages if not p.remaining_dependencies]
|
||||
if not packages_without_deps: die('No package without dependency - where to start?')
|
||||
def _get_dependencies(self):
|
||||
pkg_dep_prefix = 'TERMUX_PKG_DEPENDS='
|
||||
subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS='
|
||||
|
||||
# Sort alphabetically, but with libandroid-support first (since dependency on libandroid-support
|
||||
# does not need to be declared explicitly, so anything might in theory depend on it to build):
|
||||
packages_without_deps.sort(key=lambda p: 'aaaa' if p.name == 'libandroid-support' else p.name, reverse=True)
|
||||
with open(self.path) as f:
|
||||
prefix = None
|
||||
for line in f:
|
||||
if line.startswith(pkg_dep_prefix):
|
||||
prefix = pkg_dep_prefix
|
||||
elif line.startswith(subpkg_dep_prefix):
|
||||
prefix = subpkg_dep_prefix
|
||||
else:
|
||||
continue
|
||||
|
||||
# Topological sorting
|
||||
build_order = []
|
||||
while packages_without_deps:
|
||||
pkg = packages_without_deps.pop()
|
||||
build_order.append(pkg)
|
||||
for other_package in pkg.prerequisite_for:
|
||||
other_package.remaining_dependencies.discard(pkg.name) # Remove this package
|
||||
other_package.remaining_dependencies.difference_update(pkg.sub_packages) # .. and all its subpackages
|
||||
if not other_package.remaining_dependencies:
|
||||
# Check if the other package is ready to build now
|
||||
packages_without_deps.append(other_package)
|
||||
comma_deps = line[len(prefix):].replace('"', '')
|
||||
|
||||
if len(all_packages) != len(build_order):
|
||||
print("ERROR: Cycle exists. Remaining: ");
|
||||
for pkg in all_packages:
|
||||
if pkg not in build_order: print(pkg.name)
|
||||
sys.exit(1)
|
||||
return set([
|
||||
dep.strip() for dep in comma_deps.split(',')
|
||||
if 'libandroid-support' not in dep
|
||||
])
|
||||
|
||||
for pkg in build_order: print(pkg.name)
|
||||
# no deps found
|
||||
return set()
|
||||
|
||||
|
||||
class TermuxPackage(object):
|
||||
PACKAGES_DIR = 'packages'
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.dir = os.path.join(self.PACKAGES_DIR, name)
|
||||
|
||||
# search package build.sh
|
||||
build_sh_path = os.path.join(self.dir, 'build.sh')
|
||||
if not os.path.isfile(build_sh_path):
|
||||
raise Exception("build.sh not found")
|
||||
|
||||
self.buildfile = TermuxBuildFile(build_sh_path)
|
||||
self.deps = self.buildfile._get_dependencies()
|
||||
|
||||
# search subpackages
|
||||
self.subpkgs = []
|
||||
|
||||
for filename in os.listdir(self.dir):
|
||||
if not filename.endswith('.subpackage.sh'):
|
||||
continue
|
||||
|
||||
subpkg_name = filename.split('.subpackage.sh')[0]
|
||||
subpkg = TermuxSubPackage(subpkg_name, parent=self)
|
||||
|
||||
self.subpkgs.append(subpkg)
|
||||
self.deps |= subpkg.deps
|
||||
|
||||
# Do not depend on itself
|
||||
self.deps.discard(self.name)
|
||||
# Do not depend on any sub package
|
||||
self.deps.difference_update([subpkg.name for subpkg in self.subpkgs])
|
||||
|
||||
self.needed_by = set() # to be completed outside, reverse of deps
|
||||
|
||||
def __repr__(self):
|
||||
return "<{} '{}'>".format(self.__class__.__name__, self.name)
|
||||
|
||||
|
||||
class TermuxSubPackage(TermuxPackage):
|
||||
|
||||
def __init__(self, name, parent=None):
|
||||
if parent is None:
|
||||
raise Exception("SubPackages should have a parent")
|
||||
|
||||
self.name = name
|
||||
self.parent = parent
|
||||
self.buildfile = TermuxBuildFile(os.path.join(self.parent.dir, name + '.subpackage.sh'))
|
||||
self.deps = self.buildfile._get_dependencies()
|
||||
|
||||
def __repr__(self):
|
||||
return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
|
||||
|
||||
|
||||
# Tracks non-visited deps for each package
|
||||
remaining_deps = {}
|
||||
|
||||
# Mapping from package name to TermuxPackage
|
||||
# (if subpackage, mapping from subpackage name to parent package)
|
||||
pkgs_map = {}
|
||||
|
||||
# Reverse dependencies
|
||||
pkg_depends_on = {}
|
||||
|
||||
PACKAGES_DIR = 'packages'
|
||||
|
||||
|
||||
def populate():
|
||||
|
||||
for pkgdir_name in sorted(os.listdir(PACKAGES_DIR)):
|
||||
|
||||
pkg = TermuxPackage(pkgdir_name)
|
||||
|
||||
pkgs_map[pkg.name] = pkg
|
||||
|
||||
for subpkg in pkg.subpkgs:
|
||||
pkgs_map[subpkg.name] = pkg
|
||||
remaining_deps[subpkg.name] = set(subpkg.deps)
|
||||
|
||||
remaining_deps[pkg.name] = set(pkg.deps)
|
||||
|
||||
all_pkg_names = set(pkgs_map.keys())
|
||||
|
||||
for name, pkg in pkgs_map.items():
|
||||
for dep_name in remaining_deps[name]:
|
||||
if dep_name not in all_pkg_names:
|
||||
die('Package %s depends on non-existing package "%s"' % (
|
||||
name, dep_name
|
||||
))
|
||||
|
||||
dep_pkg = pkgs_map[dep_name]
|
||||
dep_pkg.needed_by.add(pkg)
|
||||
|
||||
|
||||
def generate_full_buildorder():
|
||||
build_order = []
|
||||
|
||||
# List of all TermuxPackages without dependencies
|
||||
leaf_pkgs = [pkg for name, pkg in pkgs_map.items() if not pkg.deps]
|
||||
|
||||
if not leaf_pkgs:
|
||||
die('No package without dependencies - where to start?')
|
||||
|
||||
# Sort alphabetically, but with libandroid-support first (since dependency on libandroid-support
|
||||
# does not need to be declared explicitly, so anything might in theory depend on it to build):
|
||||
pkg_queue = sorted(leaf_pkgs, key=lambda p: '' if p.name == 'libandroid-support' else p.name)
|
||||
|
||||
# Topological sorting
|
||||
visited = set()
|
||||
|
||||
while pkg_queue:
|
||||
pkg = pkg_queue.pop(0)
|
||||
if pkg.name in visited:
|
||||
continue
|
||||
|
||||
# print("Processing {}:".format(pkg.name), pkg.needed_by)
|
||||
visited.add(pkg.name)
|
||||
build_order.append(pkg)
|
||||
|
||||
for other_pkg in sorted(pkg.needed_by, key=lambda p: p.name):
|
||||
# Remove this pkg from deps
|
||||
remaining_deps[other_pkg.name].discard(pkg.name)
|
||||
# ... and all its subpackages
|
||||
remaining_deps[other_pkg.name].difference_update(
|
||||
[subpkg.name for subpkg in pkg.subpkgs]
|
||||
)
|
||||
|
||||
if not remaining_deps[other_pkg.name]: # all deps were already appended?
|
||||
pkg_queue.append(other_pkg) # should be processed
|
||||
|
||||
if set(pkgs_map.values()) != set(build_order):
|
||||
print("ERROR: Cycle exists. Remaining: ")
|
||||
for name, pkg in pkgs_map.items():
|
||||
if pkg not in build_order:
|
||||
print(name, remaining_deps[name])
|
||||
|
||||
sys.exit(1)
|
||||
|
||||
return build_order
|
||||
|
||||
|
||||
def deps_then_me(pkg):
|
||||
l = []
|
||||
|
||||
for dep in sorted(pkg.deps):
|
||||
l += deps_then_me(pkgs_map[dep])
|
||||
l += [pkg]
|
||||
|
||||
return l
|
||||
|
||||
|
||||
def generate_targets_buildorder(targetnames):
|
||||
buildorder = []
|
||||
|
||||
for pkgname in targetnames:
|
||||
buildorder += deps_then_me(pkgs_map[pkgname])
|
||||
|
||||
return unique_everseen(buildorder)
|
||||
|
||||
if __name__ == '__main__':
|
||||
populate()
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
bo = generate_full_buildorder()
|
||||
else:
|
||||
bo = generate_targets_buildorder(sys.argv[1:])
|
||||
|
||||
for pkg in bo:
|
||||
print(pkg.name)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user