diff --git a/.travis.yml b/.travis.yml index 691d43524..3a7737342 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ cache: bundler services: docker before_script: mkdir debs && chmod 777 debs env: - - TRAVIS_ARCH=aarch64 - - TRAVIS_ARCH=arm - - TRAVIS_ARCH=i686 - - TRAVIS_ARCH=x86_64 + matrix: + - ARGS="-i -a aarch64" + - ARGS="-i -a arm" + - ARGS="-i -a i686" + - ARGS="-i -a x86_64" +script: bundle exec rake build["${ARGS}"] diff --git a/Rakefile b/Rakefile index bf41db863..408d0ece3 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'pty' task default: %w[build] -task :build do +task :build, [:options] do |t, args| repo = Rugged::Repository.new('.') commit = repo.head.target parent = commit.parents.first @@ -20,13 +20,18 @@ task :build do puts "Building #{pkg}" begin # Start blocking build loop - PTY.spawn("./scripts/run-docker.sh ./build-package.sh -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| + PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} #{pkg}") do |stdout, stdin, pid| begin + stdout.sync stdout.each { |line| print line } - rescue Errno::EIO + rescue Errno::EIO => e + puts e + ensure + ::Process.wait pid end end - rescue PTY::ChildExited + rescue PTY::ChildExited => e + puts e puts "Process exited" end # Exit if PTY return a non-zero code diff --git a/build-all.sh b/build-all.sh index cd2976082..0b2cf76ae 100755 --- a/build-all.sh +++ b/build-all.sh @@ -8,20 +8,24 @@ test -f $HOME/.termuxrc && . $HOME/.termuxrc : ${TERMUX_TOPDIR:="$HOME/.termux-build"} : ${TERMUX_ARCH:="aarch64"} : ${TERMUX_DEBUG:=""} +: ${TERMUX_INSTALL_DEPS:="-s"} +# Set TERMUX_INSTALL_DEPS to -s unless set to -i -_show_usage() { - echo "Usage: ./build-all.sh [-a ARCH] [-d] [-o DIR]" +_show_usage () { + echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR]" echo "Build all packages." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." + echo " -i Build dependencies." echo " -o Specify deb directory. Default: debs/." exit 1 } -while getopts :a:hdDso: option; do +while getopts :a:hdio: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; d) TERMUX_DEBUG='-d';; + i) TERMUX_INSTALL_DEPS='-i';; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; h) _show_usage;; esac @@ -63,8 +67,8 @@ for package_path in `cat $BUILDORDER_FILE`; do echo -n "Building $package... " BUILD_START=`date "+%s"` - bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s \ - $TERMUX_DEBUG ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $package \ + bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \ + ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \ > $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err BUILD_END=`date "+%s"` BUILD_SECONDS=$(( $BUILD_END - $BUILD_START )) diff --git a/build-package.sh b/build-package.sh index a55d1498c..3692aabeb 100755 --- a/build-package.sh +++ b/build-package.sh @@ -247,25 +247,29 @@ termux_setup_cmake() { # First step is to handle command-line arguments. Not to be overridden by packages. termux_step_handle_arguments() { - _show_usage() { - echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-q] [-s] [-o DIR] PACKAGE" + _show_usage () { + echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-i] [-I] [-q] [-s] [-o DIR] PACKAGE" echo "Build a package by creating a .deb file in the debs/ folder." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." + echo " -i Download and extract dependencies instead of building them." + echo " -I Download and extract dependencies instead of building them, keep existing /data/data/com.termux files." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." exit 1 } - while getopts :a:hdDfqso: option; do + while getopts :a:hdDfiIqso: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; h) _show_usage;; d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; + i) export TERMUX_INSTALL_DEPS=true;; + I) export TERMUX_INSTALL_DEPS=true && export TERMUX_NO_CLEAN=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -280,7 +284,7 @@ termux_step_handle_arguments() { # Handle 'all' arch: if [ -n "${TERMUX_ARCH+x}" ] && [ "${TERMUX_ARCH}" = 'all' ]; then for arch in 'aarch64' 'arm' 'i686' 'x86_64'; do - ./build-package.sh ${TERMUX_FORCE_BUILD+-f} -a $arch \ + ./build-package.sh ${TERMUX_FORCE_BUILD+-f} -a $arch ${TERMUX_INSTALL_DEPS+-i} \ ${TERMUX_DEBUG+-d} ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} "$1" done exit @@ -320,9 +324,27 @@ termux_step_setup_variables() { : "${TERMUX_ANDROID_HOME:="/data/data/com.termux/files/home"}" : "${TERMUX_DEBUG:=""}" : "${TERMUX_PKG_API_LEVEL:="21"}" + : "${TERMUX_NO_CLEAN:="true"}" + : "${TERMUX_QUIET_BUILD:="false"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" + : "${TERMUX_SKIP_DEPCHECK:="false"}" + : "${TERMUX_INSTALL_DEPS:="false"}" + : "${TERMUX_REPO_SIGNING_KEYS:="packages/apt/trusted.gpg packages/termux-keyring/grimler.gpg packages/termux-keyring/xeffyr.gpg"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" + if [ -z ${TERMUX_REPO_URL+x} ]; then + TERMUX_REPO_URL=(https://termux.net/dists) + # TERMUX_REPO_URL=(https://termux.net/dists https://grimler.se/dists https://dl.bintray.com/xeffyr/unstable-packages/dists) + fi + if [ -z ${TERMUX_REPO_DISTRIBUTION+x} ]; then + TERMUX_REPO_DISTRIBUTION=(stable) + # TERMUX_REPO_DISTRIBUTION=(stable root unstable) + fi + if [ -z ${TERMUX_REPO_COMPONENT+x} ]; then + TERMUX_REPO_COMPONENT=(main) + # TERMUX_REPO_COMPONENT=(main stable main) + fi + if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then TERMUX_ARCH_BITS=64 else @@ -423,6 +445,100 @@ termux_step_handle_buildarch() { echo "$TERMUX_ARCH" > $TERMUX_ARCH_FILE } +# Function to get TERMUX_PKG_VERSION from build.sh +source scripts/termux_extract_dep_info.sh + +termux_download_deb() { + local package=$1 + local package_arch=$2 + local version=$3 + local deb_file=${package}_${version}_${package_arch}.deb + for idx in $(seq ${#TERMUX_REPO_URL[@]}); do + local TERMUX_REPO_NAME=$(echo ${TERMUX_REPO_URL[$idx-1]} | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/dists%%g' -e 's%/%-%g') + local PACKAGE_FILE_PATH="${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-${TERMUX_REPO_COMPONENT[$idx-1]}-Packages" + PKG_HASH=$(./scripts/get_hash_from_file.py "${TERMUX_COMMON_CACHEDIR}-$arch/$PACKAGE_FILE_PATH" $package $version) + if ! [ "$PKG_HASH" = "" ]; then + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then + echo "Found $package in ${TERMUX_REPO_URL[$idx-1]}" + fi + break + fi + done + if [ "$PKG_HASH" = "" ]; then + return 1 + else + termux_download ${TERMUX_REPO_URL[$idx-1]}/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/${TERMUX_REPO_COMPONENT[$idx-1]}/binary-${package_arch}/${deb_file} \ + $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ + $PKG_HASH + return 0 + fi +} + +# Script to download InRelease, verify it's signature and then download Packages.xz by hash +termux_step_get_repo_files() { + # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) + mkdir -p "$TERMUX_COMMON_CACHEDIR" \ + "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ + "$TERMUX_COMMON_CACHEDIR-all" \ + "$TERMUX_DEBDIR" \ + "$TERMUX_PKG_BUILDDIR" \ + "$TERMUX_PKG_PACKAGEDIR" \ + "$TERMUX_PKG_TMPDIR" \ + "$TERMUX_PKG_CACHEDIR" \ + "$TERMUX_PKG_MASSAGEDIR" \ + $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} + if [ "$TERMUX_INSTALL_DEPS" = true ]; then + if [ "$TERMUX_NO_CLEAN" = false ]; then + # Remove all previously extracted/built files from $TERMUX_PREFIX: + rm -rf $TERMUX_PREFIX + rm -f /data/data/.built-packages/* + # Setup bootstrap + if [ $TERMUX_ARCH == aarch64 ]; then + local bootstrap_sha256=2944ad699814329007d1f9c056e7c8323243c8b4a257cbd05904216f89fc3746 + elif [ $TERMUX_ARCH == i686 ]; then + local bootstrap_sha256=8f4dee0b1e161689b60f330ac0cc813b56ab479f2cd789eb8459165a3be13bdb + elif [ $TERMUX_ARCH == arm ]; then + local bootstrap_sha256=f471c0af326677d87ca4926d54860d10d751dd4f8d615d5b1de902841601b41e + elif [ $TERMUX_ARCH == x86_64 ]; then + local bootstrap_sha256=93384f0343c13f604dbacd069276291bd7042fc6d42c6d7514c7e573d968c614 + fi + termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ + ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ + $bootstrap_sha256 + unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX + ( + cd $TERMUX_PREFIX + while read link; do + ln -sf ${link/←/ } + done= 1.0) and similar version tags with sed: + _PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS//,/ } | sed "s/[(][^)]*[)]//g") + _PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS//,/ } + for pkg in $_PKG_DEPENDS $_PKG_BUILD_DEPENDS; do + # llvm doesn't build if ndk-sysroot is installed: + if [ "$pkg" = "ndk-sysroot" ]; then continue; fi + read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") + + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then + echo "Downloading dependency $pkg@$dep_version if necessary..." + fi + if ! termux_download_deb $pkg $dep_arch $dep_version; then + echo "Download of $pkg@$dep_version from $TERMUX_REPO_URL failed, building instead" + ./build-package.sh -a $TERMUX_ARCH -I "$pkg" + continue + else + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $pkg..."; fi + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x ${pkg}_${dep_version}_${dep_arch}.deb data.tar.xz + tar -xf data.tar.xz --no-overwrite-dir -C / + ) + fi + + if termux_download_deb $pkg-dev $dep_arch $dep_version; then + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x $pkg-dev_${dep_version}_${dep_arch}.deb data.tar.xz + tar xf data.tar.xz --no-overwrite-dir -C / + ) + else + echo "Download of $pkg-dev@$dep_version from $TERMUX_REPO_URL failed" + fi + mkdir -p /data/data/.built-packages + echo "$dep_version" > "/data/data/.built-packages/$pkg" + done + elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then + # Build dependencies + local pkg + for pkg in $TERMUX_ALL_DEPS; do + echo "Building dependency $pkg if necessary..." # Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one - ./build-package.sh -a $TERMUX_ARCH -s "$p" + ./build-package.sh -a $TERMUX_ARCH -s "$pkg" done fi @@ -454,7 +611,7 @@ termux_step_start_build() { TERMUX_PKG_FULLVERSION+="-$TERMUX_PKG_REVISION" fi - if [ "$TERMUX_DEBUG" == "true" ]; then + if [ "$TERMUX_DEBUG" = true ]; then if [ "$TERMUX_PKG_HAS_DEBUG" == "yes" ]; then DEBUG="-dbg" else @@ -483,7 +640,7 @@ termux_step_start_build() { # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_DEBDIR" \ + "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ "$TERMUX_PKG_TMPDIR" \ @@ -1315,6 +1472,22 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } +termux_step_compare_debs() { + if [ "${TERMUX_INSTALL_DEPS}" = true ]; then + cd ${TERMUX_SCRIPTDIR} + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi + + termux_download_deb $TERMUX_PKG_NAME $TERMUX_ARCH $TERMUX_PKG_FULLVERSION \ + && ( + deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb + + # `|| true` to prevent debdiff's exit code from stopping build + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi + ) || echo "Download of ${TERMUX_PKG_NAME}@${TERMUX_PKG_FULLVERSION} failed, not comparing debs" + fi +} + # Finish the build. Not to be overridden by package scripts. termux_step_finish_build() { echo "termux - build of '$TERMUX_PKG_NAME' done" @@ -1327,6 +1500,7 @@ termux_step_finish_build() { termux_step_handle_arguments "$@" termux_step_setup_variables termux_step_handle_buildarch +termux_step_get_repo_files termux_step_start_build termux_step_extract_package cd "$TERMUX_PKG_SRCDIR" @@ -1355,4 +1529,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile +termux_step_compare_debs termux_step_finish_build diff --git a/scripts/get_hash_from_file.py b/scripts/get_hash_from_file.py new file mode 100755 index 000000000..bf7dfd009 --- /dev/null +++ b/scripts/get_hash_from_file.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import os, sys + +def get_pkg_hash_from_Packages(Packages_file, package, version, hash="SHA256"): + with open(Packages_file, 'r') as Packages: + package_list = Packages.read().split('\n\n') + for pkg in package_list: + if pkg.split('\n')[0] == "Package: "+package: + for line in pkg.split('\n'): + if line.startswith('Version:'): + if line != 'Version: '+version: + # Seems the repo contains the wrong version, or several versions + # We can't use this one so continue looking + break + elif line.startswith(hash): + print(line.split(" ")[1]) + break + +def get_Packages_hash_from_InRelease(InRelease_file, arch, component, hash="SHA256"): + string_to_find = component+'/binary-'+arch+'/Packages.xz' + with open(InRelease_file, 'r') as InRelease: + hash_list = InRelease.readlines() + for i in range(len(hash_list)): + if hash_list[i].startswith(hash+':'): + break + for j in range(i, len(hash_list)): + if string_to_find in hash_list[j].strip(' '): + print(hash_list[j].strip(' ').split(' ')[0]) + break + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit('Too few arguments, I need the path to a Packages file, a package name and a version, or an InRelease file, an architecture and a component name. Exiting') + + if sys.argv[1].endswith('Packages'): + get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2], sys.argv[3]) + elif sys.argv[1].endswith('InRelease'): + get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2], sys.argv[3]) + else: + sys.exit(sys.argv[1]+' does not seem to be a path to a Packages or InRelease file') diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index ae4b111ea..160c92b75 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -37,6 +37,8 @@ PACKAGES+=" libexpat1-dev" # Needed by ghostscript PACKAGES+=" libjpeg-dev" # Needed by ghostscript PACKAGES+=" gawk" # Needed by apr-util PACKAGES+=" libssl-dev" # Needed to build Rust +PACKAGES+=" gnupg" # Needed to verify downloaded .debs +PACKAGES+=" devscripts" # Provides utility "debdiff". sudo DEBIAN_FRONTEND=noninteractive \ apt-get install -yq --no-install-recommends $PACKAGES diff --git a/scripts/termux_extract_dep_info.sh b/scripts/termux_extract_dep_info.sh new file mode 100644 index 000000000..91b99c301 --- /dev/null +++ b/scripts/termux_extract_dep_info.sh @@ -0,0 +1,41 @@ +termux_extract_dep_info() { + package=$1 + if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then + # We are dealing with a subpackage + TERMUX_ARCH=$( + # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed + TERMUX_PKG_PLATFORM_INDEPENDENT="" + source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh + TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT + source $(find packages/ -name "$package.subpackage.sh") + if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then + echo all + else + echo $TERMUX_ARCH + fi + ) + + package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) + elif [ "${package/-dev/}-dev" == "${package}" ]; then + # dev package + package=${package/-dev/} + fi + ( + # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't + # mandatory in a build.sh. Otherwise these will equal the main package's values for + # deps that should have the default values + TERMUX_PKG_PLATFORM_INDEPENDENT="" + TERMUX_PKG_REVISION="0" + source packages/$package/build.sh + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi + if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then + TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" + fi + echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} + ) +} + +# Make script standalone executable as well as sourceable +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + termux_extract_dep_info "$@" +fi