From ed9b855275758658b602654979ce0b5e104b7558 Mon Sep 17 00:00:00 2001 From: Leonid Pliushch Date: Mon, 4 Oct 2021 13:17:04 +0300 Subject: [PATCH] auto updates: get rid of github-projects.txt From now, all configuration related to automatic updates should be done in build.sh scripts. It requires specifying at least TERMUX_PKG_AUTO_UPDATE=true which indicates that package is eligible for automatic updates, and optionally TERMUX_PKG_AUTO_UPDATE_TAG_REGEXP="$version_regexp" which specifies a regular expression used to extract the version part from Git tag. --- scripts/updates/check-updates.sh | 167 ++++++++------ scripts/updates/github-projects.txt | 345 ---------------------------- 2 files changed, 96 insertions(+), 416 deletions(-) delete mode 100644 scripts/updates/github-projects.txt diff --git a/scripts/updates/check-updates.sh b/scripts/updates/check-updates.sh index 822d4ae1e..417498a00 100755 --- a/scripts/updates/check-updates.sh +++ b/scripts/updates/check-updates.sh @@ -11,84 +11,109 @@ BASEDIR=$(dirname "$(realpath "$0")") : "${GIT_PUSH_PACKAGES:=false}" if [ -z "${GITHUB_API_TOKEN-}" ]; then - echo "You need a Github Personal Access Token be set in variable GITHUB_API_TOKEN." + echo "Error: you need a Github Personal Access Token be set in variable GITHUB_API_TOKEN." exit 1 fi -if [ -f "${BASEDIR}/github-projects.txt" ]; then - for line in $(grep -P '^[a-z0-9]' "${BASEDIR}/github-projects.txt"); do - unset package project version_regexp termux_version termux_epoch latest_version - package=$(echo "$line" | cut -d'|' -f1) - project=$(echo "$line" | cut -d'|' -f2) - version_regexp=$(echo "$line" | cut -d'|' -f3-) +for pkg_dir in "${BASEDIR}"/../../packages/*; do + if [ -f "${pkg_dir}/build.sh" ]; then + package=$(basename "$pkg_dir") + else + # Fail if detected a non-package directory. + echo "Error: directory '${pkg_dir}' is not a package." + exit 1 + fi - if [ ! -d "${BASEDIR}/../../packages/${package}" ]; then - echo "Package '$package' is not available, skipping." - continue - fi + # Extract the package auto-update configuration. + build_vars=$( + set +e +u + . "${BASEDIR}/../../packages/${package}/build.sh" 2>/dev/null + echo "auto_update_flag=${TERMUX_PKG_AUTO_UPDATE};" + echo "termux_version=\"${TERMUX_PKG_VERSION}\";" + echo "srcurl=\"${TERMUX_PKG_SRCURL}\";" + echo "version_regexp=\"${TERMUX_PKG_AUTO_UPDATE_TAG_REGEXP//\\/\\\\}\";" + ) + auto_update_flag=""; termux_version=""; srcurl=""; version_regexp=""; + eval "$build_vars" - # Our local version of package. - termux_version=$(set +e +u;. "${BASEDIR}/../../packages/${package}/build.sh" 2>/dev/null; echo "$TERMUX_PKG_VERSION") - termux_epoch="$(echo "$termux_version" | cut -d: -f1)" - termux_version=$(echo "$termux_version" | cut -d: -f2-) - if [ "$termux_version" == "$termux_epoch" ]; then - # No epoch set. - termux_epoch="" + # Ignore packages that have auto-update disabled. + if [ "${auto_update_flag}" != "true" ]; then + continue + fi + + # Extract github project from TERMUX_PKG_SRCURL + project="$(echo "${srcurl}" | grep github.com | cut -d / -f4-5)" + if [ -z "${project}" ]; then + echo "Error: package '${package}' doesn't use GitHub archive source URL but has been configured for automatic updates." + exit 1 + fi + + # Our local version of package. + termux_epoch="$(echo "$termux_version" | cut -d: -f1)" + termux_version=$(echo "$termux_version" | cut -d: -f2-) + if [ "$termux_version" == "$termux_epoch" ]; then + # No epoch set. + termux_epoch="" + else + termux_epoch+=":" + fi + + # Get the latest release tag. + latest_tag=$(curl --silent --location -H "Authorization: token ${GITHUB_API_TOKEN}" "https://api.github.com/repos/${project}/releases/latest" | jq -r .tag_name) + + # If the github api returns error + if [ -z "$latest_tag" ] || [ "${latest_tag}" = "null" ]; then + echo "Error: failed to get the latest release tag for '${package}'. GitHub API returned 'null' which indicates that no releases available." + exit 1 + fi + + # Remove leading 'v' which is common in version tag. + latest_version=${latest_tag#v} + + # If needed, filter version numbers from tag by using regexp. + if [ -n "$version_regexp" ]; then + latest_version=$(grep -oP "$version_regexp" <<< "$latest_version" || true) + fi + if [ -z "$latest_version" ]; then + echo "Error: failed to get latest version for '${package}'. Check whether the TERMUX_PKG_AUTO_UPDATE_TAG_REGEXP='${version_regexp}' is work right with latest_release='${latest_tag}'." + exit 1 + fi + + # Translate "_" into ".": some packages use underscores to seperate + # version numbers, but we require them to be separated by dots. + latest_version=${latest_version//_/.} + + # We have no better choice for comparing versions. + if [ "$(echo -e "${termux_version}\n${latest_version}" | sort -V | head -n 1)" != "$latest_version" ] ;then + if [ "$BUILD_PACKAGES" = "false" ]; then + echo "Package '${package}' needs update to '${latest_version}'." else - termux_epoch+=":" - fi + echo "Updating '${package}' to '${latest_version}'." + sed -i "s/^\(TERMUX_PKG_VERSION=\)\(.*\)\$/\1${termux_epoch}${latest_version}/g" "${BASEDIR}/../../packages/${package}/build.sh" + sed -i "/TERMUX_PKG_REVISION=/d" "${BASEDIR}/../../packages/${package}/build.sh" + echo n | "${BASEDIR}/../bin/update-checksum" "$package" || { + echo "Warning: failed to update checksum for '${package}', skipping..." + git checkout -- "${BASEDIR}/../../packages/${package}" + git pull --rebase + continue + } - # Latest version is the current release tag on Github. - latest_version=$(curl --silent --location -H "Authorization: token ${GITHUB_API_TOKEN}" "https://api.github.com/repos/${project}/releases/latest" | jq -r .tag_name) - - # Remove leading 'v' which is common in version tag. - latest_version=${latest_version#v} - - # If needed, filter version numbers from tag by using regexp. - if [ -n "$version_regexp" ]; then - latest_version=$(grep -oP "$version_regexp" <<< "$latest_version" || true) - fi - - if [[ -z "$latest_version" || "$latest_version" = "null" ]]; then - echo "Failed to get latest version for '${package}'. Update 'github-projects.txt'." - exit 1 - fi - - # Translate "_" into ".". - latest_version=${latest_version//_/.} - - # We have no better choice for comparing versions. - if [ "$(echo -e "${termux_version}\n${latest_version}" | sort -V | head -n 1)" != "$latest_version" ] ;then - if [ "$BUILD_PACKAGES" = "false" ]; then - echo "Package '${package}' needs update to '${latest_version}'." - else - echo "Updating '${package}' to '${latest_version}'." - sed -i "s/^\(TERMUX_PKG_VERSION=\)\(.*\)\$/\1${termux_epoch}${latest_version}/g" "${BASEDIR}/../../packages/${package}/build.sh" - sed -i "/TERMUX_PKG_REVISION=/d" "${BASEDIR}/../../packages/${package}/build.sh" - echo n | "${BASEDIR}/../bin/update-checksum" "$package" || { - echo "Failed to update checksum for '${package}', skipping..." - git checkout -- "${BASEDIR}/../../packages/${package}" - git pull --rebase - continue - } - - echo "Trying to build package '${package}'." - if "${BASEDIR}/../run-docker.sh" ./build-package.sh -a aarch64 -I "$package" && \ - "${BASEDIR}/../run-docker.sh" ./build-package.sh -a arm -I "$package"; then - if [ "$GIT_COMMIT_PACKAGES" = "true" ]; then - git add "${BASEDIR}/../../packages/${package}" - git commit -m "$(echo -e "${package}: update to ${latest_version}\n\nThis commit has been automatically submitted by Github Actions.")" - fi - - if [ "$GIT_PUSH_PACKAGES" = "true" ]; then - git pull --rebase - git push - fi - else - echo "Failed to build '${package}'." - git checkout -- "${BASEDIR}/../../packages/${package}" + echo "Trying to build package '${package}'." + if "${BASEDIR}/../run-docker.sh" ./build-package.sh -a aarch64 -I "$package" && \ + "${BASEDIR}/../run-docker.sh" ./build-package.sh -a arm -I "$package"; then + if [ "$GIT_COMMIT_PACKAGES" = "true" ]; then + git add "${BASEDIR}/../../packages/${package}" + git commit -m "$(echo -e "${package}: update to ${latest_version}\n\nThis commit has been automatically submitted by Github Actions.")" fi + + if [ "$GIT_PUSH_PACKAGES" = "true" ]; then + git pull --rebase + git push + fi + else + echo "Warning: failed to build '${package}'." + git checkout -- "${BASEDIR}/../../packages/${package}" fi fi - done -fi + fi +done diff --git a/scripts/updates/github-projects.txt b/scripts/updates/github-projects.txt deleted file mode 100644 index 195527b68..000000000 --- a/scripts/updates/github-projects.txt +++ /dev/null @@ -1,345 +0,0 @@ -## -## A list of package upstreams located on Github which should be -## checked for updates. -## -## Each entry has a following format: -## -## {package name}|{project path on Github}|{optionally: version regexp} -## -## Example: acr|radare/acr -## cronie|cronie-crond/cronie|\d+.\d+.\d+ -## -## Lines starting with "#" are comments. -## -## What should not be included there: -## -## - Big packages like rust or swift. -## - Projects which belong to Termux GitHub organization. -## - Projects with releases from multiple branches: boinc. -## - Projects without GitHub releases. -## - Other projects unsuitable for automatic updates: -## -## * ghostscript -## * jbig2dec -## * libicu -## * libllvm -## * libsearpc -## * rust-analyzer -## * texlive-bin -## - -acr|radare/acr -algernon|xyproto/algernon -android-tools|nmeum/android-tools -antibody|getantibody/antibody -aptly|aptly-dev/aptly -aria2|aria2/aria2|\d+\.\d+\.\d+ -asciidoc|asciidoc/asciidoc-py3 -asciinema|asciinema/asciinema -assimp|assimp/assimp -atomicparsley|wez/atomicparsley -axel|axel-download-accelerator/axel -bash-completion|scop/bash-completion -bat|sharkdp/bat -beanshell|beanshell/beanshell -bgrep|rsharo/bgrep|\d+\.\d+ -binaryen|WebAssembly/binaryen|\d+ -bitcoin|bitcoin/bitcoin -blogc|blogc/blogc -bmon|tgraf/bmon -borgbackup|borgbackup/borg -brook|txthinking/brook -brotli|google/brotli -btfs|TRON-US/go-btfs|\d+\.\d+\.\d+ -caddy|caddyserver/caddy -calc|lcn2/calc -capstone|aquynh/capstone -c-ares|c-ares/c-ares|\d+.\d+.\d+ -cava|karlstav/cava -cavif-rs|kornelski/cavif-rs -ccache|ccache/ccache -check|libcheck/check -chezmoi|twpayne/chezmoi -chromaprint|acoustid/chromaprint -cicada|mitnk/cicada -cmark|github/cmark -cmatrix|abishekvashok/cmatrix -cmus|cmus/cmus -croc|schollz/croc -cronie|cronie-crond/cronie|\d+\.\d+\.\d+ -cryptopp|weidai11/cryptopp|\d+.\d+.\d+ -c-toxcore|TokTok/toxcore -ctypes-sh|taviso/ctypes.sh -darkhttpd|emikulic/darkhttpd -dasm|dasm-assembler/dasm -delve|go-delve/delve -deutex|Doom-Utils/deutex -direnv|direnv/direnv -diskus|sharkdp/diskus -dns2tcp|alex-sector/dns2tcp -dnsmap|resurrecting-open-source-projects/dnsmap -docopt|docopt/docopt.cpp -dosfstools|dosfstools/dosfstools -double-conversion|google/double-conversion -duc|zevv/duc -duf|muesli/duf -dust|bootandy/dust -dvtm|martanne/dvtm -eja|eja/eja -elixir|elixir-lang/elixir -enchant|AbiWord/enchant -erlang|erlang/otp|\d+\.\d+ -espeak|espeak-ng/espeak-ng -et|MisterTea/EternalTerminal|\d+\.\d+\.\d+ -exa|ogham/exa -exiv2|Exiv2/exiv2 -fastmod|facebookincubator/fastmod -fd|sharkdp/fd -fdupes|adrianlopezroche/fdupes -findomain|Findomain/Findomain -fish|fish-shell/fish-shell -flatbuffers|google/flatbuffers -flex|westes/flex -flyctl|superfly/flyctl -fmt|fmtlib/fmt -fribidi|fribidi/fribidi -frobtads|realnc/frobtads -fselect|jhspetersson/fselect -fsmon|nowsecure/fsmon -fzf|junegunn/fzf -fzy|jhawthorn/fzy -gbt|jtyr/gbt -germanium|matsuyoshi30/germanium -geth|ethereum/go-ethereum -gflags|gflags/gflags -gh|cli/cli -gifski|ImageOptim/gifski -git-delta|dandavison/delta -gitea|go-gitea/gitea -git-lfs|git-lfs/git-lfs -glow|charmbracelet/glow -gogs|gogs/gogs -google-glog|google/glog -googletest|google/googletest|\d+\.\d+\.\d+ -gopass|gopasspw/gopass -gotty|sorenisanerd/gotty -gping|orf/gping|\d+\.\d+\.\d+ -gumbo-parser|google/gumbo-parser -harfbuzz|harfbuzz/harfbuzz -hashdeep|jessek/hashdeep -helm|helm/helm -hexcurse|LonnyGomes/hexcurse -hexyl|sharkdp/hexyl -heyu|HeyuX10Automation/heyu -hiptext|jart/hiptext -hors|WindSoilder/hors -hub|github/hub -hugo|gohugoio/hugo -hunspell|hunspell/hunspell -hydroxide|emersion/hydroxide -hyperfine|sharkdp/hyperfine -i2pd|PurpleI2P/i2pd -imgflo|imgflo/imgflo -inotify-tools|rvoicilas/inotify-tools -ipfs|ipfs/go-ipfs -irssi|irssi/irssi -iverilog|steveicarus/iverilog|\d+.\d+ -iwyu|include-what-you-use/include-what-you-use -jfrog-cli|jfrog/jfrog-cli -jftui|Aanok/jftui -jo|jpmens/jo -jp2a|Talinx/jp2a -jq|stedolan/jq|\d+\.\d+ -jsoncpp|open-source-parsers/jsoncpp -k9s|derailed/k9s -kak-lsp|kak-lsp/kak-lsp -kakoune|mawww/kakoune -keybase|keybase/client -keychain|funtoo/keychain -keystone|keystone-engine/keystone -kibi|ilai-deutel/kibi -kona|kevinlawler/kona|\d{8} -lastpass-cli|lastpass/lastpass-cli -latino|primitivorm/latino-termux -lazygit|jesseduffield/lazygit -ledger|ledger/ledger -leptonica|DanBloomberg/leptonica -lesspipe|wofr06/lesspipe -leveldb|google/leveldb -lf|gokcehan/lf|\d+ -libarchive|libarchive/libarchive -libass|libass/libass -libcap-ng|stevegrubb/libcap-ng -libconfig|hyperrealm/libconfig -libconfuse|martinh/libconfuse -libcue|lipnitsk/libcue -libcunit|Linaro/libcunit|\d+\.\d+.\d+ -libcurl|curl/curl|\d+.\d+.\d+ -libczmq|zeromq/czmq -libde265|strukturag/libde265 -libdispatch|apple/swift-corelibs-libdispatch|\d+\.\d+ -libevent|libevent/libevent|\d+\.\d+\.\d+ -libexif|libexif/libexif|\d+.\d+.\d+ -libexpat|libexpat/libexpat|\d+.\d+.\d+ -libgd|libgd/libgd|\d+\.\d+\.\d+ -libgit2|libgit2/libgit2 -libgraphite|silnrsi/graphite -libheif|strukturag/libheif -libical|libical/libical -libimobiledevice|libimobiledevice/libimobiledevice -libjansson|akheron/jansson -libjasper|mdadams/jasper|\d+\.\d+\.\d+ -libjxl|libjxl/libjxl -liblz4|lz4/lz4 -libmaxminddb|maxmind/libmaxminddb -libmesode|boothj5/libmesode -libmsgpack|msgpack/msgpack-c|\d+\.\d+\.\d+ -libnet|libnet/libnet -libnghttp2|nghttp2/nghttp2 -libnl|thom311/libnl|\d+.\d+.\d+ -libogg|xiph/ogg -libplist|libimobiledevice/libplist -libprotobuf|protocolbuffers/protobuf -libprotobuf-c|protobuf-c/protobuf-c -libpsl|rockdaboot/libpsl -libqrencode|fukuchi/libqrencode -libraqm|HOST-Oman/libraqm -librav1e|xiph/rav1e|\d+\.\d+\.\d+ -librsync|librsync/librsync -libsndfile|erikd/libsndfile -libsodium|jedisct1/libsodium|\d+\.\d+\.\d+ -libstrophe|strophe/libstrophe -libtreesitter|tree-sitter/tree-sitter -libunibilium|neovim/unibilium -libusb|libusb/libusb -libusbmuxd|libimobiledevice/libusbmuxd -libvips|libvips/libvips -libwren|wren-lang/wren -libyaml|yaml/libyaml -libzip|nih-at/libzip -libzmq|zeromq/libzmq -libzopfli|google/zopfli|\d+\.\d+\.\d+ -llbuild|apple/swift-llbuild -logrotate|logrotate/logrotate -loksh|dimkr/loksh -lsd|Peltoche/lsd -lsof|lsof-org/lsof -luv|luvit/luv -lychee|lycheeverse/lychee -macchina|Macchina-CLI/macchina -mdns-scan|alteholz/mdns-scan -mg|hboetes/mg|\d{8} -microsocks|rofl0r/microsocks -minisign|jedisct1/minisign -mktorrent|Rudde/mktorrent -mosh|mobile-shell/mosh|\d+\.\d+\.\d+ -mpv|mpv-player/mpv -mu|djcb/mu -navi|denisidoro/navi -ncompress|vapier/ncompress -neofetch|dylanaraps/neofetch -neomutt|neomutt/neomutt|\d{8} -neovim|neovim/neovim -ninja|ninja-build/ninja -nlohmann-json|nlohmann/json -nnn|jarun/nnn -nushell|nushell/nushell -nyancat|klange/nyancat -nzbget|nzbget/nzbget -o-editor|xyproto/o -okc-agents|DDoSolitary/okc-agents -oniguruma|kkos/oniguruma -openethereum|openethereum/openethereum -openjpeg|uclouvain/openjpeg -opusfile|xiph/opusfile -p7zip|jinfeihan57/p7zip -par2|Parchive/par2cmdline -pass-otp|tadfisher/pass-otp -patchelf|NixOS/patchelf -pathpicker|facebook/PathPicker -pdf2svg|db9052/pdf2svg -php-apcu|krakjoe/apcu -php-zephir-parser|zephir-lang/php-zephir-parser -pick|calleerlandsson/pick -profanity|profanity-im/profanity -proxmark3|RfidResearchGroup/proxmark3 -proxychains-ng|rofl0r/proxychains-ng -psutils|rrthomas/psutils -ptunnel-ng|lnslbrty/ptunnel-ng -pup|ericchiang/pup -qalc|Qalculate/libqalculate -qpdf|qpdf/qpdf|\d+\.\d+\.\d+ -radare2|radare/radare2 -rclone|rclone/rclone -rdiff-backup|rdiff-backup/rdiff-backup -recode|rrthomas/recode -redir|troglobit/redir -restic|restic/restic -rgbds|gbdev/rgbds -rhash|rhash/RHash -ripgrep|BurntSushi/ripgrep -ripgrep-all|phiresky/ripgrep-all -rsnapshot|rsnapshot/rsnapshot -rtorrent|rakshasa/rtorrent -rush|shenwei356/rush -rustscan|RustScan/RustScan -sc-im|andmarti1424/sc-im -screenfetch|KittyKatt/screenFetch -scrub|chaos/scrub -seafile-client|haiwen/seafile -shc|neurobin/shc -shell2http|msoap/shell2http -shellharden|anordal/shellharden -shellinabox|shellinabox/shellinabox -shfmt|mvdan/sh -shiori|go-shiori/shiori -silicon|Aloxaf/silicon -sl|mtoyoda/sl -sleuthkit|sleuthkit/sleuthkit|\d+\.\d+\.\d+ -slides|maaslalani/slides -solidity|ethereum/solidity -sslscan|rbsec/sslscan -stag|seenaburns/stag -starship|starship/starship -stdman|jeaye/stdman -strace|strace/strace -syncthing|syncthing/syncthing -taskwarrior|GothenburgBitFactory/taskwarrior -teckit|silnrsi/teckit -tergent|aeolwyr/tergent -termimage|nabijaczleweli/termimage -tesseract|tesseract-ocr/tesseract -tidy|htacg/tidy-html5 -tig|jonas/tig|\d+\.\d+\.\d+ -tin-summer|vmchale/tin-summer -tinyproxy|tinyproxy/tinyproxy -tmate|tmate-io/tmate -tmux|tmux/tmux -tokei|XAMPPRocky/tokei -toxic|JFreegman/toxic -tracepath|iputils/iputils|\d{8} -translate-shell|soimort/translate-shell -ttyd|tsl0922/ttyd -tweego|tmedwards/tweego -unicorn|unicorn-engine/unicorn -unshield|twogood/unshield -usbmuxd|libimobiledevice/usbmuxd -utf8cpp|nemtrif/utfcpp -utf8proc|JuliaLang/utf8proc -utfdecode|fornwall/utfdecode -vegeta|tsenart/vegeta -vifm|vifm/vifm -virustotal-cli|VirusTotal/vt-cli -vis|martanne/vis -vtutils|fornwall/vtutils -woff2|google/woff2 -wordgrinder|davidgiven/wordgrinder -wren|wren-lang/wren-cli -wuzz|asciimoo/wuzz -xmake|xmake-io/xmake -xxhash|Cyan4973/xxHash -youtubedr|kkdai/youtube -z3|Z3Prover/z3|\d+\.\d+\.\d+ -zbar|mchehab/zbar -zoxide|ajeetdsouza/zoxide -zstd|facebook/zstd