From 7553f15030bcfba44b3dea09391ea2884276959a Mon Sep 17 00:00:00 2001 From: Twaik Yont <9674930+twaik@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:39:16 +0200 Subject: [PATCH 1/2] fix(ci): fixing package cleanup to work with toybox df. --- .github/workflows/packages.yml | 4 ++-- build-package.sh | 5 ++++- scripts/build/termux_step_cleanup_packages.sh | 12 ++++++++---- .../utils/termux_pkg_upgrade_version.sh | 18 ++++++++++++++---- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2617e8ce645462..7520dcf60abd66 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -228,9 +228,9 @@ jobs: done if [ "$DOCKER_BUILD" == 'false' ]; then - NDK=$ANDROID_NDK ANDROID_HOME=$ANDROID_SDK_ROOT ./build-package.sh -I -a ${{ matrix.target_arch }} $packages + NDK=$ANDROID_NDK ANDROID_HOME=$ANDROID_SDK_ROOT ./build-package.sh -I -a -C ${{ matrix.target_arch }} $packages elif [ -n "$packages" ]; then - ./scripts/run-docker.sh ./build-package.sh -I -a ${{ matrix.target_arch }} $packages + ./scripts/run-docker.sh ./build-package.sh -I -a -C ${{ matrix.target_arch }} $packages fi - name: Generate build artifacts diff --git a/build-package.sh b/build-package.sh index 3e619817f41d67..987e819ba7a72f 100755 --- a/build-package.sh +++ b/build-package.sh @@ -423,6 +423,8 @@ _show_usage() { echo "Available options:" [ "$TERMUX_ON_DEVICE_BUILD" = "false" ] && echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." + echo " -c Continue building." + echo " -C Cleanup already built packages on low disk space." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." echo " -F Force build even if package and its dependencies have already been built." @@ -521,6 +523,7 @@ while (($# >= 1)); do fi ;; -c) TERMUX_CONTINUE_BUILD=true;; + -C) TERMUX_CLEANUP_BUILT_PACKAGES_ON_LOW_DISK_SPACE=true;; -*) termux_error_exit "./build-package.sh: illegal option '$1'";; *) PACKAGE_LIST+=("$1");; esac @@ -588,7 +591,7 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do ${TERMUX_FORCE_BUILD+-f} ${TERMUX_INSTALL_DEPS+-i} ${TERMUX_IS_DISABLED+-D} \ ${TERMUX_DEBUG_BUILD+-d} ${TERMUX_OUTPUT_DIR+-o $TERMUX_OUTPUT_DIR} \ ${TERMUX_FORCE_BUILD_DEPENDENCIES+-F} ${TERMUX_GLOBAL_LIBRARY+-L} \ - ${TERMUX_WITHOUT_DEPVERSION_BINDING+-w} \ + ${TERMUX_WITHOUT_DEPVERSION_BINDING+-w} ${TERMUX_CLEANUP_BUILT_PACKAGES_ON_LOW_DISK_SPACE+-C} \ --format ${TERMUX_PACKAGE_FORMAT:=debian} \ --library ${TERMUX_PACKAGE_LIBRARY:=bionic} "${PACKAGE_LIST[i]}" done diff --git a/scripts/build/termux_step_cleanup_packages.sh b/scripts/build/termux_step_cleanup_packages.sh index d17aedbf53e06c..04895a2b7d7225 100644 --- a/scripts/build/termux_step_cleanup_packages.sh +++ b/scripts/build/termux_step_cleanup_packages.sh @@ -1,11 +1,15 @@ termux_step_cleanup_packages() { + [[ "${TERMUX_CLEANUP_BUILT_PACKAGES_ON_LOW_DISK_SPACE:=false}" == "true" ]] || return 0 [[ -d "$TERMUX_TOPDIR" ]] || return 0 - # Extract available disk space in GB - local AVAILABLE=`df -B $((1024**3)) --output=avail "$TERMUX_TOPDIR" | tail -1` + # How much space is considered to be enough + local CLEANUP_THRESHOLD="$(( 5 * 1024 ** 3 ))" # 5 GiB - # No need to cleanup if there is enough disk space (more than 5 GB) - [ "$AVAILABLE" -lt 5 ] || return 0 + # Extract available disk space in bytes + local AVAILABLE="$(df "$TERMUX_TOPDIR" | awk 'NR==2 {print $4 * 1024}')" + + # No need to cleanup if there is enough disk space + (( AVAILABLE <= CLEANUP_THRESHOLD )) || return 0 local TERMUX_PACKAGES_DIRECTORIES="$(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json)" diff --git a/scripts/updates/utils/termux_pkg_upgrade_version.sh b/scripts/updates/utils/termux_pkg_upgrade_version.sh index 6a02acf2c73b14..b9af9e66f7f7bd 100755 --- a/scripts/updates/utils/termux_pkg_upgrade_version.sh +++ b/scripts/updates/utils/termux_pkg_upgrade_version.sh @@ -96,8 +96,18 @@ termux_pkg_upgrade_version() { fi done - # Get available space (df outputs sizes in 1024-byte blocks) - local space_available="$(df -P "/var/lib/docker" | awk 'NR==2 {print $4}')" + local force_cleanup="false" + + if [[ -d "/var/lib/docker" ]]; then + # Get available space in bytes + local space_available="$(df "/var/lib/docker" | awk 'NR==2 { print $4 * 1024 }')" + local cleanup_threshold="$(( 5 * 1024 ** 3 ))" # 3 GiB + + if (( space_available <= cleanup_threshold )); then + force_cleanup="true" + fi + fi + local big_package=false while IFS= read -r p; do @@ -111,7 +121,7 @@ termux_pkg_upgrade_version() { "${TERMUX_SCRIPTDIR}/scripts/run-docker.sh" ./clean.sh fi - if ! "${TERMUX_SCRIPTDIR}/scripts/run-docker.sh" ./build-package.sh -a "${TERMUX_ARCH}" -i "${TERMUX_PKG_NAME}"; then + if ! "${TERMUX_SCRIPTDIR}/scripts/run-docker.sh" ./build-package.sh -C -a "${TERMUX_ARCH}" -i "${TERMUX_PKG_NAME}"; then if [[ "${big_package}" == "true" ]]; then "${TERMUX_SCRIPTDIR}/scripts/run-docker.sh" ./clean.sh fi @@ -119,7 +129,7 @@ termux_pkg_upgrade_version() { termux_error_exit "ERROR: failed to build." fi - if [[ "${big_package}" == "true" ]] || (( space_available <= 2 * 1024 ** 2 )); then + if [[ "${big_package}" == "true" ]] || [[ "${force_cleanup}" == "true" ]]; then "${TERMUX_SCRIPTDIR}/scripts/run-docker.sh" ./clean.sh fi From dc6ea3a9f9a1d681d239f772d9027c2d9e456913 Mon Sep 17 00:00:00 2001 From: Twaik Yont <9674930+twaik@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:39:16 +0200 Subject: [PATCH 2/2] fix(ci): fixing package cleanup to work with toybox df. --- scripts/build/termux_step_cleanup_packages.sh | 19 +++++++++++-------- scripts/properties.sh | 1 + .../utils/termux_pkg_upgrade_version.sh | 6 +++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/build/termux_step_cleanup_packages.sh b/scripts/build/termux_step_cleanup_packages.sh index 04895a2b7d7225..cf93d1aac28478 100644 --- a/scripts/build/termux_step_cleanup_packages.sh +++ b/scripts/build/termux_step_cleanup_packages.sh @@ -2,24 +2,27 @@ termux_step_cleanup_packages() { [[ "${TERMUX_CLEANUP_BUILT_PACKAGES_ON_LOW_DISK_SPACE:=false}" == "true" ]] || return 0 [[ -d "$TERMUX_TOPDIR" ]] || return 0 - # How much space is considered to be enough - local CLEANUP_THRESHOLD="$(( 5 * 1024 ** 3 ))" # 5 GiB + local AVAILABLE TERMUX_PACKAGES_DIRECTORIES PKGS PKG_REGEX # Extract available disk space in bytes - local AVAILABLE="$(df "$TERMUX_TOPDIR" | awk 'NR==2 {print $4 * 1024}')" + AVAILABLE="$(df "$TERMUX_TOPDIR" | awk 'NR==2 {print $4 * 1024}')" # No need to cleanup if there is enough disk space - (( AVAILABLE <= CLEANUP_THRESHOLD )) || return 0 + (( AVAILABLE <= TERMUX_CLEANUP_BUILT_PACKAGES_THRESHOLD )) || return 0 - local TERMUX_PACKAGES_DIRECTORIES="$(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json)" + TERMUX_PACKAGES_DIRECTORIES="$(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}"/repo.json)" # Build package name regex to be used with `find`, avoiding loops. - local PKG_REGEX="$(find ${TERMUX_PACKAGES_DIRECTORIES} -mindepth 1 -maxdepth 1 -type d -printf '%f|')^$" + PKGS="$(find ${TERMUX_PACKAGES_DIRECTORIES} -mindepth 1 -maxdepth 1 -type d -printf '%f\n')" + [[ -z "$PKGS" ]] && return 0 # Exclude current package from the list. - PKG_REGEX="${PKG_REGEX//$TERMUX_PKG_NAME|/}" + PKGS="$(printf "%s" "$PKGS" | grep -Fxv "$TERMUX_PKG_NAME")" + [[ -z "$PKGS" ]] && return 0 + + PKG_REGEX="$(printf "%s" "$PKGS" | sed -zE 's/[][\.|$(){}?+*^]/\\&/g' | sed -E 's/(.*)/(\1)/g' | sed -zE -e 's/[\n]+/|/g' -e 's/(.*)/(\1)/g')" echo "INFO: cleaning up some disk space for building \"${TERMUX_PKG_NAME}\"." - find "$TERMUX_TOPDIR" -mindepth 1 -maxdepth 1 -type d -regextype posix-extended -regex ".*/($PKG_REGEX)$" -exec rm -rf {} + + (cd "$TERMUX_TOPDIR" && find . -mindepth 1 -maxdepth 1 -type d -regextype posix-extended -regex "^\./$PKG_REGEX$" -exec rm -rf "{}" +) } diff --git a/scripts/properties.sh b/scripts/properties.sh index 69e27486c2fee4..f7fea5ebca4c44 100644 --- a/scripts/properties.sh +++ b/scripts/properties.sh @@ -41,6 +41,7 @@ TERMUX_ETC_PREFIX_DIR_PATH="${TERMUX_PREFIX}/etc" TERMUX_PROFILE_D_PREFIX_DIR_PATH="${TERMUX_ETC_PREFIX_DIR_PATH}/profile.d" TERMUX_CONFIG_PREFIX_DIR_PATH="${TERMUX_ETC_PREFIX_DIR_PATH}/termux" TERMUX_BOOTSTRAP_CONFIG_DIR_PATH="${TERMUX_CONFIG_PREFIX_DIR_PATH}/bootstrap" +TERMUX_CLEANUP_BUILT_PACKAGES_THRESHOLD="$(( 5 * 1024 ** 3 ))" # 5 GiB # Path to CGCT tools CGCT_DEFAULT_PREFIX="/data/data/com.termux/files/usr/glibc" diff --git a/scripts/updates/utils/termux_pkg_upgrade_version.sh b/scripts/updates/utils/termux_pkg_upgrade_version.sh index b9af9e66f7f7bd..d7f532a8d72788 100755 --- a/scripts/updates/utils/termux_pkg_upgrade_version.sh +++ b/scripts/updates/utils/termux_pkg_upgrade_version.sh @@ -97,13 +97,13 @@ termux_pkg_upgrade_version() { done local force_cleanup="false" + local space_available if [[ -d "/var/lib/docker" ]]; then # Get available space in bytes - local space_available="$(df "/var/lib/docker" | awk 'NR==2 { print $4 * 1024 }')" - local cleanup_threshold="$(( 5 * 1024 ** 3 ))" # 3 GiB + space_available="$(df "/var/lib/docker" | awk 'NR==2 { print $4 * 1024 }')" - if (( space_available <= cleanup_threshold )); then + if (( space_available <= TERMUX_CLEANUP_BUILT_PACKAGES_THRESHOLD )); then force_cleanup="true" fi fi