From 63016ca899dff6df94fe7e1814f37f72602bb3e4 Mon Sep 17 00:00:00 2001 From: Twaik Yont <9674930+twaik@users.noreply.github.com> Date: Mon, 12 May 2025 15:20:47 +0300 Subject: [PATCH 1/3] termux_step_get_dependencies: fill array with dependencies data Main loop of the function will be splitted in more pieces so we should make sure all data will be fetched only once and accessible outside the first loop. --- scripts/build/termux_step_get_dependencies.sh | 92 ++++++++++--------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/scripts/build/termux_step_get_dependencies.sh b/scripts/build/termux_step_get_dependencies.sh index f9b601c70d0ace..404df947198cba 100644 --- a/scripts/build/termux_step_get_dependencies.sh +++ b/scripts/build/termux_step_get_dependencies.sh @@ -2,27 +2,35 @@ termux_step_get_dependencies() { [[ "$TERMUX_SKIP_DEPCHECK" == "true" || "$TERMUX_PKG_METAPACKAGE" == "true" ]] && return 0 [[ "$TERMUX_INSTALL_DEPS" == "true" ]] && termux_download_repo_file # Download repo files + local -i idx=0 + local -A deps=() + while read -r PKG PKG_DIR; do + local name="pkg_$(( idx++ ))" + local -A "$name" + declare -n dep="$name" + deps["${PKG_DIR}"]="$name" + dep=([name]="${PKG}" [versioned]="$PKG" [dir]="${PKG_DIR}" [cyclic]="false" [download]="false" [build]="false") + # Checking for duplicate dependencies - local cyclic_dependence="false" - if termux_check_package_in_building_packages_list "$PKG_DIR"; then - echo "A circular dependency was found on '$PKG', the old version of the package will be installed to resolve the conflict" - cyclic_dependence="true" + if termux_check_package_in_building_packages_list "${dep[dir]}"; then + echo "A circular dependency was found on '${dep[name]}', the old version of the package will be installed to resolve the conflict" + dep[cyclic]="true" [[ "$TERMUX_INSTALL_DEPS" == "false" ]] && termux_download_repo_file fi - [[ -z "$PKG" ]] && continue - [[ "$PKG" == "ERROR" ]] && termux_error_exit "Obtaining buildorder failed" + [[ -z "${dep[name]}" ]] && continue + [[ "${dep[name]}" == "ERROR" ]] && termux_error_exit "Obtaining buildorder failed" - if [[ "$TERMUX_INSTALL_DEPS" == "true" || "$cyclic_dependence" = "true" ]]; then - [[ "$PKG" == "ndk-sysroot" ]] && continue # llvm doesn't build if ndk-sysroot is installed: - read -r DEP_ARCH DEP_VERSION DEP_VERSION_PAC DEP_ON_DEVICE_NOT_SUPPORTED < <(termux_extract_dep_info "${PKG}" "${PKG_DIR}") - local pkg_versioned="$PKG" build_dependency="false" force_build_dependency="$TERMUX_FORCE_BUILD_DEPENDENCIES" - [[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && pkg_versioned+="@$DEP_VERSION" - if [[ "$cyclic_dependence" == "false" ]]; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Downloading dependency $pkg_versioned if necessary..." - if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" && "$DEP_ON_DEVICE_NOT_SUPPORTED" == "true" ]]; then - echo "Building dependency $PKG on device is not supported. It will be downloaded..." + if [[ "$TERMUX_INSTALL_DEPS" == "true" || "${dep[cyclic]}" = "true" ]]; then + [[ "${dep[name]}" == "ndk-sysroot" ]] && continue # llvm doesn't build if ndk-sysroot is installed: + read -r dep[arch] dep[version] dep[version_pac] dep[on_device_not_supported] < <(termux_extract_dep_info "${dep[name]}" "${dep[dir]}") + local force_build_dependency="$TERMUX_FORCE_BUILD_DEPENDENCIES" + [[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && dep[versioned]+="@${dep[version]}" + if [[ "${dep[cyclic]}" == "false" ]]; then + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Downloading dependency ${dep[versioned]} if necessary..." + if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" && "${dep[on_device_not_supported]}" == "true" ]]; then + echo "Building dependency '${dep[name]}' on device is not supported. It will be downloaded..." force_build_dependency="false" fi else @@ -30,53 +38,53 @@ termux_step_get_dependencies() { fi if [[ "$force_build_dependency" = "true" ]]; then termux_force_check_package_dependency && continue || : - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency $PKG instead of downloading due to -I flag..." - build_dependency="true" + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency ${dep[name]} instead of downloading due to -I flag..." + dep[build]="true" else - if termux_package__is_package_version_built "$PKG" "$DEP_VERSION"; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency $pkg_versioned" + if termux_package__is_package_version_built "${dep[name]}" "${dep[version]}"; then + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency ${dep[versioned]}" continue fi - if ! TERMUX_WITHOUT_DEPVERSION_BINDING="$([[ "${cyclic_dependence}" == "true" ]] && echo "true" || echo "${TERMUX_WITHOUT_DEPVERSION_BINDING}")" termux_download_deb_pac $PKG $DEP_ARCH $DEP_VERSION $DEP_VERSION_PAC; then - [[ "$cyclic_dependence" == "true" || ( "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" ) ]] \ - && termux_error_exit "Download of $PKG$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" && "${cyclic_dependence}" == "false" ]] && echo "@$DEP_VERSION") from $TERMUX_REPO_URL failed" - echo "Download of $pkg_versioned from $TERMUX_REPO_URL failed, building instead" - build_dependency="true" + if ! TERMUX_WITHOUT_DEPVERSION_BINDING="$([[ "${dep[cyclic]}" == "true" ]] && echo "true" || echo "${TERMUX_WITHOUT_DEPVERSION_BINDING}")" termux_download_deb_pac "${dep[name]}" "${dep[arch]}" "${dep[version]}" "${dep[version_pac]}"; then + [[ "${dep[cyclic]}" == "true" || ( "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" ) ]] \ + && termux_error_exit "Download of ${dep[name]}$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" && "${dep[cyclic]}" == "false" ]] && echo "@${dep[version]}") from $TERMUX_REPO_URL failed" + echo "Download of ${dep[versioned]} from $TERMUX_REPO_URL failed, building instead" + dep[build]="true" fi fi - if [[ "$cyclic_dependence" == "false" ]]; then - [[ "$build_dependency" == "true" ]] && termux_run_build-package && continue - termux_add_package_to_built_packages_list "$PKG" + if [[ "${dep[cyclic]}" == "false" ]]; then + [[ "${dep[build]}" == "true" ]] && termux_run_build-package && continue + termux_add_package_to_built_packages_list "${dep[name]}" fi if [[ "$TERMUX_ON_DEVICE_BUILD" == "false" ]]; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "extracting $PKG to $TERMUX_COMMON_CACHEDIR-$DEP_ARCH..." + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "extracting ${dep[name]} to $TERMUX_COMMON_CACHEDIR-${dep[arch]}..." ( - cd "$TERMUX_COMMON_CACHEDIR-$DEP_ARCH" + cd "$TERMUX_COMMON_CACHEDIR-${dep[arch]}" if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" ]]; then # Ignore topdir `.`, to avoid possible permission errors from tar - ar p "${PKG}_${DEP_VERSION}_${DEP_ARCH}.deb" "data.tar.xz" | \ + ar p "${dep[name]}_${dep[version]}_${dep[arch]}.deb" "data.tar.xz" | \ tar xJ --no-overwrite-dir --transform='s#^.$#data#' -C / elif [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" ]]; then - tar -xJf "${PKG}-${DEP_VERSION_PAC}-${DEP_ARCH}.pkg.tar.xz" \ + tar -xJf "${dep[name]}-${dep[version_pac]}-${dep[arch]}.pkg.tar.xz" \ --anchored --exclude=.{BUILDINFO,PKGINFO,MTREE,INSTALL} \ --force-local --no-overwrite-dir -C / fi ) fi mkdir -p "$TERMUX_BUILT_PACKAGES_DIRECTORY" - if [[ "$cyclic_dependence" == "false" && ( "$TERMUX_WITHOUT_DEPVERSION_BINDING" == "false" || "$TERMUX_ON_DEVICE_BUILD" == "false" ) ]]; then - echo "$DEP_VERSION" > "$TERMUX_BUILT_PACKAGES_DIRECTORY/$PKG" + if [[ "${dep[cyclic]}" == "false" && ( "$TERMUX_WITHOUT_DEPVERSION_BINDING" == "false" || "$TERMUX_ON_DEVICE_BUILD" == "false" ) ]]; then + echo "${dep[version]}" > "$TERMUX_BUILT_PACKAGES_DIRECTORY/${dep[name]}" fi else # Build dependencies # Built dependencies are put in the default TERMUX_OUTPUT_DIR instead of the specified one if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" ]]; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency $PKG..." - read -r DEP_ARCH DEP_VERSION DEP_VERSION_PAC DEP_ON_DEVICE_NOT_SUPPORTED < <(termux_extract_dep_info $PKG "${PKG_DIR}") - [[ "$TERMUX_ON_DEVICE_BUILD" == "true" && "$DEP_ON_DEVICE_NOT_SUPPORTED" == "true" ]] \ - && termux_error_exit "Building $PKG on device is not supported. Consider passing -I flag to download it instead" + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency ${dep[name]}..." + read -r dep[arch] dep[version] dep[version_pac] dep[on_device_not_supported] < <(termux_extract_dep_info ${dep[name]} "${dep[dir]}") + [[ "$TERMUX_ON_DEVICE_BUILD" == "true" && "${dep[on_device_not_supported]}" == "true" ]] \ + && termux_error_exit "Building ${dep[name]} on device is not supported. Consider passing -I flag to download it instead" termux_force_check_package_dependency && continue else - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Building dependency $PKG if necessary..." + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Building dependency ${dep[name]} if necessary..." fi termux_run_build-package fi @@ -84,8 +92,8 @@ termux_step_get_dependencies() { } termux_force_check_package_dependency() { - if termux_check_package_in_built_packages_list "$PKG" && termux_package__is_package_version_built "$PKG" "$DEP_VERSION"; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency $PKG$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && echo "@$DEP_VERSION")" + if termux_check_package_in_built_packages_list "${dep[name]}" && termux_package__is_package_version_built "${dep[name]}" "${dep[version]}"; then + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency ${dep[name]}$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && echo "@${dep[version]}")" return 0 fi return 1 @@ -97,7 +105,7 @@ termux_run_build-package() { set_library="$TERMUX_PACKAGE_LIBRARY -L" else set_library="bionic" - if termux_package__is_package_name_have_glibc_prefix "$PKG"; then + if termux_package__is_package_name_have_glibc_prefix "${dep[name]}"; then set_library="glibc" fi fi @@ -106,7 +114,7 @@ termux_run_build-package() { $([[ "${TERMUX_FORCE_BUILD}" == "true" && "${TERMUX_FORCE_BUILD_DEPENDENCIES}" == "true" ]] && echo "-F") \ $([[ "${TERMUX_PKGS__BUILD__RM_ALL_PKG_BUILD_DEPENDENT_DIRS}" == "true" ]] && echo "-r") \ $([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" = "true" ]] && echo "-w") \ - --format $TERMUX_PACKAGE_FORMAT --library $set_library "${PKG_DIR}" + --format $TERMUX_PACKAGE_FORMAT --library $set_library "${dep[dir]}" } termux_download_repo_file() { From 55d1e83e683e5b75656aa1ab8b12da06c3c931c7 Mon Sep 17 00:00:00 2001 From: Twaik Yont <9674930+twaik@users.noreply.github.com> Date: Mon, 12 May 2025 21:07:48 +0300 Subject: [PATCH 2/3] scripts: separated termux_buffered_output to new function in new file --- build-package.sh | 4 ++++ scripts/build/termux_buffered_output.sh | 24 ++++++++++++++++++++++++ scripts/build/termux_get_repo_files.sh | 22 +--------------------- 3 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 scripts/build/termux_buffered_output.sh diff --git a/build-package.sh b/build-package.sh index 98fea42fa04458..53f09706d61dc0 100755 --- a/build-package.sh +++ b/build-package.sh @@ -193,6 +193,10 @@ source "$TERMUX_SCRIPTDIR/scripts/build/termux_extract_dep_info.sh" # shellcheck source=scripts/build/termux_download_deb_pac.sh source "$TERMUX_SCRIPTDIR/scripts/build/termux_download_deb_pac.sh" +# Function that makes output of parallel jobs consistent and not garbled. +# shellcheck source=scripts/build/termux_buffered_output.sh +source "$TERMUX_SCRIPTDIR/scripts/build/termux_buffered_output.sh" + # Script to download InRelease, verify it's signature and then download Packages.xz by hash # shellcheck source=scripts/build/termux_get_repo_files.sh source "$TERMUX_SCRIPTDIR/scripts/build/termux_get_repo_files.sh" diff --git a/scripts/build/termux_buffered_output.sh b/scripts/build/termux_buffered_output.sh new file mode 100644 index 00000000000000..c6a58c8fa1968a --- /dev/null +++ b/scripts/build/termux_buffered_output.sh @@ -0,0 +1,24 @@ +termux_buffered_output() { + ( # Do everything in a subshell to avoid any kind of mess. + TAG="$1" + set +e + # curl progress meter uses carriage return instead of newlines, fixing it + sed -u 's/\r/\n/g' | while :; do + local buffer=() + # One second buffer to prevent mixing lines and make output consistent. + sleep 1; + while :; do + # read with 0 timeout does not read any data so giving minimal timeout + IFS='' read -t 0.001 -r line; rc=$? + # append job name to the start for tracking multiple jobs + [[ $rc == 0 ]] && buffer+=( "[$TAG]: $line" ) + # Probably EOF or timeout + [[ $rc == 1 || $rc -ge 128 ]] && break + done + + # prevent output garbling by using scripts directory for locking + [[ "${#buffer[@]}" -ge 1 ]] && flock --no-fork . printf "%s\n" "${buffer[@]}" + [[ $rc == 1 ]] && break # exit on EOF + done + ) +} diff --git a/scripts/build/termux_get_repo_files.sh b/scripts/build/termux_get_repo_files.sh index 0d0f1b5d817114..219d8e7985668c 100644 --- a/scripts/build/termux_get_repo_files.sh +++ b/scripts/build/termux_get_repo_files.sh @@ -56,27 +56,7 @@ termux_get_repo_files() { fi done termux_error_exit "Failed to download package repository metadata. Try to build without -i/-I option." - ) 2>&1 | ( - set +e - # curl progress meter uses carriage return instead of newlines, fixing it - sed -u 's/\r/\n/g' | while :; do - local buffer=() - # Half second buffer to prevent mixing lines and make output consistent. - sleep 0.5; - while :; do - # read with 0 timeout does not read any data so giving minimal timeout - IFS='' read -t 0.001 -r line; rc=$? - # append job name to the start for tracking multiple jobs - [[ $rc == 0 ]] && buffer+=( "[$TERMUX_REPO_NAME]: $line" ) - # Probably EOF or timeout - [[ $rc == 1 || $rc -ge 128 ]] && break - done - - # prevent output garbling by using stdout as a lock file - [[ "${#buffer[@]}" -ge 1 ]] && flock --no-fork . printf "%s\n" "${buffer[@]}" - [[ $rc == 1 ]] && break # exit on EOF - done - ) & + ) 2>&1 | termux_buffered_output "$TERMUX_REPO_NAME" & pids+=( $! ) done From 08319ef4175b237a5f867a5b2b15dda2a7f1fef7 Mon Sep 17 00:00:00 2001 From: Twaik Yont <9674930+twaik@users.noreply.github.com> Date: Mon, 12 May 2025 21:23:00 +0300 Subject: [PATCH 3/3] scripts(termux_step_get_dependencies): parallelizing deps downloading --- build-package.sh | 5 + scripts/build/termux_step_get_dependencies.sh | 92 +++++++++++++------ scripts/build/termux_step_setup_variables.sh | 1 + 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/build-package.sh b/build-package.sh index 53f09706d61dc0..36a303169c7362 100755 --- a/build-package.sh +++ b/build-package.sh @@ -504,6 +504,7 @@ _show_usage() { echo " -o Specify directory where to put built packages. Default: output/." echo " --format Specify package output format (debian, pacman)." echo " --library Specify library of package (bionic, glibc)." + echo " --disable-dependency-download-parallelizing disable dependency downloading parallelizing" exit 1 } @@ -536,6 +537,9 @@ while (($# >= 1)); do termux_error_exit "./build-package.sh: option '--library' requires an argument" fi ;; + --disable-dependency-download-parallelizing) + TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING="false" + ;; -a) if [ $# -ge 2 ]; then shift 1 @@ -662,6 +666,7 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do $(test "${TERMUX_WITHOUT_DEPVERSION_BINDING:-}" = "true" && echo "-w") \ $(test -n "${TERMUX_PACKAGE_FORMAT:-}" && echo "--format $TERMUX_PACKAGE_FORMAT") \ $(test -n "${TERMUX_PACKAGE_LIBRARY:-}" && echo "--library $TERMUX_PACKAGE_LIBRARY") \ + $(test "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING-true}" = "false" ]] && echo "--disable-dependency-download-parallelizing") \ "${PACKAGE_LIST[i]}" done exit diff --git a/scripts/build/termux_step_get_dependencies.sh b/scripts/build/termux_step_get_dependencies.sh index 404df947198cba..e354ffd79b33e8 100644 --- a/scripts/build/termux_step_get_dependencies.sh +++ b/scripts/build/termux_step_get_dependencies.sh @@ -1,16 +1,17 @@ termux_step_get_dependencies() { + local DOWNLOAD_FAILED_EXIT_CODE=69 [[ "$TERMUX_SKIP_DEPCHECK" == "true" || "$TERMUX_PKG_METAPACKAGE" == "true" ]] && return 0 [[ "$TERMUX_INSTALL_DEPS" == "true" ]] && termux_download_repo_file # Download repo files - local -i idx=0 - local -A deps=() + local pkg_idx=0 rc=0 + local -a deps=() while read -r PKG PKG_DIR; do - local name="pkg_$(( idx++ ))" + local name="pkg_$(( pkg_idx++ ))" local -A "$name" declare -n dep="$name" - deps["${PKG_DIR}"]="$name" dep=([name]="${PKG}" [versioned]="$PKG" [dir]="${PKG_DIR}" [cyclic]="false" [download]="false" [build]="false") + read -r dep[arch] dep[version] dep[version_pac] dep[on_device_not_supported] < <(termux_extract_dep_info "${dep[name]}" "${dep[dir]}") # Checking for duplicate dependencies if termux_check_package_in_building_packages_list "${dep[dir]}"; then @@ -24,7 +25,6 @@ termux_step_get_dependencies() { if [[ "$TERMUX_INSTALL_DEPS" == "true" || "${dep[cyclic]}" = "true" ]]; then [[ "${dep[name]}" == "ndk-sysroot" ]] && continue # llvm doesn't build if ndk-sysroot is installed: - read -r dep[arch] dep[version] dep[version_pac] dep[on_device_not_supported] < <(termux_extract_dep_info "${dep[name]}" "${dep[dir]}") local force_build_dependency="$TERMUX_FORCE_BUILD_DEPENDENCIES" [[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && dep[versioned]+="@${dep[version]}" if [[ "${dep[cyclic]}" == "false" ]]; then @@ -45,32 +45,72 @@ termux_step_get_dependencies() { [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency ${dep[versioned]}" continue fi - if ! TERMUX_WITHOUT_DEPVERSION_BINDING="$([[ "${dep[cyclic]}" == "true" ]] && echo "true" || echo "${TERMUX_WITHOUT_DEPVERSION_BINDING}")" termux_download_deb_pac "${dep[name]}" "${dep[arch]}" "${dep[version]}" "${dep[version_pac]}"; then - [[ "${dep[cyclic]}" == "true" || ( "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" ) ]] \ - && termux_error_exit "Download of ${dep[name]}$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" && "${dep[cyclic]}" == "false" ]] && echo "@${dep[version]}") from $TERMUX_REPO_URL failed" - echo "Download of ${dep[versioned]} from $TERMUX_REPO_URL failed, building instead" + fi + + if [[ "${dep[build]}" != "true" ]]; then + dep[download]="true" + ( + if ! TERMUX_WITHOUT_DEPVERSION_BINDING="$([[ "${dep[cyclic]}" == "true" ]] && echo "true" || echo "${TERMUX_WITHOUT_DEPVERSION_BINDING}")" termux_download_deb_pac "${dep[name]}" "${dep[arch]}" "${dep[version]}" "${dep[version_pac]}"; then + if [[ "${dep[cyclic]}" == "true" || ( "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" ) ]]; then + echo "Download of ${dep[name]}$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" && "${dep[cyclic]}" == "false" ]] && echo "@${dep[version]}") from $TERMUX_REPO_URL failed" >&2 + exit "${DOWNLOAD_FAILED_EXIT_CODE}" + fi + echo "Download of ${dep[versioned]} from $TERMUX_REPO_URL failed, building instead" + fi + ) 2>&1 | termux_buffered_output "${dep[dir]}" & + # for the case if user explicitly disabled dependency downloading + [[ "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING-true}" == "false" ]] && ! wait -n && (( $? == DOWNLOAD_FAILED_EXIT_CODE )) && exit 1 + fi + fi + + # Enqueue the dependency for deferred processing; resume work once all background downloads complete + deps+=("$name") + done < <(./scripts/buildorder.py $([[ "${TERMUX_INSTALL_DEPS}" == "true" ]] && echo "-i") "$TERMUX_PKG_BUILDER_DIR" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR") + + while [[ -n "$(jobs -p)" ]]; do + if ! wait -n && (( $? == DOWNLOAD_FAILED_EXIT_CODE )); then + # One of jobs exited with fatal error, we should return error too and kill all background jobs + if [[ "${CI-false}" != "true" ]]; then + # In the case of local building we'd want to finish these downloads and reuse debs later + wait + else + # In the case of CI it has no sense and we'd want to finish earlier + echo "FATAL: one of downloads failed, exiting" + kill $(jobs -p) 2>/dev/null + fi + exit 1 + fi + done + + for name in "${deps[@]}"; do + declare -n dep="$name" + if [[ "$TERMUX_INSTALL_DEPS" == "true" || "${dep[cyclic]}" = "true" ]]; then + # We can not explicitly check obtain PID from `wait` or `wait -n` so we will simply check if downloading failed. + if [[ "${dep[download]}" == "true" ]]; then + if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" && ! -f "$TERMUX_COMMON_CACHEDIR-${dep[arch]}/${dep[name]}_${dep[version]}_${dep[arch]}.deb" ]] \ + || [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" && ! -f "$TERMUX_COMMON_CACHEDIR-${dep[arch]}/${dep[name]}-${dep[version_pac]}-${dep[arch]}.pkg.tar.xz" ]]; then dep[build]="true" fi fi + if [[ "${dep[cyclic]}" == "false" ]]; then - [[ "${dep[build]}" == "true" ]] && termux_run_build-package && continue + [[ "${dep[build]}" == "true" ]] && { termux_run_build-package || exit $?; } && continue termux_add_package_to_built_packages_list "${dep[name]}" fi - if [[ "$TERMUX_ON_DEVICE_BUILD" == "false" ]]; then + + [[ "$TERMUX_ON_DEVICE_BUILD" == "false" ]] && ( [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "extracting ${dep[name]} to $TERMUX_COMMON_CACHEDIR-${dep[arch]}..." - ( - cd "$TERMUX_COMMON_CACHEDIR-${dep[arch]}" - if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" ]]; then - # Ignore topdir `.`, to avoid possible permission errors from tar - ar p "${dep[name]}_${dep[version]}_${dep[arch]}.deb" "data.tar.xz" | \ - tar xJ --no-overwrite-dir --transform='s#^.$#data#' -C / - elif [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" ]]; then - tar -xJf "${dep[name]}-${dep[version_pac]}-${dep[arch]}.pkg.tar.xz" \ - --anchored --exclude=.{BUILDINFO,PKGINFO,MTREE,INSTALL} \ - --force-local --no-overwrite-dir -C / - fi - ) - fi + cd "$TERMUX_COMMON_CACHEDIR-${dep[arch]}" + if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" ]]; then + # Ignore topdir `.`, to avoid possible permission errors from tar + ar p "${dep[name]}_${dep[version]}_${dep[arch]}.deb" "data.tar.xz" | \ + tar xJ --no-overwrite-dir --transform='s#^.$#data#' -C / + elif [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" ]]; then + tar -xJf "${dep[name]}-${dep[version_pac]}-${dep[arch]}.pkg.tar.xz" \ + --anchored --exclude=.{BUILDINFO,PKGINFO,MTREE,INSTALL} \ + --force-local --no-overwrite-dir -C / + fi + ) mkdir -p "$TERMUX_BUILT_PACKAGES_DIRECTORY" if [[ "${dep[cyclic]}" == "false" && ( "$TERMUX_WITHOUT_DEPVERSION_BINDING" == "false" || "$TERMUX_ON_DEVICE_BUILD" == "false" ) ]]; then echo "${dep[version]}" > "$TERMUX_BUILT_PACKAGES_DIRECTORY/${dep[name]}" @@ -79,7 +119,6 @@ termux_step_get_dependencies() { # Built dependencies are put in the default TERMUX_OUTPUT_DIR instead of the specified one if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" ]]; then [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency ${dep[name]}..." - read -r dep[arch] dep[version] dep[version_pac] dep[on_device_not_supported] < <(termux_extract_dep_info ${dep[name]} "${dep[dir]}") [[ "$TERMUX_ON_DEVICE_BUILD" == "true" && "${dep[on_device_not_supported]}" == "true" ]] \ && termux_error_exit "Building ${dep[name]} on device is not supported. Consider passing -I flag to download it instead" termux_force_check_package_dependency && continue @@ -88,7 +127,7 @@ termux_step_get_dependencies() { fi termux_run_build-package fi - done < <(./scripts/buildorder.py $([[ "${TERMUX_INSTALL_DEPS}" == "true" ]] && echo "-i") "$TERMUX_PKG_BUILDER_DIR" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR") + done } termux_force_check_package_dependency() { @@ -114,6 +153,7 @@ termux_run_build-package() { $([[ "${TERMUX_FORCE_BUILD}" == "true" && "${TERMUX_FORCE_BUILD_DEPENDENCIES}" == "true" ]] && echo "-F") \ $([[ "${TERMUX_PKGS__BUILD__RM_ALL_PKG_BUILD_DEPENDENT_DIRS}" == "true" ]] && echo "-r") \ $([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" = "true" ]] && echo "-w") \ + $([[ "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING-true}" = "false" ]] && echo "--disable-dependency-download-parallelizing") \ --format $TERMUX_PACKAGE_FORMAT --library $set_library "${dep[dir]}" } diff --git a/scripts/build/termux_step_setup_variables.sh b/scripts/build/termux_step_setup_variables.sh index 0ac9dbffa6982c..9976eff60de064 100644 --- a/scripts/build/termux_step_setup_variables.sh +++ b/scripts/build/termux_step_setup_variables.sh @@ -15,6 +15,7 @@ termux_step_setup_variables() { : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_GLOBAL_LIBRARY:="false"}" : "${TERMUX_TOPDIR:="$HOME/.termux-build"}" + : "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING:=true}" : "${TERMUX_PACMAN_PACKAGE_COMPRESSION:="xz"}" if [ -z "${TERMUX_PACKAGE_FORMAT-}" ]; then