diff --git a/build-package.sh b/build-package.sh index 98fea42fa04458..36a303169c7362 100755 --- a/build-package.sh +++ b/build-package.sh @@ -193,6 +193,10 @@ source "$TERMUX_SCRIPTDIR/scripts/build/termux_extract_dep_info.sh" # shellcheck source=scripts/build/termux_download_deb_pac.sh source "$TERMUX_SCRIPTDIR/scripts/build/termux_download_deb_pac.sh" +# Function that makes output of parallel jobs consistent and not garbled. +# shellcheck source=scripts/build/termux_buffered_output.sh +source "$TERMUX_SCRIPTDIR/scripts/build/termux_buffered_output.sh" + # Script to download InRelease, verify it's signature and then download Packages.xz by hash # shellcheck source=scripts/build/termux_get_repo_files.sh source "$TERMUX_SCRIPTDIR/scripts/build/termux_get_repo_files.sh" @@ -500,6 +504,7 @@ _show_usage() { echo " -o Specify directory where to put built packages. Default: output/." echo " --format Specify package output format (debian, pacman)." echo " --library Specify library of package (bionic, glibc)." + echo " --disable-dependency-download-parallelizing disable dependency downloading parallelizing" exit 1 } @@ -532,6 +537,9 @@ while (($# >= 1)); do termux_error_exit "./build-package.sh: option '--library' requires an argument" fi ;; + --disable-dependency-download-parallelizing) + TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING="false" + ;; -a) if [ $# -ge 2 ]; then shift 1 @@ -658,6 +666,7 @@ for ((i=0; i<${#PACKAGE_LIST[@]}; i++)); do $(test "${TERMUX_WITHOUT_DEPVERSION_BINDING:-}" = "true" && echo "-w") \ $(test -n "${TERMUX_PACKAGE_FORMAT:-}" && echo "--format $TERMUX_PACKAGE_FORMAT") \ $(test -n "${TERMUX_PACKAGE_LIBRARY:-}" && echo "--library $TERMUX_PACKAGE_LIBRARY") \ + $(test "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING-true}" = "false" ]] && echo "--disable-dependency-download-parallelizing") \ "${PACKAGE_LIST[i]}" done exit diff --git a/scripts/build/termux_buffered_output.sh b/scripts/build/termux_buffered_output.sh new file mode 100644 index 00000000000000..c6a58c8fa1968a --- /dev/null +++ b/scripts/build/termux_buffered_output.sh @@ -0,0 +1,24 @@ +termux_buffered_output() { + ( # Do everything in a subshell to avoid any kind of mess. + TAG="$1" + set +e + # curl progress meter uses carriage return instead of newlines, fixing it + sed -u 's/\r/\n/g' | while :; do + local buffer=() + # One second buffer to prevent mixing lines and make output consistent. + sleep 1; + while :; do + # read with 0 timeout does not read any data so giving minimal timeout + IFS='' read -t 0.001 -r line; rc=$? + # append job name to the start for tracking multiple jobs + [[ $rc == 0 ]] && buffer+=( "[$TAG]: $line" ) + # Probably EOF or timeout + [[ $rc == 1 || $rc -ge 128 ]] && break + done + + # prevent output garbling by using scripts directory for locking + [[ "${#buffer[@]}" -ge 1 ]] && flock --no-fork . printf "%s\n" "${buffer[@]}" + [[ $rc == 1 ]] && break # exit on EOF + done + ) +} diff --git a/scripts/build/termux_get_repo_files.sh b/scripts/build/termux_get_repo_files.sh index 0d0f1b5d817114..219d8e7985668c 100644 --- a/scripts/build/termux_get_repo_files.sh +++ b/scripts/build/termux_get_repo_files.sh @@ -56,27 +56,7 @@ termux_get_repo_files() { fi done termux_error_exit "Failed to download package repository metadata. Try to build without -i/-I option." - ) 2>&1 | ( - set +e - # curl progress meter uses carriage return instead of newlines, fixing it - sed -u 's/\r/\n/g' | while :; do - local buffer=() - # Half second buffer to prevent mixing lines and make output consistent. - sleep 0.5; - while :; do - # read with 0 timeout does not read any data so giving minimal timeout - IFS='' read -t 0.001 -r line; rc=$? - # append job name to the start for tracking multiple jobs - [[ $rc == 0 ]] && buffer+=( "[$TERMUX_REPO_NAME]: $line" ) - # Probably EOF or timeout - [[ $rc == 1 || $rc -ge 128 ]] && break - done - - # prevent output garbling by using stdout as a lock file - [[ "${#buffer[@]}" -ge 1 ]] && flock --no-fork . printf "%s\n" "${buffer[@]}" - [[ $rc == 1 ]] && break # exit on EOF - done - ) & + ) 2>&1 | termux_buffered_output "$TERMUX_REPO_NAME" & pids+=( $! ) done diff --git a/scripts/build/termux_step_get_dependencies.sh b/scripts/build/termux_step_get_dependencies.sh index f9b601c70d0ace..e354ffd79b33e8 100644 --- a/scripts/build/termux_step_get_dependencies.sh +++ b/scripts/build/termux_step_get_dependencies.sh @@ -1,28 +1,36 @@ termux_step_get_dependencies() { + local DOWNLOAD_FAILED_EXIT_CODE=69 [[ "$TERMUX_SKIP_DEPCHECK" == "true" || "$TERMUX_PKG_METAPACKAGE" == "true" ]] && return 0 [[ "$TERMUX_INSTALL_DEPS" == "true" ]] && termux_download_repo_file # Download repo files + local pkg_idx=0 rc=0 + local -a deps=() + while read -r PKG PKG_DIR; do + local name="pkg_$(( pkg_idx++ ))" + local -A "$name" + declare -n dep="$name" + dep=([name]="${PKG}" [versioned]="$PKG" [dir]="${PKG_DIR}" [cyclic]="false" [download]="false" [build]="false") + read -r dep[arch] dep[version] dep[version_pac] dep[on_device_not_supported] < <(termux_extract_dep_info "${dep[name]}" "${dep[dir]}") + # Checking for duplicate dependencies - local cyclic_dependence="false" - if termux_check_package_in_building_packages_list "$PKG_DIR"; then - echo "A circular dependency was found on '$PKG', the old version of the package will be installed to resolve the conflict" - cyclic_dependence="true" + if termux_check_package_in_building_packages_list "${dep[dir]}"; then + echo "A circular dependency was found on '${dep[name]}', the old version of the package will be installed to resolve the conflict" + dep[cyclic]="true" [[ "$TERMUX_INSTALL_DEPS" == "false" ]] && termux_download_repo_file fi - [[ -z "$PKG" ]] && continue - [[ "$PKG" == "ERROR" ]] && termux_error_exit "Obtaining buildorder failed" + [[ -z "${dep[name]}" ]] && continue + [[ "${dep[name]}" == "ERROR" ]] && termux_error_exit "Obtaining buildorder failed" - if [[ "$TERMUX_INSTALL_DEPS" == "true" || "$cyclic_dependence" = "true" ]]; then - [[ "$PKG" == "ndk-sysroot" ]] && continue # llvm doesn't build if ndk-sysroot is installed: - read -r DEP_ARCH DEP_VERSION DEP_VERSION_PAC DEP_ON_DEVICE_NOT_SUPPORTED < <(termux_extract_dep_info "${PKG}" "${PKG_DIR}") - local pkg_versioned="$PKG" build_dependency="false" force_build_dependency="$TERMUX_FORCE_BUILD_DEPENDENCIES" - [[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && pkg_versioned+="@$DEP_VERSION" - if [[ "$cyclic_dependence" == "false" ]]; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Downloading dependency $pkg_versioned if necessary..." - if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" && "$DEP_ON_DEVICE_NOT_SUPPORTED" == "true" ]]; then - echo "Building dependency $PKG on device is not supported. It will be downloaded..." + if [[ "$TERMUX_INSTALL_DEPS" == "true" || "${dep[cyclic]}" = "true" ]]; then + [[ "${dep[name]}" == "ndk-sysroot" ]] && continue # llvm doesn't build if ndk-sysroot is installed: + local force_build_dependency="$TERMUX_FORCE_BUILD_DEPENDENCIES" + [[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && dep[versioned]+="@${dep[version]}" + if [[ "${dep[cyclic]}" == "false" ]]; then + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Downloading dependency ${dep[versioned]} if necessary..." + if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" && "${dep[on_device_not_supported]}" == "true" ]]; then + echo "Building dependency '${dep[name]}' on device is not supported. It will be downloaded..." force_build_dependency="false" fi else @@ -30,62 +38,101 @@ termux_step_get_dependencies() { fi if [[ "$force_build_dependency" = "true" ]]; then termux_force_check_package_dependency && continue || : - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency $PKG instead of downloading due to -I flag..." - build_dependency="true" + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency ${dep[name]} instead of downloading due to -I flag..." + dep[build]="true" else - if termux_package__is_package_version_built "$PKG" "$DEP_VERSION"; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency $pkg_versioned" + if termux_package__is_package_version_built "${dep[name]}" "${dep[version]}"; then + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency ${dep[versioned]}" continue fi - if ! TERMUX_WITHOUT_DEPVERSION_BINDING="$([[ "${cyclic_dependence}" == "true" ]] && echo "true" || echo "${TERMUX_WITHOUT_DEPVERSION_BINDING}")" termux_download_deb_pac $PKG $DEP_ARCH $DEP_VERSION $DEP_VERSION_PAC; then - [[ "$cyclic_dependence" == "true" || ( "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" ) ]] \ - && termux_error_exit "Download of $PKG$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" && "${cyclic_dependence}" == "false" ]] && echo "@$DEP_VERSION") from $TERMUX_REPO_URL failed" - echo "Download of $pkg_versioned from $TERMUX_REPO_URL failed, building instead" - build_dependency="true" - fi fi - if [[ "$cyclic_dependence" == "false" ]]; then - [[ "$build_dependency" == "true" ]] && termux_run_build-package && continue - termux_add_package_to_built_packages_list "$PKG" - fi - if [[ "$TERMUX_ON_DEVICE_BUILD" == "false" ]]; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "extracting $PKG to $TERMUX_COMMON_CACHEDIR-$DEP_ARCH..." + + if [[ "${dep[build]}" != "true" ]]; then + dep[download]="true" ( - cd "$TERMUX_COMMON_CACHEDIR-$DEP_ARCH" - if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" ]]; then - # Ignore topdir `.`, to avoid possible permission errors from tar - ar p "${PKG}_${DEP_VERSION}_${DEP_ARCH}.deb" "data.tar.xz" | \ - tar xJ --no-overwrite-dir --transform='s#^.$#data#' -C / - elif [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" ]]; then - tar -xJf "${PKG}-${DEP_VERSION_PAC}-${DEP_ARCH}.pkg.tar.xz" \ - --anchored --exclude=.{BUILDINFO,PKGINFO,MTREE,INSTALL} \ - --force-local --no-overwrite-dir -C / + if ! TERMUX_WITHOUT_DEPVERSION_BINDING="$([[ "${dep[cyclic]}" == "true" ]] && echo "true" || echo "${TERMUX_WITHOUT_DEPVERSION_BINDING}")" termux_download_deb_pac "${dep[name]}" "${dep[arch]}" "${dep[version]}" "${dep[version_pac]}"; then + if [[ "${dep[cyclic]}" == "true" || ( "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" && "$TERMUX_ON_DEVICE_BUILD" == "true" ) ]]; then + echo "Download of ${dep[name]}$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" && "${dep[cyclic]}" == "false" ]] && echo "@${dep[version]}") from $TERMUX_REPO_URL failed" >&2 + exit "${DOWNLOAD_FAILED_EXIT_CODE}" + fi + echo "Download of ${dep[versioned]} from $TERMUX_REPO_URL failed, building instead" fi - ) + ) 2>&1 | termux_buffered_output "${dep[dir]}" & + # for the case if user explicitly disabled dependency downloading + [[ "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING-true}" == "false" ]] && ! wait -n && (( $? == DOWNLOAD_FAILED_EXIT_CODE )) && exit 1 + fi + fi + + # Enqueue the dependency for deferred processing; resume work once all background downloads complete + deps+=("$name") + done < <(./scripts/buildorder.py $([[ "${TERMUX_INSTALL_DEPS}" == "true" ]] && echo "-i") "$TERMUX_PKG_BUILDER_DIR" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR") + + while [[ -n "$(jobs -p)" ]]; do + if ! wait -n && (( $? == DOWNLOAD_FAILED_EXIT_CODE )); then + # One of jobs exited with fatal error, we should return error too and kill all background jobs + if [[ "${CI-false}" != "true" ]]; then + # In the case of local building we'd want to finish these downloads and reuse debs later + wait + else + # In the case of CI it has no sense and we'd want to finish earlier + echo "FATAL: one of downloads failed, exiting" + kill $(jobs -p) 2>/dev/null + fi + exit 1 + fi + done + + for name in "${deps[@]}"; do + declare -n dep="$name" + if [[ "$TERMUX_INSTALL_DEPS" == "true" || "${dep[cyclic]}" = "true" ]]; then + # We can not explicitly check obtain PID from `wait` or `wait -n` so we will simply check if downloading failed. + if [[ "${dep[download]}" == "true" ]]; then + if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" && ! -f "$TERMUX_COMMON_CACHEDIR-${dep[arch]}/${dep[name]}_${dep[version]}_${dep[arch]}.deb" ]] \ + || [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" && ! -f "$TERMUX_COMMON_CACHEDIR-${dep[arch]}/${dep[name]}-${dep[version_pac]}-${dep[arch]}.pkg.tar.xz" ]]; then + dep[build]="true" + fi fi + + if [[ "${dep[cyclic]}" == "false" ]]; then + [[ "${dep[build]}" == "true" ]] && { termux_run_build-package || exit $?; } && continue + termux_add_package_to_built_packages_list "${dep[name]}" + fi + + [[ "$TERMUX_ON_DEVICE_BUILD" == "false" ]] && ( + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "extracting ${dep[name]} to $TERMUX_COMMON_CACHEDIR-${dep[arch]}..." + cd "$TERMUX_COMMON_CACHEDIR-${dep[arch]}" + if [[ "$TERMUX_REPO_PKG_FORMAT" == "debian" ]]; then + # Ignore topdir `.`, to avoid possible permission errors from tar + ar p "${dep[name]}_${dep[version]}_${dep[arch]}.deb" "data.tar.xz" | \ + tar xJ --no-overwrite-dir --transform='s#^.$#data#' -C / + elif [[ "$TERMUX_REPO_PKG_FORMAT" == "pacman" ]]; then + tar -xJf "${dep[name]}-${dep[version_pac]}-${dep[arch]}.pkg.tar.xz" \ + --anchored --exclude=.{BUILDINFO,PKGINFO,MTREE,INSTALL} \ + --force-local --no-overwrite-dir -C / + fi + ) mkdir -p "$TERMUX_BUILT_PACKAGES_DIRECTORY" - if [[ "$cyclic_dependence" == "false" && ( "$TERMUX_WITHOUT_DEPVERSION_BINDING" == "false" || "$TERMUX_ON_DEVICE_BUILD" == "false" ) ]]; then - echo "$DEP_VERSION" > "$TERMUX_BUILT_PACKAGES_DIRECTORY/$PKG" + if [[ "${dep[cyclic]}" == "false" && ( "$TERMUX_WITHOUT_DEPVERSION_BINDING" == "false" || "$TERMUX_ON_DEVICE_BUILD" == "false" ) ]]; then + echo "${dep[version]}" > "$TERMUX_BUILT_PACKAGES_DIRECTORY/${dep[name]}" fi else # Build dependencies # Built dependencies are put in the default TERMUX_OUTPUT_DIR instead of the specified one if [[ "$TERMUX_FORCE_BUILD_DEPENDENCIES" == "true" ]]; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency $PKG..." - read -r DEP_ARCH DEP_VERSION DEP_VERSION_PAC DEP_ON_DEVICE_NOT_SUPPORTED < <(termux_extract_dep_info $PKG "${PKG_DIR}") - [[ "$TERMUX_ON_DEVICE_BUILD" == "true" && "$DEP_ON_DEVICE_NOT_SUPPORTED" == "true" ]] \ - && termux_error_exit "Building $PKG on device is not supported. Consider passing -I flag to download it instead" + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Force building dependency ${dep[name]}..." + [[ "$TERMUX_ON_DEVICE_BUILD" == "true" && "${dep[on_device_not_supported]}" == "true" ]] \ + && termux_error_exit "Building ${dep[name]} on device is not supported. Consider passing -I flag to download it instead" termux_force_check_package_dependency && continue else - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Building dependency $PKG if necessary..." + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Building dependency ${dep[name]} if necessary..." fi termux_run_build-package fi - done < <(./scripts/buildorder.py $([[ "${TERMUX_INSTALL_DEPS}" == "true" ]] && echo "-i") "$TERMUX_PKG_BUILDER_DIR" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR") + done } termux_force_check_package_dependency() { - if termux_check_package_in_built_packages_list "$PKG" && termux_package__is_package_version_built "$PKG" "$DEP_VERSION"; then - [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency $PKG$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && echo "@$DEP_VERSION")" + if termux_check_package_in_built_packages_list "${dep[name]}" && termux_package__is_package_version_built "${dep[name]}" "${dep[version]}"; then + [[ "$TERMUX_QUIET_BUILD" != "true" ]] && echo "Skipping already built dependency ${dep[name]}$([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" == "false" ]] && echo "@${dep[version]}")" return 0 fi return 1 @@ -97,7 +144,7 @@ termux_run_build-package() { set_library="$TERMUX_PACKAGE_LIBRARY -L" else set_library="bionic" - if termux_package__is_package_name_have_glibc_prefix "$PKG"; then + if termux_package__is_package_name_have_glibc_prefix "${dep[name]}"; then set_library="glibc" fi fi @@ -106,7 +153,8 @@ termux_run_build-package() { $([[ "${TERMUX_FORCE_BUILD}" == "true" && "${TERMUX_FORCE_BUILD_DEPENDENCIES}" == "true" ]] && echo "-F") \ $([[ "${TERMUX_PKGS__BUILD__RM_ALL_PKG_BUILD_DEPENDENT_DIRS}" == "true" ]] && echo "-r") \ $([[ "${TERMUX_WITHOUT_DEPVERSION_BINDING}" = "true" ]] && echo "-w") \ - --format $TERMUX_PACKAGE_FORMAT --library $set_library "${PKG_DIR}" + $([[ "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING-true}" = "false" ]] && echo "--disable-dependency-download-parallelizing") \ + --format $TERMUX_PACKAGE_FORMAT --library $set_library "${dep[dir]}" } termux_download_repo_file() { diff --git a/scripts/build/termux_step_setup_variables.sh b/scripts/build/termux_step_setup_variables.sh index 0ac9dbffa6982c..9976eff60de064 100644 --- a/scripts/build/termux_step_setup_variables.sh +++ b/scripts/build/termux_step_setup_variables.sh @@ -15,6 +15,7 @@ termux_step_setup_variables() { : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_GLOBAL_LIBRARY:="false"}" : "${TERMUX_TOPDIR:="$HOME/.termux-build"}" + : "${TERMUX_DEPENDENCY_DOWNLOAD_PARALLELIZING:=true}" : "${TERMUX_PACMAN_PACKAGE_COMPRESSION:="xz"}" if [ -z "${TERMUX_PACKAGE_FORMAT-}" ]; then