diff --git a/scripts/bin/update-packages b/scripts/bin/update-packages index 61f0d676b1015a..5e62c42d24bcc7 100755 --- a/scripts/bin/update-packages +++ b/scripts/bin/update-packages @@ -70,14 +70,17 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$ # shellcheck source=scripts/updates/internal/termux_repology_auto_update.sh . "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_repology_auto_update.sh +# shellcheck source=scripts/updates/internal/termux_github_graphql.sh +. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_graphql.sh + # Main script to: # - by default, decide which update method to use, -# - but can be overrided by build.sh to use custom update method. -# - For example: see neovim-nightly's build.sh. +# - but can be overridden by build.sh to use custom update method. +# - For example: see cmake's build.sh. # shellcheck source=scripts/updates/termux_pkg_auto_update.sh . "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh -# Converts milliseconds to human-readable format. +# Converts milliseconds to human-readable format. # Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms ms_to_human_readable() { echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //' @@ -126,9 +129,9 @@ declare -A _LATEST_TAGS=() declare -A _FAILED_UPDATES=() declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped. -# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system. -# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time. -# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly. +# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system. +# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time. +# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly. # We avoid packages with overwritten termux_pkg_auto_update to prevent unexpected modifications to the package`s build.sh. _fetch_and_cache_tags() { if [ "$(uname -o)" = "Android" ] || [ -e "/system/bin/app_process" ]; then @@ -137,7 +140,7 @@ _fetch_and_cache_tags() { return 0 fi fi - + if ! command -v ninja &> /dev/null; then echo "INFO: Fetching ninja build system" . "${TERMUX_SCRIPTDIR}"/scripts/build/termux_download.sh @@ -150,20 +153,64 @@ _fetch_and_cache_tags() { # First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata. quiet termux_repology_api_get_latest_version ' ' - local __PACKAGES=() + local __PACKAGES=() + local __GITHUB_PACKAGES=() for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do for pkg_dir in "${repo_dir}"/*; do - ! quiet _should_update "${pkg_dir}" && continue # Skip if not needed. + quiet _should_update "${pkg_dir}" || continue # Skip if not needed. grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update - __PACKAGES+=("${pkg_dir}") + if grep '^TERMUX_PKG_SRCURL=' "${pkg_dir}/build.sh" | grep -q 'github.com'; then + __GITHUB_PACKAGES+=("${pkg_dir}") + else + __PACKAGES+=("${pkg_dir}") + fi done done + echo "INFO: Building GraphQL queries" + + local GITHUB_GRAPHQL_QUERIES=() COUNTER=0 + declare -A __GITHUB_GRAPHQL_PACKAGES=() + # Ignore non-constant sources + # shellcheck disable=SC1091 + for pkg_dir in "${__GITHUB_PACKAGES[@]}"; do + local PKG_SRCURL TAG_TYPE OWNER REPO + read -r PKG_SRCURL TAG_TYPE < <( + set +u + source "${pkg_dir}/build.sh" + echo "${TERMUX_PKG_SRCURL} ${TERMUX_PKG_UPDATE_TAG_TYPE}" + ) + + read -r OWNER REPO < <(sed -E 's|^(git\+)?https?://github.com/([^/]+)/([^/]+).*|\2 \3|' <<< "$PKG_SRCURL") + + if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url. + if [[ "${PKG_SRCURL:0:4}" == "git+" ]]; then + # Get newest tag. + TAG_TYPE="newest-tag" + else + # Get the latest release tag. + TAG_TYPE="latest-release-tag" + fi + fi + + # We prepare the query snippets for `termux_github_graphql` here + # since we already have the needed information available. + GITHUB_GRAPHQL_QUERIES+=( "_$((COUNTER++)): repository(owner: \\\"${OWNER}\\\", name: \\\"${REPO}\\\") { ..._${TAG_TYPE//-/_} }" ) + + unset PKG_SRCURL TAG_TYPE OWNER REPO + done + + # This is called indirectly in the ninja file + # So silence shellcheck's unreachable code warning + # shellcheck disable=SC2317 __main__() { - cd ${TERMUX_SCRIPTDIR} + cd "${TERMUX_SCRIPTDIR}" export TERMUX_PKG_NAME="${1##*/}" TERMUX_PKG_BUILDER_DIR=${1} - set +eu; - for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do source ${i}; done + set +eu + for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do + # shellcheck disable=SC1090 + source "${i}" + done set -eu termux_pkg_upgrade_version() { [[ -n "$1" ]] && echo "PKG|$TERMUX_PKG_NAME|${1#*:}" @@ -176,23 +223,33 @@ _fetch_and_cache_tags() { termux_pkg_auto_update } + # This function generates a ninja file for all packages in the ${__PACKAGES[@]} + # So shut up shellcheck nagging us about using `sed` + # shellcheck disable=SC2001 __generate__() { echo "rule update" - echo " command = bash -c \"\$\$F\" -- \$pkg_dir ||:" + echo " command = bash -c \"\$\$F\" -- \$pkg_dir || :" sed 's/[^ ]\+/\nbuild &: update\n pkg_dir=&/g' <<< "${__PACKAGES[@]}" - echo "build run_all: phony ${__PACKAGES[@]}" + echo "build run_all: phony ${__PACKAGES[*]}" echo "default run_all" } - - local LATEST_TAGS="$(\ + + echo "INFO: Fetching GitHub packages via GraphQL API" + LATEST_TAGS_GITHUB="$( + termux_github_graphql "${GITHUB_GRAPHQL_QUERIES[@]}" + )" + + echo "INFO: Fetching non-GitHub packages" + local LATEST_TAGS='' LATEST_TAGS_GITHUB='' + LATEST_TAGS="$( F="$(declare -p TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE); $(declare -f __main__ | sed 's/__main__ ()//')" \ - env --chdir=/tmp ninja -f /dev/stdin <<< "$(__generate__)" |& grep "^PKG|" + env --chdir=/tmp ninja -f /dev/stdin < <(__generate__) |& grep "^PKG|" )" - - unset -f __main__ __generate__ + unset -f __main__ __generate__ + while IFS='|' read -r _ pkg version; do _LATEST_TAGS["${pkg:-_}"]="$version" - done <<< "$LATEST_TAGS" + done < <(printf '%s\n' "$LATEST_TAGS" "$LATEST_TAGS_GITHUB") quiet declare -p _LATEST_TAGS } @@ -303,7 +360,7 @@ _update_dependencies() { termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")" fi _should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}" - done <<<"$("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")" + done < <("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR") } echo "INFO: Running update for: $*" @@ -312,23 +369,23 @@ if [[ "$1" == "@all" ]]; then _fetch_and_cache_tags for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do for pkg_dir in "${repo_dir}"/*; do - _unix_millis=$(date +%s%N | cut -b1-13) + _unix_millis="$(date +%10s%3N)" ! _should_update "${pkg_dir}" && continue # Skip if not needed. _check_updated "${pkg_dir}" && continue # Skip if already up-to-date. - # Update all its dependencies first. - _update_dependencies "${pkg_dir}" - # NOTE: I am not cheacking whether dependencies were updated successfully or not. - # There is no way I could know whether this package will build with current + _update_dependencies "${pkg_dir}" # Update all its dependencies first. + # NOTE: We are not checking whether dependencies were updated successfully or not. + # There is no way to know whether this package will build with current # available verions of its dependencies or needs new ones. # So, whatever the case may be. We just need packages to be updated in order # and not care about anything else in between. If something fails to update, # it will be reported by failure handling code, so no worries. _run_update "${pkg_dir}" - echo "termux - took $(ms_to_human_readable $(( $(date +%s%N | cut -b1-13) - _unix_millis )))" + echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))" done done else for pkg in "$@"; do + _unix_millis="$(date +%10s%3N)" if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory. for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do if [ -d "${repo_dir}/${pkg}" ]; then @@ -341,8 +398,10 @@ else ! _should_update "${pkg}" && continue _update_dependencies "${pkg}" _run_update "${pkg}" + echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))" done fi +unset _unix_millis ################################################FAILURE HANDLING################################################# @@ -367,7 +426,7 @@ _gh_create_new_issue() { fi # Extract origin URL, commit hash and builder directory and put everything together - link="$(git config --get remote.origin.url |sed -E 's|\.git$||; s|git@([^:]+):(.+)|https://\1/\2|')/blob/$(git rev-parse HEAD)/$(echo */$1)" + link="$(git config --get remote.origin.url | sed -E 's|\.git$||; s|git@([^:]+):(.+)|https://\1/\2|')/blob/$(git rev-parse HEAD)/$(echo */"$1")" body="$( cat <<-EOF diff --git a/scripts/updates/internal/termux_github_graphql.sh b/scripts/updates/internal/termux_github_graphql.sh new file mode 100644 index 00000000000000..679599e9c60ee0 --- /dev/null +++ b/scripts/updates/internal/termux_github_graphql.sh @@ -0,0 +1,74 @@ +# Takes in a list of GraphQL query snippets +termux_github_graphql() { + local -a GITHUB_GRAPHQL_QUERIES=( "$@" ) + + # Batch size for fetching tags, 100 seems to work consistently. + local BATCH BATCH_SIZE=100 + for (( BATCH = 0; ${#GITHUB_GRAPHQL_QUERIES[@]} >= BATCH_SIZE * BATCH ; BATCH++ )); do + + echo "Starting batch $BATCH at: ${GITHUB_GRAPHQL_QUERIES[$BATCH * $BATCH_SIZE]//\\/}" >&2 + + # JSON strings cannot contain tabs or newlines + # so shutup shellcheck complaining about escapes in single quotes + local QUERY + + # Start the GraphQL query with our two fragments for getting the latest tag from a release, and from refs/tags + # These are defined only if needed, so this one is for '_latest_release_tag' + grep -q '_latest_release_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && { + QUERY+="$(printf '%s\n' \ + 'fragment _latest_release_tag on Repository {' \ + ' latestRelease { tagName }' \ + '}')" + } + + grep -q '_newest_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && { + QUERY+="$(printf '%s\n' \ + 'fragment _newest_tag on Repository {' \ + ' refs( refPrefix: \"refs/tags/\" first: 1 orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {' \ + ' nodes { name }' \ + ' }' \ + '}')" + } + + QUERY+='query {' + + # Fill out the query body with the package repos we need to query for updates + # Lastly fetch the rate limit utilization + printf -v QUERY '%s\n' \ + "${QUERY}" \ + "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" \ + 'ratelimit: rateLimit { cost limit remaining used resetAt }' \ + '}' \ + + # echo "// Batch: $BATCH" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries + # printf '%s' "${QUERY}" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries + + local response + response="$(printf '{ "query": "%s" }' "${QUERY//$'\n'/ }" | curl -fL \ + --no-progress-meter \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H 'Accept: application/vnd.github.v3+json' \ + -H 'Content-Type: application/json' \ + -X POST \ + --data @- \ + https://api.github.com/graphql 2>&1 + )" || termux_error_exit "ERR - termux_github_graphql: $response" + + + unset QUERY + local TAGS idx i=0 + TAGS="$(jq -r '.data # From the data: table + | del(.ratelimit) # Remove the ratelimit: table + | to_entries[] # convert the remaining entries to an array + | .value # For each .value + | (.latestRelease?.tagName # Print out the tag name of the latest release + // .refs.nodes[]?.name # or of the latest tag + // null)' <<< "$response")" # If neither exists return null + + while IFS=$'\n' read -r idx; do + printf 'PKG|%s|%s\n' \ + "${__GITHUB_PACKAGES[$BATCH * $BATCH_SIZE + $(( i++ ))]##*/}" \ + "${idx}" + done <<< "$TAGS" + done +}