这是indexloc提供的服务，不要输入任何密码
Skip to content

Make auto updates rate-limit aware #22994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 23, 2025
Merged

Make auto updates rate-limit aware #22994

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
545f675
chore(script/update-packages): Script cleanup
TomJo2000 Jan 30, 2025
9245d4e
enhance(script/update-packages): Fetch github releases/tags via batch…
TomJo2000 Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 88 additions & 29 deletions scripts/bin/update-packages
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,17 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$
# shellcheck source=scripts/updates/internal/termux_repology_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_repology_auto_update.sh

# shellcheck source=scripts/updates/internal/termux_github_graphql.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/internal/termux_github_graphql.sh

# Main script to:
# - by default, decide which update method to use,
# - but can be overrided by build.sh to use custom update method.
# - For example: see neovim-nightly's build.sh.
# - but can be overridden by build.sh to use custom update method.
# - For example: see cmake's build.sh.
# shellcheck source=scripts/updates/termux_pkg_auto_update.sh
. "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh

# Converts milliseconds to human-readable format.
# Converts milliseconds to human-readable format.
# Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms
ms_to_human_readable() {
echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //'
Expand Down Expand Up @@ -126,9 +129,9 @@ declare -A _LATEST_TAGS=()
declare -A _FAILED_UPDATES=()
declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped.

# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system.
# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time.
# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly.
# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system.
# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time.
# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly.
# We avoid packages with overwritten termux_pkg_auto_update to prevent unexpected modifications to the package`s build.sh.
_fetch_and_cache_tags() {
if [ "$(uname -o)" = "Android" ] || [ -e "/system/bin/app_process" ]; then
Expand All @@ -137,7 +140,7 @@ _fetch_and_cache_tags() {
return 0
fi
fi

if ! command -v ninja &> /dev/null; then
echo "INFO: Fetching ninja build system"
. "${TERMUX_SCRIPTDIR}"/scripts/build/termux_download.sh
Expand All @@ -150,20 +153,64 @@ _fetch_and_cache_tags() {
# First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata.
quiet termux_repology_api_get_latest_version ' '

local __PACKAGES=()
local __PACKAGES=()
local __GITHUB_PACKAGES=()
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
for pkg_dir in "${repo_dir}"/*; do
! quiet _should_update "${pkg_dir}" && continue # Skip if not needed.
quiet _should_update "${pkg_dir}" || continue # Skip if not needed.
grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update
__PACKAGES+=("${pkg_dir}")
if grep '^TERMUX_PKG_SRCURL=' "${pkg_dir}/build.sh" | grep -q 'github.com'; then
__GITHUB_PACKAGES+=("${pkg_dir}")
else
__PACKAGES+=("${pkg_dir}")
fi
done
done

echo "INFO: Building GraphQL queries"

local GITHUB_GRAPHQL_QUERIES=() COUNTER=0
declare -A __GITHUB_GRAPHQL_PACKAGES=()
# Ignore non-constant sources
# shellcheck disable=SC1091
for pkg_dir in "${__GITHUB_PACKAGES[@]}"; do
local PKG_SRCURL TAG_TYPE OWNER REPO
read -r PKG_SRCURL TAG_TYPE < <(
set +u
source "${pkg_dir}/build.sh"
echo "${TERMUX_PKG_SRCURL} ${TERMUX_PKG_UPDATE_TAG_TYPE}"
)

read -r OWNER REPO < <(sed -E 's|^(git\+)?https?://github.com/([^/]+)/([^/]+).*|\2 \3|' <<< "$PKG_SRCURL")

if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url.
if [[ "${PKG_SRCURL:0:4}" == "git+" ]]; then
# Get newest tag.
TAG_TYPE="newest-tag"
else
# Get the latest release tag.
TAG_TYPE="latest-release-tag"
fi
fi

# We prepare the query snippets for `termux_github_graphql` here
# since we already have the needed information available.
GITHUB_GRAPHQL_QUERIES+=( "_$((COUNTER++)): repository(owner: \\\"${OWNER}\\\", name: \\\"${REPO}\\\") { ..._${TAG_TYPE//-/_} }" )

unset PKG_SRCURL TAG_TYPE OWNER REPO
done

# This is called indirectly in the ninja file
# So silence shellcheck's unreachable code warning
# shellcheck disable=SC2317
__main__() {
cd ${TERMUX_SCRIPTDIR}
cd "${TERMUX_SCRIPTDIR}"
export TERMUX_PKG_NAME="${1##*/}" TERMUX_PKG_BUILDER_DIR=${1}
set +eu;
for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do source ${i}; done
set +eu
for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do
# shellcheck disable=SC1090
source "${i}"
done
set -eu
termux_pkg_upgrade_version() {
[[ -n "$1" ]] && echo "PKG|$TERMUX_PKG_NAME|${1#*:}"
Expand All @@ -176,23 +223,33 @@ _fetch_and_cache_tags() {
termux_pkg_auto_update
}

# This function generates a ninja file for all packages in the ${__PACKAGES[@]}
# So shut up shellcheck nagging us about using `sed`
# shellcheck disable=SC2001
__generate__() {
echo "rule update"
echo " command = bash -c \"\$\$F\" -- \$pkg_dir ||:"
echo " command = bash -c \"\$\$F\" -- \$pkg_dir || :"
sed 's/[^ ]\+/\nbuild &: update\n pkg_dir=&/g' <<< "${__PACKAGES[@]}"
echo "build run_all: phony ${__PACKAGES[@]}"
echo "build run_all: phony ${__PACKAGES[*]}"
echo "default run_all"
}

local LATEST_TAGS="$(\

echo "INFO: Fetching GitHub packages via GraphQL API"
LATEST_TAGS_GITHUB="$(
termux_github_graphql "${GITHUB_GRAPHQL_QUERIES[@]}"
)"

echo "INFO: Fetching non-GitHub packages"
local LATEST_TAGS='' LATEST_TAGS_GITHUB=''
LATEST_TAGS="$(
F="$(declare -p TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE); $(declare -f __main__ | sed 's/__main__ ()//')" \
env --chdir=/tmp ninja -f /dev/stdin <<< "$(__generate__)" |& grep "^PKG|"
env --chdir=/tmp ninja -f /dev/stdin < <(__generate__) |& grep "^PKG|"
)"

unset -f __main__ __generate__
unset -f __main__ __generate__

while IFS='|' read -r _ pkg version; do
_LATEST_TAGS["${pkg:-_}"]="$version"
done <<< "$LATEST_TAGS"
done < <(printf '%s\n' "$LATEST_TAGS" "$LATEST_TAGS_GITHUB")
quiet declare -p _LATEST_TAGS
}

Expand Down Expand Up @@ -303,7 +360,7 @@ _update_dependencies() {
termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")"
fi
_should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}"
done <<<"$("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")"
done < <("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")
}

echo "INFO: Running update for: $*"
Expand All @@ -312,23 +369,23 @@ if [[ "$1" == "@all" ]]; then
_fetch_and_cache_tags
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
for pkg_dir in "${repo_dir}"/*; do
_unix_millis=$(date +%s%N | cut -b1-13)
_unix_millis="$(date +%10s%3N)"
! _should_update "${pkg_dir}" && continue # Skip if not needed.
_check_updated "${pkg_dir}" && continue # Skip if already up-to-date.
# Update all its dependencies first.
_update_dependencies "${pkg_dir}"
# NOTE: I am not cheacking whether dependencies were updated successfully or not.
# There is no way I could know whether this package will build with current
_update_dependencies "${pkg_dir}" # Update all its dependencies first.
# NOTE: We are not checking whether dependencies were updated successfully or not.
# There is no way to know whether this package will build with current
# available verions of its dependencies or needs new ones.
# So, whatever the case may be. We just need packages to be updated in order
# and not care about anything else in between. If something fails to update,
# it will be reported by failure handling code, so no worries.
_run_update "${pkg_dir}"
echo "termux - took $(ms_to_human_readable $(( $(date +%s%N | cut -b1-13) - _unix_millis )))"
echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))"
done
done
else
for pkg in "$@"; do
_unix_millis="$(date +%10s%3N)"
if [ ! -d "${pkg}" ]; then # If only package name is given, try to find it's directory.
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
if [ -d "${repo_dir}/${pkg}" ]; then
Expand All @@ -341,8 +398,10 @@ else
! _should_update "${pkg}" && continue
_update_dependencies "${pkg}"
_run_update "${pkg}"
echo "termux - took $(ms_to_human_readable $(( $(date +%10s%3N) - _unix_millis )))"
done
fi
unset _unix_millis

################################################FAILURE HANDLING#################################################

Expand All @@ -367,7 +426,7 @@ _gh_create_new_issue() {
fi

# Extract origin URL, commit hash and builder directory and put everything together
link="$(git config --get remote.origin.url |sed -E 's|\.git$||; s|git@([^:]+):(.+)|https://\1/\2|')/blob/$(git rev-parse HEAD)/$(echo */$1)"
link="$(git config --get remote.origin.url | sed -E 's|\.git$||; s|git@([^:]+):(.+)|https://\1/\2|')/blob/$(git rev-parse HEAD)/$(echo */"$1")"

body="$(
cat <<-EOF
Expand Down
74 changes: 74 additions & 0 deletions scripts/updates/internal/termux_github_graphql.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Takes in a list of GraphQL query snippets
termux_github_graphql() {
local -a GITHUB_GRAPHQL_QUERIES=( "$@" )

# Batch size for fetching tags, 100 seems to work consistently.
local BATCH BATCH_SIZE=100
for (( BATCH = 0; ${#GITHUB_GRAPHQL_QUERIES[@]} >= BATCH_SIZE * BATCH ; BATCH++ )); do

echo "Starting batch $BATCH at: ${GITHUB_GRAPHQL_QUERIES[$BATCH * $BATCH_SIZE]//\\/}" >&2

# JSON strings cannot contain tabs or newlines
# so shutup shellcheck complaining about escapes in single quotes
local QUERY

# Start the GraphQL query with our two fragments for getting the latest tag from a release, and from refs/tags
# These are defined only if needed, so this one is for '_latest_release_tag'
grep -q '_latest_release_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && {
QUERY+="$(printf '%s\n' \
'fragment _latest_release_tag on Repository {' \
' latestRelease { tagName }' \
'}')"
}

grep -q '_newest_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && {
QUERY+="$(printf '%s\n' \
'fragment _newest_tag on Repository {' \
' refs( refPrefix: \"refs/tags/\" first: 1 orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {' \
' nodes { name }' \
' }' \
'}')"
}

QUERY+='query {'

# Fill out the query body with the package repos we need to query for updates
# Lastly fetch the rate limit utilization
printf -v QUERY '%s\n' \
"${QUERY}" \
"${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" \
'ratelimit: rateLimit { cost limit remaining used resetAt }' \
'}' \

# echo "// Batch: $BATCH" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries
# printf '%s' "${QUERY}" >> /tmp/query-12345 # Uncomment for debugging GraphQL queries

local response
response="$(printf '{ "query": "%s" }' "${QUERY//$'\n'/ }" | curl -fL \
--no-progress-meter \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H 'Accept: application/vnd.github.v3+json' \
-H 'Content-Type: application/json' \
-X POST \
--data @- \
https://api.github.com/graphql 2>&1
)" || termux_error_exit "ERR - termux_github_graphql: $response"


unset QUERY
local TAGS idx i=0
TAGS="$(jq -r '.data # From the data: table
| del(.ratelimit) # Remove the ratelimit: table
| to_entries[] # convert the remaining entries to an array
| .value # For each .value
| (.latestRelease?.tagName # Print out the tag name of the latest release
// .refs.nodes[]?.name # or of the latest tag
// null)' <<< "$response")" # If neither exists return null

while IFS=$'\n' read -r idx; do
printf 'PKG|%s|%s\n' \
"${__GITHUB_PACKAGES[$BATCH * $BATCH_SIZE + $(( i++ ))]##*/}" \
"${idx}"
done <<< "$TAGS"
done
}