From 9b8517ac8faae1a86155698ea16126d1c78e4038 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Fri, 2 May 2025 13:39:47 +0200 Subject: [PATCH 01/12] Initial update --- .github/workflows/checker.yml | 4 +- .github/workflows/container.yml | 127 +++++++ .github/workflows/data-update.yml | 8 +- .github/workflows/integration.yml | 156 ++++---- .github/workflows/security.yml | 8 +- .github/workflows/translations-update.yml | 8 +- Makefile | 4 +- manage | 421 +++++++++++++++++----- 8 files changed, 542 insertions(+), 194 deletions(-) create mode 100644 .github/workflows/container.yml diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index d0074712d3b..d308bf56d48 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -1,5 +1,7 @@ name: "Checker" -on: # yamllint disable-line rule:truthy + +# yamllint disable-line rule:truthy +on: schedule: - cron: "0 4 * * 5" workflow_dispatch: diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 00000000000..5c0cc09f7c3 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,127 @@ +--- +name: Container + +# yamllint disable-line rule:truthy +on: + workflow_run: + workflows: + - Integration + types: + - completed +# TODO: Uncomment +# branches: +# - master + +permissions: + contents: read + # Organization GHCR + packages: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + build: + if: github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + - arch: + arm64 + armv7 + os: ubuntu-24.04-arm + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-" + path: "./local" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + key: "container-mounts-${{ hashFiles('./Dockerfile') }}" + restore-keys: "container-mounts-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make -e GIT_URL=$(git remote get-url origin) ci.container.build + + release: + # TODO: Uncomment before merge + # if: github.repository_owner == 'searxng' + if: false + name: Release (${{ matrix.arch }}) + runs-on: ubuntu-24.04-arm + needs: build + strategy: + matrix: + arch: + - amd64 + - arm64 + - armv7 + + steps: + - if: env.DOCKERHUB_USERNAME != '' + name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.push" can get the git history + fetch-depth: "0" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: "docker.io" + username: "${{ env.DOCKERHUB_USERNAME }}" + password: "${{ secrets.DOCKERHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Release + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make -e GIT_URL=$(git remote get-url origin) ci.container.push diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 39893127d5a..ba7b6f8005b 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -1,8 +1,10 @@ -name: "Update searx.data" -on: # yamllint disable-line rule:truthy +name: Update searx.data + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "59 23 28 * *" - workflow_dispatch: jobs: updateData: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ddc0498f762..b0170d7945a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,70 +1,96 @@ name: Integration -on: # yamllint disable-line rule:truthy +# yamllint disable-line rule:truthy +on: + workflow_dispatch: push: - branches: ["master"] + branches: + - master pull_request: - branches: ["master"] + branches: + - master permissions: contents: read jobs: python: + # TODO: Remove this + if: false name: Python ${{ matrix.python-version }} runs-on: ubuntu-24.04 strategy: matrix: - os: [ubuntu-24.04] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" + - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages + run: sudo ./utils/searxng.sh install packages + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' + python-version: "${{ matrix.python-version }}" + - name: Run tests run: make V=1 ci.test themes: + # TODO: Remove this + if: false name: Themes runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" + - name: Install Ubuntu packages run: sudo ./utils/searxng.sh install buildhost + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' + python-version: "3.13" + - name: Build themes run: make themes.all documentation: + # TODO: Remove this + if: false name: Documentation runs-on: ubuntu-24.04 permissions: - contents: write # for JamesIves/github-pages-deploy-action to push changes in repo + # for JamesIves/github-pages-deploy-action to push + contents: write + steps: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' - persist-credentials: false + persist-credentials: "false" + fetch-depth: "0" + - name: Install Ubuntu packages run: sudo ./utils/searxng.sh install buildhost + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' + python-version: "3.13" + - name: Cache Python dependencies id: cache-python uses: actions/cache@v4 @@ -73,42 +99,50 @@ jobs: ./local ./.nvm ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + key: python-ubuntu-24.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + - name: Build documentation - run: | - make V=1 docs.clean docs.html - - name: Deploy - if: github.ref == 'refs/heads/master' + run: make V=1 docs.clean docs.html + + - if: github.ref == 'refs/heads/master' + name: Deploy uses: JamesIves/github-pages-deploy-action@3.7.1 with: - GITHUB_TOKEN: ${{ github.token }} - BRANCH: gh-pages - FOLDER: dist/docs - CLEAN: true # Automatically remove deleted files from the deploy branch - SINGLE_COMMIT: true - COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' + GITHUB_TOKEN: "${{ github.token }}" + BRANCH: "gh-pages" + FOLDER: "dist/docs" + # Automatically remove deleted files from the deploy branch + CLEAN: "true" + SINGLE_COMMIT: "true" + COMMIT_MESSAGE: "[doc] build from commit ${{ github.sha }}" babel: + # TODO: Remove this + if: false + # if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' name: Update translations branch runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} needs: - python - themes - documentation + permissions: - contents: write # for make V=1 weblate.push.translations + # for make V=1 weblate.push.translations + contents: write + steps: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' + python-version: "3.13" + - name: Cache Python dependencies id: cache-python uses: actions/cache@v4 @@ -117,63 +151,17 @@ jobs: ./local ./.nvm ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + key: python-ubuntu-20.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + - name: weblate & git setup env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} + WEBLATE_CONFIG: "${{ secrets.WEBLATE_CONFIG }}" run: | mkdir -p ~/.config echo "${WEBLATE_CONFIG}" > ~/.config/weblate git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.name "searxng-bot" + - name: Update transations id: update - run: | - make V=1 weblate.push.translations - - dockers: - name: Docker - if: github.ref == 'refs/heads/master' - needs: - - python - - themes - - documentation - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - runs-on: ubuntu-24.04 - steps: - - name: Checkout - if: env.DOCKERHUB_USERNAME != null - uses: actions/checkout@v4 - with: - # make sure "make docker.push" can get the git history - fetch-depth: '0' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: Set up QEMU - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - if: env.DOCKERHUB_USERNAME != null - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - if: env.DOCKERHUB_USERNAME != null - run: make -e GIT_URL=$(git remote get-url origin) docker.buildx + run: make V=1 weblate.push.translations diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 249db305b64..f05522314a2 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,8 +1,10 @@ -name: "Security checks" -on: # yamllint disable-line rule:truthy +name: Security checks + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "42 05 * * *" - workflow_dispatch: jobs: dockers: diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml index 85e141e7f88..a6f7fa9cedc 100644 --- a/.github/workflows/translations-update.yml +++ b/.github/workflows/translations-update.yml @@ -1,8 +1,10 @@ -name: "Update translations" -on: # yamllint disable-line rule:truthy +name: Update translations + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "05 07 * * 5" - workflow_dispatch: jobs: babel: diff --git a/Makefile b/Makefile index c1c0671495d..d7ac9eb09c3 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,8 @@ test.shell: MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean -MANAGE += docker.build docker.push docker.buildx +MANAGE += docker.build docker.buildx +MANAGE += container.build container.buildx MANAGE += gecko.driver MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean @@ -87,6 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover MANAGE += themes.all themes.fix themes.test MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs +MANAGE += ci.container.build ci.container.push PHONY += $(MANAGE) diff --git a/manage b/manage index 61bc68b748c..91f045d34a4 100755 --- a/manage +++ b/manage @@ -60,7 +60,7 @@ while IFS= read -r line; do if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then YAMLLINT_FILES+=("$line") fi -done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" +done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" RST_FILES=( 'README.rst' @@ -77,9 +77,9 @@ docs.: gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build -docker.: - build : build docker image - push : build and push docker image +container. : + build : build container image + buildx : build container image with BuildKit gecko.driver: download & install geckodriver if not already installed (required for robot_tests) @@ -97,6 +97,10 @@ pyenv.: OK : test if virtualenv is OK format.: python : format Python code source using black +ci.: + container.: + build : build container image + release : push container images to remote registry EOF go.help node.help @@ -112,7 +116,6 @@ environment ... EOF } - if [ "$VERBOSE" = "1" ]; then SPHINX_VERBOSE="-v" PYLINT_VERBOSE="-v" @@ -125,53 +128,95 @@ webapp.run() { local parent_proc="$$" ( if [ "${LIVE_THEME}" ]; then - ( themes.live "${LIVE_THEME}" ) + (themes.live "${LIVE_THEME}") kill $parent_proc fi - )& + ) & ( sleep 3 xdg-open http://127.0.0.1:8888/ - )& + ) & SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp } -docker.push() { - docker.build push -} - +# Alias docker.buildx() { - docker.build buildx + container.buildx } -# shellcheck disable=SC2119 +# Alias docker.build() { + container.build +} + +container.buildx() { + container.build buildx +} + +container.build() { pyenv.install - local SEARXNG_GIT_VERSION - local VERSION_GITCOMMIT - local GITHUB_USER - local SEARXNG_IMAGE_NAME - local BUILD + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local container_engine + local dockerfile + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + dockerfile="Dockerfile" + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + dockerfile="Dockerfile" + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + # TODO: Move ARMv7 to a separated Dockerfile + dockerfile="Dockerfile" + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman or docker is installed + if command -v podman &>/dev/null; then + container_engine="podman" + elif command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "podman/docker is not installed" + fi + build_msg CONTAINER "Engine: $container_engine" - build_msg DOCKER build - # run installation in a subprocess and activate pyenv + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "git is not installed" + fi - # See https://www.shellcheck.net/wiki/SC1001 and others .. - # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 - ( set -e + ( + set -e pyenv.activate # Check if it is a git repository if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - if [ ! -x "$(which git)" ]; then - die 1 "git is not installed" + die 1 "This is not Git repository" fi - if ! git remote get-url origin 2> /dev/null; then - die 1 "there is no remote origin" + if ! git remote get-url origin 2>/dev/null; then + die 1 "there is no remote origin" fi # This is a git repository @@ -180,42 +225,48 @@ docker.build() { eval "$(python -m searx.version)" # Get the last git commit id - VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" - - # define the docker image name - GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') - SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" - - BUILD="build" - if [ "$1" = "buildx" ]; then - # buildx includes the push option - CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" - BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" - shift + version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) + build_msg CONTAINER "Last commit: $version_gitcommit" + + if [ "$container_engine" = "docker" ]; then + if [ "$1" = "buildx" ]; then + docker_builder="buildx build" + else + docker_builder="build" + warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback" + fi + + params_build_builder="$docker_builder --platform=$platform --target=builder" + params_build="$docker_builder --platform=$platform --squash" + else + params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false" fi - build_msg DOCKER "Build command: ${BUILD}" - # build Docker image - build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" + # Define container image org/name + # shellcheck disable=SC2001 + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + build_msg CONTAINER "Building..." + # shellcheck disable=SC2086 - docker $BUILD \ - --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ - --build-arg GIT_URL="${GIT_URL}" \ - --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ - --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ - --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ - --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ - --build-arg LABEL_VCS_URL="${GIT_URL}" \ - --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ - --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ - -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . - - if [ "$1" = "push" ]; then - docker push "${SEARXNG_IMAGE_NAME}:latest" - docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" - fi + "$container_engine" $params_build_builder \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ + --tag="localhost/$container_image_organization/$container_image_name:builder" \ + --file="./$dockerfile" + + # shellcheck disable=SC2086 + "$container_engine" $params_build \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --tag="localhost/$container_image_organization/$container_image_name:latest" \ + --tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \ + --file="./$dockerfile" ) dump_return $? } @@ -226,10 +277,11 @@ gecko.driver() { build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv - ( set -e + ( + set -e pyenv.activate - INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" + INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" set +e if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then build_msg INSTALL "geckodriver already installed" @@ -237,13 +289,13 @@ gecko.driver() { fi PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in - "linux 32bit" | "linux2 32bit") ARCH="linux32";; - "linux 64bit" | "linux2 64bit") ARCH="linux64";; - "windows 32 bit") ARCH="win32";; - "windows 64 bit") ARCH="win64";; - "mac 64bit") ARCH="macos";; + "linux 32bit" | "linux2 32bit") ARCH="linux32" ;; + "linux 64bit" | "linux2 64bit") ARCH="linux64" ;; + "windows 32 bit") ARCH="win32" ;; + "windows 64 bit") ARCH="win64" ;; + "mac 64bit") ARCH="macos" ;; esac - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz" build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" @@ -258,13 +310,14 @@ gecko.driver() { py.build() { build_msg BUILD "python package ${PYDIST}" pyenv.cmd python setup.py \ - sdist -d "${PYDIST}" \ - bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" + sdist -d "${PYDIST}" \ + bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv - ( set -e + ( + set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info @@ -275,22 +328,22 @@ py.clean() { } pyenv.check() { - cat < OK') EOF } pyenv.install() { - if ! pyenv.OK; then - py.clean > /dev/null + py.clean >/dev/null fi - if pyenv.install.OK > /dev/null; then + if pyenv.install.OK >/dev/null; then return 0 fi - ( set -e + ( + set -e pyenv build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}" @@ -303,8 +356,8 @@ pyenv.install() { pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" - pyenv.cmd python setup.py develop --uninstall 2>&1 \ - | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + pyenv.cmd python setup.py develop --uninstall 2>&1 | + prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } @@ -320,17 +373,184 @@ docs.prebuild() { set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" - ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" + ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst" pyenv.cmd searxng_extra/docs_prebuild ) dump_return $? } +ci.container.build() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + pyenv.install + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local dockerfile + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + dockerfile="Dockerfile" + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + dockerfile="Dockerfile" + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + # TODO: Move ARMv7 to a separated Dockerfile + dockerfile="Dockerfile" + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "git is not installed" + fi + + ( + set -eu + pyenv.activate + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + + if ! git remote get-url origin 2>/dev/null; then + die 1 "there is no remote origin" + fi + + # This is a git repository + git update-index -q --refresh + python -m searx.version freeze + eval "$(python -m searx.version)" + + # Get the last git commit id + version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) + build_msg CONTAINER "Last commit: $version_gitcommit" + + # Define container image org/name + # shellcheck disable=SC2001 + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + build_msg CONTAINER "Building..." + + podman build --platform="$platform" --target=builder --layers --identity-label=false \ + --cache-from="ghcr.io/$container_image_organization/cache" \ + --cache-to="ghcr.io/$container_image_organization/cache" \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ + --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \ + --file="./$dockerfile" + + podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ + --cache-from="ghcr.io/$container_image_organization/cache" \ + --cache-to="ghcr.io/$container_image_organization/cache" \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \ + --file="./$dockerfile" + + podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" + ) + dump_return $? +} + +ci.container.push() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -eu + + # Define container image org/name + # shellcheck disable=SC2001 + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" + + # Get version tag + container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \ + "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant") + + # Recreate tags + podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest" + podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" + + podman push "docker.io/$container_image_organization/$container_image_name:latest" + podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" + ) + dump_return $? +} + # shellcheck disable=SC2119 main() { - local _type - local cmd="$1"; shift + local cmd="$1" + shift if [ "$cmd" == "" ]; then help @@ -339,22 +559,25 @@ main() { fi case "$cmd" in - --getenv) var="$1"; echo "${!var}";; - --help) help;; - --*) - help - err_msg "unknown option $cmd" + --getenv) + var="$1" + echo "${!var}" + ;; + --help) help ;; + --*) + help + err_msg "unknown option $cmd" + return 42 + ;; + *) + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + err_msg "unknown command: $cmd / use --help" return 42 - ;; - *) - _type="$(type -t "$cmd")" - if [ "$_type" != 'function' ]; then - err_msg "unknown command: $cmd / use --help" - return 42 - else - "$cmd" "$@" - fi - ;; + else + "$cmd" "$@" + fi + ;; esac } From bbbd61eff79e04cca7313554417ceab8bcb05154 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 4 May 2025 20:04:00 +0200 Subject: [PATCH 02/12] refactor checker.yml --- .github/workflows/checker.yml | 42 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index d308bf56d48..d2992c66b65 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -1,33 +1,43 @@ -name: "Checker" +--- +name: Checker # yamllint disable-line rule:truthy on: + workflow_dispatch: schedule: - cron: "0 4 * * 5" - workflow_dispatch: + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" jobs: checker: name: Checker - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" - - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup cache Python + uses: actions/cache@v4 with: - python-version: '3.13' - architecture: 'x64' + key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-" + path: "./local" - - name: Install Python dependencies - run: | - make V=1 install + - name: Setup venv + run: make V=1 install - name: Checker - run: | - make search.checker + continue-on-error: true + run: make search.checker From a7efdd908b322864c3bff22a290c66f35dcb6367 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 4 May 2025 20:32:27 +0200 Subject: [PATCH 03/12] refactor data-update.yml Also fixes a cache issue found on data-update.yml, this fix is applied on checker.yml and container.yml files --- .github/workflows/checker.yml | 4 +- .github/workflows/container.yml | 4 +- .github/workflows/data-update.yml | 62 +++++++++++++++++-------------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index d2992c66b65..04639726b83 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -31,8 +31,8 @@ jobs: - name: Setup cache Python uses: actions/cache@v4 with: - key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}" - restore-keys: "python-${{ env.PYTHON_VERSION }}-" + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" path: "./local" - name: Setup venv diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 5c0cc09f7c3..b273d3d5ce3 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -53,8 +53,8 @@ jobs: - name: Setup cache Python uses: actions/cache@v4 with: - key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}" - restore-keys: "python-${{ env.PYTHON_VERSION }}-" + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" path: "./local" - name: Setup cache container mounts diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index ba7b6f8005b..2d748ec5d16 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -1,3 +1,4 @@ +--- name: Update searx.data # yamllint disable-line rule:truthy @@ -6,11 +7,17 @@ on: schedule: - cron: "59 23 28 * *" +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + jobs: updateData: - name: Update data - ${{ matrix.fetch }} - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng'}} + if: github.repository_owner == 'searxng' + name: ${{ matrix.fetch }} + runs-on: ubuntu-24.04-arm strategy: fail-fast: false matrix: @@ -22,42 +29,43 @@ jobs: - update_engine_traits.py - update_wikidata_units.py - update_engine_descriptions.py + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" - - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup cache Python + uses: actions/cache@v4 with: - python-version: '3.12' - architecture: 'x64' + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local" - - name: Install Python dependencies - run: | - make V=1 install + - name: Setup venv + run: make V=1 install - name: Fetch data - env: - FETCH_SCRIPT: ./searxng_extra/update/${{ matrix.fetch }} - run: | - V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT" + run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}" - name: Create Pull Request id: cpr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: - commit-message: '[data] update searx.data - ${{ matrix.fetch }}' - committer: searxng-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: update_data_${{ matrix.fetch }} - delete-branch: true - draft: false - title: '[data] update searx.data - ${{ matrix.fetch }}' + author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" + committer: "searxng-bot " + title: "[data] update searx.data - ${{ matrix.fetch }}" + commit-message: "[data] update searx.data - ${{ matrix.fetch }}" + branch: "update_data_${{ matrix.fetch }}" + delete-branch: "true" + draft: "false" + signoff: "false" body: | update searx.data - ${{ matrix.fetch }} labels: | From 8f5296e8c736248189af3904d079e4d1f6df7184 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 4 May 2025 22:04:14 +0200 Subject: [PATCH 04/12] refactor integration.yml --- .github/workflows/container.yml | 8 +- .github/workflows/data-update.yml | 2 +- .github/workflows/integration.yml | 129 ++++++++++++++++-------------- Makefile | 3 +- 4 files changed, 74 insertions(+), 68 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index b273d3d5ce3..818d2bc6b29 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -8,9 +8,9 @@ on: - Integration types: - completed -# TODO: Uncomment -# branches: -# - master + # TODO: Test container.yml on merge https://github.com/searxng/searxng/pull/4699 + branches: + - master permissions: contents: read @@ -55,7 +55,7 @@ jobs: with: key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" - path: "./local" + path: "./local/" - name: Setup cache container mounts uses: actions/cache@v4 diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 2d748ec5d16..9665064abe6 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -46,7 +46,7 @@ jobs: with: key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" - path: "./local" + path: "./local/" - name: Setup venv run: make V=1 install diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b0170d7945a..d32a4fe9665 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,3 +1,4 @@ +--- name: Integration # yamllint disable-line rule:truthy @@ -13,8 +14,11 @@ on: permissions: contents: read +env: + PYTHON_VERSION: "3.13" + jobs: - python: + test: # TODO: Remove this if: false name: Python ${{ matrix.python-version }} @@ -29,139 +33,140 @@ jobs: - "3.13" steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ matrix.python-version }}" + - name: Checkout uses: actions/checkout@v4 with: persist-credentials: "false" - - name: Install Ubuntu packages - run: sudo ./utils/searxng.sh install packages - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup cache Python + uses: actions/cache@v4 with: - python-version: "${{ matrix.python-version }}" + key: "python-${{ matrix.python-version }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ matrix.python-version }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install - name: Run tests run: make V=1 ci.test - themes: + theme: # TODO: Remove this if: false - name: Themes - runs-on: ubuntu-24.04 + name: Theme + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@v4 with: persist-credentials: "false" - - name: Install Ubuntu packages - run: sudo ./utils/searxng.sh install buildhost + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: "./.nvmrc" - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup cache Node.js + uses: actions/cache@v4 with: - python-version: "3.13" + key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}" + path: "./client/simple/node_modules/" - - name: Build themes + - name: Build run: make themes.all documentation: # TODO: Remove this if: false name: Documentation - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm permissions: # for JamesIves/github-pages-deploy-action to push contents: write steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 with: persist-credentials: "false" fetch-depth: "0" - - name: Install Ubuntu packages - run: sudo ./utils/searxng.sh install buildhost - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Cache Python dependencies - id: cache-python + - name: Setup cache Python uses: actions/cache@v4 with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-24.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install - name: Build documentation run: make V=1 docs.clean docs.html - if: github.ref == 'refs/heads/master' name: Deploy - uses: JamesIves/github-pages-deploy-action@3.7.1 + uses: JamesIves/github-pages-deploy-action@v4 with: - GITHUB_TOKEN: "${{ github.token }}" - BRANCH: "gh-pages" - FOLDER: "dist/docs" + folder: "dist/docs" + branch: "gh-pages" + commit-message: "[doc] build from commit ${{ github.sha }}" # Automatically remove deleted files from the deploy branch - CLEAN: "true" - SINGLE_COMMIT: "true" - COMMIT_MESSAGE: "[doc] build from commit ${{ github.sha }}" + clean: "true" + single-commit: "true" babel: - # TODO: Remove this - if: false - # if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' + if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' name: Update translations branch - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm needs: - - python - - themes + - test + - theme - documentation permissions: - # for make V=1 weblate.push.translations + # For "make V=1 weblate.push.translations" contents: write steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 with: token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" fetch-depth: "0" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Cache Python dependencies - id: cache-python + - name: Setup cache Python uses: actions/cache@v4 with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install - name: weblate & git setup - env: - WEBLATE_CONFIG: "${{ secrets.WEBLATE_CONFIG }}" run: | mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.name "searxng-bot" - name: Update transations - id: update run: make V=1 weblate.push.translations diff --git a/Makefile b/Makefile index d7ac9eb09c3..97714358219 100644 --- a/Makefile +++ b/Makefile @@ -100,5 +100,6 @@ $(MANAGE): PHONY += docs docker themes docs: docs.html -docker: docker.build +container: container.build +docker: container.build themes: themes.all From 0969926adddbe37a6b072dc3ea63c6f9b7ea26f6 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 4 May 2025 22:46:31 +0200 Subject: [PATCH 05/12] refactor security.yml --- .github/workflows/security.yml | 36 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index f05522314a2..a720e0df280 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,4 +1,5 @@ -name: Security checks +--- +name: Security # yamllint disable-line rule:truthy on: @@ -6,25 +7,30 @@ on: schedule: - cron: "42 05 * * *" +permissions: + contents: read + jobs: - dockers: - name: Trivy ${{ matrix.image }} - runs-on: ubuntu-24.04 + container: + name: Container + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master + - name: Run Trivy scanner + uses: aquasecurity/trivy-action@0.30.0 with: - image-ref: 'searxng/searxng:latest' - ignore-unfixed: false - vuln-type: 'os,library' - severity: 'UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL' - format: 'sarif' - output: 'trivy-results.sarif' + image-ref: "docker.io/searxng/searxng:latest" + vuln-type: "os,library" + severity: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" + ignore-unfixed: "false" + format: "sarif" + output: "./trivy-results.sarif" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + - name: Upload SARIFs + uses: github/codeql-action/upload-sarif@v3 with: - sarif_file: 'trivy-results.sarif' + sarif_file: "./trivy-results.sarif" From ee28f8effca3e431a46464e0fad82a6b93fb4b90 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 4 May 2025 22:56:41 +0200 Subject: [PATCH 06/12] compat Dockerfile & misc fixes Again. Needed to test the workflows in a separate fork so unrelated triggers appear in files. Also, adds the new Dockerfile.compat for armv7. --- .github/workflows/checker.yml | 6 +- .github/workflows/container.yml | 23 ++-- .github/workflows/data-update.yml | 10 +- .github/workflows/documentation.yml | 65 +++++++++++ .github/workflows/integration.yml | 90 --------------- .github/workflows/l10n.yml | 130 ++++++++++++++++++++++ .github/workflows/security.yml | 4 + .github/workflows/translations-update.yml | 61 ---------- Dockerfile | 4 - Dockerfile.compat | 104 +++++++++++++++++ manage | 14 +-- 11 files changed, 331 insertions(+), 180 deletions(-) create mode 100644 .github/workflows/documentation.yml create mode 100644 .github/workflows/l10n.yml delete mode 100644 .github/workflows/translations-update.yml create mode 100644 Dockerfile.compat diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index 04639726b83..95e193d8ffb 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -14,8 +14,8 @@ env: PYTHON_VERSION: "3.13" jobs: - checker: - name: Checker + search: + name: Search runs-on: ubuntu-24.04-arm steps: - name: Setup Python @@ -38,6 +38,6 @@ jobs: - name: Setup venv run: make V=1 install - - name: Checker + - name: Search checker continue-on-error: true run: make search.checker diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 818d2bc6b29..2d369d73768 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -3,12 +3,12 @@ name: Container # yamllint disable-line rule:truthy on: + workflow_dispatch: workflow_run: workflows: - Integration types: - completed - # TODO: Test container.yml on merge https://github.com/searxng/searxng/pull/4699 branches: - master @@ -22,18 +22,22 @@ env: jobs: build: - if: github.event.workflow_run.conclusion == 'success' + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' name: Build (${{ matrix.arch }}) runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: include: - arch: amd64 os: ubuntu-24.04 - - arch: - arm64 - armv7 + emulation: false + - arch: arm64 os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true permissions: # Organization GHCR @@ -60,13 +64,14 @@ jobs: - name: Setup cache container mounts uses: actions/cache@v4 with: - key: "container-mounts-${{ hashFiles('./Dockerfile') }}" - restore-keys: "container-mounts-" + key: "container-mounts-${{ runner.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ runner.arch }}-" path: | /var/tmp/buildah-cache/ /var/tmp/buildah-cache-*/ - - name: Setup QEMU + - if: ${{ matrix.emulation }} + name: Setup QEMU uses: docker/setup-qemu-action@v3 - name: Login to GHCR @@ -89,6 +94,7 @@ jobs: runs-on: ubuntu-24.04-arm needs: build strategy: + fail-fast: false matrix: arch: - amd64 @@ -124,4 +130,5 @@ jobs: name: Release env: OVERRIDE_ARCH: "${{ matrix.arch }}" + # TODO: Try this with GHCR or testing registry before merge run: make -e GIT_URL=$(git remote get-url origin) ci.container.push diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 9665064abe6..62428a8d617 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -14,7 +14,7 @@ env: PYTHON_VERSION: "3.13" jobs: - updateData: + data: if: github.repository_owner == 'searxng' name: ${{ matrix.fetch }} runs-on: ubuntu-24.04-arm @@ -54,24 +54,22 @@ jobs: - name: Fetch data run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}" - - name: Create Pull Request + - name: Create PR id: cpr uses: peter-evans/create-pull-request@v7 with: author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" - committer: "searxng-bot " + committer: "searxng-bot " title: "[data] update searx.data - ${{ matrix.fetch }}" commit-message: "[data] update searx.data - ${{ matrix.fetch }}" branch: "update_data_${{ matrix.fetch }}" delete-branch: "true" draft: "false" signoff: "false" - body: | - update searx.data - ${{ matrix.fetch }} labels: | data - - name: Check outputs + - name: Display information run: | echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000000..fbf73879b20 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,65 @@ +--- +name: Documentation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + release: + # TODO: Uncomment + # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + if: false + name: Release + runs-on: ubuntu-24.04-arm + permissions: + # for JamesIves/github-pages-deploy-action to push + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Build documentation + run: make V=1 docs.clean docs.html + + - name: Release + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: "dist/docs" + branch: "gh-pages" + commit-message: "[doc] build from commit ${{ github.sha }}" + # Automatically remove deleted files from the deploy branch + clean: "true" + single-commit: "true" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d32a4fe9665..7de57345b4a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -80,93 +80,3 @@ jobs: - name: Build run: make themes.all - - documentation: - # TODO: Remove this - if: false - name: Documentation - runs-on: ubuntu-24.04-arm - permissions: - # for JamesIves/github-pages-deploy-action to push - contents: write - - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: "false" - fetch-depth: "0" - - - name: Setup cache Python - uses: actions/cache@v4 - with: - key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" - restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" - path: "./local/" - - - name: Setup venv - run: make V=1 install - - - name: Build documentation - run: make V=1 docs.clean docs.html - - - if: github.ref == 'refs/heads/master' - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: "dist/docs" - branch: "gh-pages" - commit-message: "[doc] build from commit ${{ github.sha }}" - # Automatically remove deleted files from the deploy branch - clean: "true" - single-commit: "true" - - babel: - if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' - name: Update translations branch - runs-on: ubuntu-24.04-arm - needs: - - test - - theme - - documentation - - permissions: - # For "make V=1 weblate.push.translations" - contents: write - - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Checkout - uses: actions/checkout@v4 - with: - token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" - fetch-depth: "0" - - - name: Setup cache Python - uses: actions/cache@v4 - with: - key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" - restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" - path: "./local/" - - - name: Setup venv - run: make V=1 install - - - name: weblate & git setup - run: | - mkdir -p ~/.config - echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - - name: Update transations - run: make V=1 weblate.push.translations diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml new file mode 100644 index 00000000000..4a49cfbfa0b --- /dev/null +++ b/.github/workflows/l10n.yml @@ -0,0 +1,130 @@ +--- +name: Translation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + schedule: + - cron: "05 07 * * 5" + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + update: + if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + name: Update + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.push.translations" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Update translations + run: make V=1 weblate.push.translations + + pr: + if: | + github.repository_owner == 'searxng' + && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') + name: Pull Request + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.translations.commit" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Merge and push translation updates + run: make V=1 weblate.translations.commit + + - name: Create PR + id: cpr + uses: peter-evans/create-pull-request@v7 + with: + author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" + committer: "searxng-bot " + title: "[l10n] update translations from Weblate" + commit-message: "[l10n] update translations from Weblate" + branch: "translations_update" + delete-branch: "true" + draft: "false" + signoff: "false" + labels: | + translation + + - name: Display information + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index a720e0df280..924d19fb82d 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -6,6 +6,10 @@ on: workflow_dispatch: schedule: - cron: "42 05 * * *" + # TODO: Remove this + pull_request: + branches: + - container-gha permissions: contents: read diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml deleted file mode 100644 index a6f7fa9cedc..00000000000 --- a/.github/workflows/translations-update.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Update translations - -# yamllint disable-line rule:truthy -on: - workflow_dispatch: - schedule: - - cron: "05 07 * * 5" - -jobs: - babel: - name: "create PR for additions from weblate" - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: weblate & git setup - env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} - run: | - mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - name: Merge and push transation updates - run: | - make V=1 weblate.translations.commit - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - commit-message: '[l10n] update translations from Weblate' - committer: searxng-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: translations_update - delete-branch: true - draft: false - title: '[l10n] update translations from Weblate' - body: | - update translations from Weblate - labels: | - translation diff --git a/Dockerfile b/Dockerfile index 9aeb2821475..f05d2a1fd3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ brotli \ - # lxml - libxml2-dev \ - libxslt1-dev \ - zlib1g-dev \ # uwsgi libpcre3-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.compat b/Dockerfile.compat new file mode 100644 index 00000000000..4b44d953775 --- /dev/null +++ b/Dockerfile.compat @@ -0,0 +1,104 @@ +FROM docker.io/library/python:3.13-slim AS builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + brotli \ + # lxml + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + # uwsgi + libpcre3-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/local/searxng/ + +COPY ./requirements.txt ./requirements.txt + +RUN --mount=type=cache,id=pip,target=$HOME/.cache/pip python -m venv ./venv \ + && . ./venv/bin/activate \ + && pip install -r requirements.txt \ + && pip install "uwsgi~=2.0" + +COPY ./searx/ ./searx/ + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 + +RUN python -m compileall -q searx \ + && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ + && touch -c --date=@$TIMESTAMP_UWSGI ./dockerfiles/uwsgi.ini \ + && find /usr/local/searxng/searx/static \ + \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ + -type f -exec gzip -9 -k {} + -exec brotli --best {} + + +ARG SEARXNG_UID=977 +ARG SEARXNG_GID=977 + +RUN grep -m1 root /etc/group > /tmp/.searxng.group \ + && grep -m1 root /etc/passwd > /tmp/.searxng.passwd \ + && echo "searxng:x:$SEARXNG_GID:" >> /tmp/.searxng.group \ + && echo "searxng:x:$SEARXNG_UID:$SEARXNG_GID:searxng:/usr/local/searxng:/bin/bash" >> /tmp/.searxng.passwd + +FROM docker.io/library/python:3.13-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # uwsgi + libpcre3 \ + libxml2 \ + mailcap \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=root:root --from=builder /tmp/.searxng.passwd /etc/passwd +COPY --chown=root:root --from=builder /tmp/.searxng.group /etc/group + +ARG LABEL_DATE="0001-01-01T00:00:00Z" +ARG GIT_URL="unspecified" +ARG SEARXNG_GIT_VERSION="unspecified" +ARG LABEL_VCS_REF="unspecified" +ARG LABEL_VCS_URL="unspecified" + +WORKDIR /usr/local/searxng/ + +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ +COPY --chown=searxng:searxng ./dockerfiles/ ./dockerfiles/ + +LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ + org.opencontainers.image.created=$LABEL_DATE \ + org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ + org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later" \ + org.opencontainers.image.revision=$LABEL_VCS_REF \ + org.opencontainers.image.source=$LABEL_VCS_URL \ + org.opencontainers.image.title="searxng" \ + org.opencontainers.image.url=$LABEL_VCS_URL \ + org.opencontainers.image.version=$SEARXNG_GIT_VERSION + +ENV CONFIG_PATH=/etc/searxng \ + DATA_PATH=/var/cache/searxng + +ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ + INSTANCE_NAME=searxng \ + AUTOCOMPLETE="" \ + BASE_URL="" \ + BIND_ADDRESS=[::]:8080 \ + MORTY_KEY="" \ + MORTY_URL="" \ + SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ + UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ + UWSGI_WORKERS=%k \ + UWSGI_THREADS=4 + +VOLUME $CONFIG_PATH +VOLUME $DATA_PATH + +EXPOSE 8080 + +USER searxng:searxng + +HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 + +ENTRYPOINT ["/usr/local/searxng/dockerfiles/docker-entrypoint.sh"] diff --git a/manage b/manage index 91f045d34a4..654a08ac052 100755 --- a/manage +++ b/manage @@ -178,8 +178,7 @@ container.build() { platform="linux/$arch" ;; "ARMV7" | "armhf" | "armv7l" | "armv7") - # TODO: Move ARMv7 to a separated Dockerfile - dockerfile="Dockerfile" + dockerfile="Dockerfile.compat" arch="arm" variant="v7" platform="linux/$arch/$variant" @@ -239,8 +238,8 @@ container.build() { params_build_builder="$docker_builder --platform=$platform --target=builder" params_build="$docker_builder --platform=$platform --squash" else - params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false" - params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false" + params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" fi # Define container image org/name @@ -407,8 +406,7 @@ ci.container.build() { platform="linux/$arch" ;; "ARMV7" | "armhf" | "armv7l" | "armv7") - # TODO: Move ARMv7 to a separated Dockerfile - dockerfile="Dockerfile" + dockerfile="Dockerfile.compat" arch="arm" variant="v7" platform="linux/$arch/$variant" @@ -459,7 +457,7 @@ ci.container.build() { build_msg CONTAINER "Building..." - podman build --platform="$platform" --target=builder --layers --identity-label=false \ + podman build --format=docker --platform="$platform" --target=builder --layers --identity-label=false \ --cache-from="ghcr.io/$container_image_organization/cache" \ --cache-to="ghcr.io/$container_image_organization/cache" \ --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ @@ -467,7 +465,7 @@ ci.container.build() { --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \ --file="./$dockerfile" - podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ + podman build --format=docker --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ --cache-from="ghcr.io/$container_image_organization/cache" \ --cache-to="ghcr.io/$container_image_organization/cache" \ --build-arg="GIT_URL=$GIT_URL" \ From b1ffeb00ac135dfde67b4177f5519d2e0f57f3ff Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Tue, 6 May 2025 11:32:59 +0200 Subject: [PATCH 07/12] add test job Needed to test the workflows in a separate fork so unrelated triggers appear in files. Run tests before container release, if there are any problems the job fails and the images are not released. At the moment this is basic, but it should cover the most serious issues that may appear. This will prevent fiascos like https://github.com/searxng/searxng/issues/4718 to ever happen again (hopefully) --- .github/workflows/container.yml | 63 ++++++++++++++-- .github/workflows/integration.yml | 118 +++++++++++++++++++++++++++++- .github/workflows/l10n.yml | 4 +- Dockerfile | 3 +- Makefile | 2 +- manage | 85 ++++++++++++++++++++- 6 files changed, 260 insertions(+), 15 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 2d369d73768..13ca7ffb2b7 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -22,7 +22,9 @@ env: jobs: build: - if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + # TODO: Uncomment + # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + if: false name: Build (${{ matrix.arch }}) runs-on: ${{ matrix.os }} strategy: @@ -53,6 +55,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" + # make sure "make ci.container.build" can get the git branches + fetch-depth: "0" - name: Setup cache Python uses: actions/cache@v4 @@ -64,8 +68,8 @@ jobs: - name: Setup cache container mounts uses: actions/cache@v4 with: - key: "container-mounts-${{ runner.arch }}-${{ hashFiles('./Dockerfile*') }}" - restore-keys: "container-mounts-${{ runner.arch }}-" + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" path: | /var/tmp/buildah-cache/ /var/tmp/buildah-cache-*/ @@ -84,7 +88,53 @@ jobs: - name: Build env: OVERRIDE_ARCH: "${{ matrix.arch }}" - run: make -e GIT_URL=$(git remote get-url origin) ci.container.build + run: make ci.container.build + + test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.test" can get the git branches + fetch-depth: "0" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make ci.container.test release: # TODO: Uncomment before merge @@ -92,7 +142,7 @@ jobs: if: false name: Release (${{ matrix.arch }}) runs-on: ubuntu-24.04-arm - needs: build + needs: test strategy: fail-fast: false matrix: @@ -130,5 +180,4 @@ jobs: name: Release env: OVERRIDE_ARCH: "${{ matrix.arch }}" - # TODO: Try this with GHCR or testing registry before merge - run: make -e GIT_URL=$(git remote get-url origin) ci.container.push + run: make ci.container.push diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7de57345b4a..25e9268b86c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ on: - master pull_request: branches: - - master + - container-gha permissions: contents: read @@ -80,3 +80,119 @@ jobs: - name: Build run: make themes.all + + # TODO: Remove this + container-build: + # TODO: Uncomment + # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.build" can get the git branches + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make ci.container.build + + # TODO: Remove this + container-test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: container-build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.test" can get the git branches + # fetch-depth: "0" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make ci.container.test diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml index 4a49cfbfa0b..6975ea32de2 100644 --- a/.github/workflows/l10n.yml +++ b/.github/workflows/l10n.yml @@ -22,7 +22,9 @@ env: jobs: update: - if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + # TODO: Uncomment + # if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + if: false name: Update runs-on: ubuntu-24.04-arm permissions: diff --git a/Dockerfile b/Dockerfile index f05d2a1fd3b..f31a1183b8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,7 @@ WORKDIR /usr/local/searxng/ COPY ./requirements.txt ./requirements.txt -# Readd on #4707 "--mount=type=cache,id=pip,target=/root/.cache/pip" -RUN python -m venv ./venv \ +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ && pip install "uwsgi~=2.0" diff --git a/Makefile b/Makefile index 97714358219..0b35621ccf3 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover MANAGE += themes.all themes.fix themes.test MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs -MANAGE += ci.container.build ci.container.push +MANAGE += ci.container.build ci.container.test ci.container.push PHONY += $(MANAGE) diff --git a/manage b/manage index 654a08ac052..7bdefc3123e 100755 --- a/manage +++ b/manage @@ -429,7 +429,7 @@ ci.container.build() { fi ( - set -eu + set -e pyenv.activate # Check if it is a git repository @@ -446,6 +446,13 @@ ci.container.build() { python -m searx.version freeze eval "$(python -m searx.version)" + # TODO: Remove this + echo $VERSION_STRING + echo $VERSION_TAG + echo $DOCKER_TAG + echo $GIT_URL + echo $GIT_BRANCH + # Get the last git commit id version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) build_msg CONTAINER "Last commit: $version_gitcommit" @@ -481,6 +488,77 @@ ci.container.build() { dump_return $? } +ci.container.test() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -e + + # Define container image org/name + git_url="$(git remote get-url origin)" + # shellcheck disable=SC2001 + container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + podman pull "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" + + name="$container_image_name-$(date +%N)" + + podman create --name="$name" --rm --timeout=60 --network="host" \ + "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" >/dev/null + + podman start "$name" >/dev/null + podman logs -f "$name" & + pid_logs=$! + + # Wait until container is ready + sleep 5 + + # TODO: Test failing + curl -vf --max-time 5 "http://localhost:8080/healthz" + + kill $pid_logs &>/dev/null || true + podman stop "$name" >/dev/null + ) + dump_return $? +} + ci.container.push() { if ! "$GITHUB_ACTIONS"; then die 1 "This command is intended to be run in GitHub Actions" @@ -521,11 +599,12 @@ ci.container.push() { fi ( - set -eu + set -e # Define container image org/name + git_url="$(git remote get-url origin)" # shellcheck disable=SC2001 - container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" From e37bd4481f7195af400f5b82137c7ab7a48c179a Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Wed, 7 May 2025 12:06:16 +0200 Subject: [PATCH 08/12] run workflows sequentially by default concurrency rules applies to each branch --- .github/workflows/checker.yml | 4 ++++ .github/workflows/container.yml | 4 ++++ .github/workflows/data-update.yml | 4 ++++ .github/workflows/documentation.yml | 4 ++++ .github/workflows/integration.yml | 4 ++++ .github/workflows/l10n.yml | 4 ++++ .github/workflows/security.yml | 4 ++++ 7 files changed, 28 insertions(+) diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index 95e193d8ffb..4e900dd4c54 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -7,6 +7,10 @@ on: schedule: - cron: "0 4 * * 5" +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 13ca7ffb2b7..15f9ba13607 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -12,6 +12,10 @@ on: branches: - master +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read # Organization GHCR diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 62428a8d617..f8c7eeccab1 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -7,6 +7,10 @@ on: schedule: - cron: "59 23 28 * *" +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index fbf73879b20..713ee40e79b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -12,6 +12,10 @@ on: branches: - master +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 25e9268b86c..34d22803a48 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -11,6 +11,10 @@ on: branches: - container-gha +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml index 6975ea32de2..690bc7ed13d 100644 --- a/.github/workflows/l10n.yml +++ b/.github/workflows/l10n.yml @@ -14,6 +14,10 @@ on: schedule: - cron: "05 07 * * 5" +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 924d19fb82d..3a435e199be 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -11,6 +11,10 @@ on: branches: - container-gha +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + permissions: contents: read From a311c4a2b0e606f8635a11f853a58a09f9073bbe Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Wed, 7 May 2025 12:25:47 +0200 Subject: [PATCH 09/12] fix getting branch on fork branches When running the version.py in CI, it only fetches a detached HEAD failing to get the reference. As I do not know very well if modifying the function will affect "normal" cases, I have preferred to use the variables provided by GHA to obtain the branch and the location of the repository. --- .github/workflows/container.yml | 6 ------ .github/workflows/integration.yml | 4 ---- manage | 7 ------- searx/version.py | 6 ++++++ 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 15f9ba13607..5e80ae8a059 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -59,8 +59,6 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" - # make sure "make ci.container.build" can get the git branches - fetch-depth: "0" - name: Setup cache Python uses: actions/cache@v4 @@ -121,8 +119,6 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" - # make sure "make ci.container.test" can get the git branches - fetch-depth: "0" - if: ${{ matrix.emulation }} name: Setup QEMU @@ -161,8 +157,6 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" - # make sure "make ci.container.push" can get the git history - fetch-depth: "0" - if: env.DOCKERHUB_USERNAME != '' name: Login to GHCR diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 34d22803a48..a64e7a07eef 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -119,8 +119,6 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" - # make sure "make ci.container.build" can get the git branches - fetch-depth: "0" - name: Setup cache Python uses: actions/cache@v4 @@ -182,8 +180,6 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" - # make sure "make ci.container.test" can get the git branches - # fetch-depth: "0" - if: ${{ matrix.emulation }} name: Setup QEMU diff --git a/manage b/manage index 7bdefc3123e..0c9feda543d 100755 --- a/manage +++ b/manage @@ -446,13 +446,6 @@ ci.container.build() { python -m searx.version freeze eval "$(python -m searx.version)" - # TODO: Remove this - echo $VERSION_STRING - echo $VERSION_TAG - echo $DOCKER_TAG - echo $GIT_URL - echo $GIT_BRANCH - # Get the last git commit id version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) build_msg CONTAINER "Last commit: $version_gitcommit" diff --git a/searx/version.py b/searx/version.py index d2013808ba4..565cc7e7abd 100644 --- a/searx/version.py +++ b/searx/version.py @@ -41,6 +41,12 @@ def subprocess_run(args, **kwargs): def get_git_url_and_branch(): + # handle GHA directly + if "GITHUB_REPOSITORY" in os.environ and "GITHUB_REF_NAME" in os.environ: + git_url = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}" + git_branch = os.environ["GITHUB_REF_NAME"] + return git_url, git_branch + try: ref = subprocess_run("git rev-parse --abbrev-ref @{upstream}") except subprocess.CalledProcessError: From be04143b2a66cd1dcd8ed484ec17fd57cad43052 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Wed, 7 May 2025 13:14:04 +0200 Subject: [PATCH 10/12] migrate image release process --- .github/workflows/container.yml | 14 ++-- manage | 113 ++++++++++++++++++++------------ 2 files changed, 75 insertions(+), 52 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 5e80ae8a059..9e73eba694b 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -134,23 +134,16 @@ jobs: - name: Test env: OVERRIDE_ARCH: "${{ matrix.arch }}" + GIT_URL: "${{ steps.build.outputs.git_url }}" run: make ci.container.test release: # TODO: Uncomment before merge # if: github.repository_owner == 'searxng' if: false - name: Release (${{ matrix.arch }}) + name: Release runs-on: ubuntu-24.04-arm needs: test - strategy: - fail-fast: false - matrix: - arch: - - amd64 - - arm64 - - armv7 - steps: - if: env.DOCKERHUB_USERNAME != '' name: Checkout @@ -177,5 +170,6 @@ jobs: - if: env.DOCKERHUB_USERNAME != '' name: Release env: - OVERRIDE_ARCH: "${{ matrix.arch }}" + GIT_URL: "${{ steps.build.outputs.git_url }}" + DOCKER_TAG: "${{ steps.build.outputs.docker_tag }}" run: make ci.container.push diff --git a/manage b/manage index 0c9feda543d..f0c53024ae8 100755 --- a/manage +++ b/manage @@ -477,6 +477,15 @@ ci.container.build() { --file="./$dockerfile" podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" + + # From version.py + { + echo "version_string=$VERSION_STRING" + echo "version_tag=$VERSION_TAG" + echo "docker_tag=$DOCKER_TAG" + echo "git_url=$GIT_URL" + echo "git_branch=$GIT_BRANCH" + } >>"$GITHUB_OUTPUT" ) dump_return $? } @@ -524,9 +533,8 @@ ci.container.test() { set -e # Define container image org/name - git_url="$(git remote get-url origin)" # shellcheck disable=SC2001 - container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" podman pull "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" @@ -543,7 +551,6 @@ ci.container.test() { # Wait until container is ready sleep 5 - # TODO: Test failing curl -vf --max-time 5 "http://localhost:8080/healthz" kill $pid_logs &>/dev/null || true @@ -557,34 +564,36 @@ ci.container.push() { die 1 "This command is intended to be run in GitHub Actions" fi - local parch=${OVERRIDE_ARCH:-$(uname -m)} - local arch - local variant - local platform - - # Setup arch specific - case $parch in - "X64" | "x86_64" | "amd64") - arch="amd64" - variant="" - platform="linux/$arch" - ;; - "ARM64" | "aarch64" | "arm64") - arch="arm64" - variant="" - platform="linux/$arch" - ;; - "ARMV7" | "armhf" | "armv7l" | "armv7") - arch="arm" - variant="v7" - platform="linux/$arch/$variant" - ;; - *) - err_msg "Unsupported architecture; (PARCH=\"$parch\")" - exit 1 - ;; - esac - build_msg CONTAINER "Selected platform: $platform" + # Architectures to release + local release_archs=("amd64" "arm64" "armv7") + + local archs=() + local variants=() + local platforms=() + + for arch in "${release_archs[@]}"; do + case $arch in + "X64" | "x86_64" | "amd64") + archs+=("amd64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARM64" | "aarch64" | "arm64") + archs+=("arm64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARMV7" | "armv7" | "armhf" | "arm") + archs+=("arm") + variants+=("v7") + platforms+=("linux/${archs[-1]}/${variants[-1]}") + ;; + *) + err_msg "Unsupported architecture; (ARCH=\"${arch[-1]}\")" + exit 1 + ;; + esac + done # Check if podman is installed if ! command -v podman &>/dev/null; then @@ -595,23 +604,43 @@ ci.container.push() { set -e # Define container image org/name - git_url="$(git remote get-url origin)" # shellcheck disable=SC2001 - container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" - podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" + # Pull archs + for i in "${!archs[@]}"; do + podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-${archs[$i]}${variants[$i]}" + done + + # Tags to release + tags=("latest") + tags+=("$DOCKER_TAG") + + # Create manifests + for tag in "${tags[@]}"; do + if ! podman manifest exists "localhost/$container_image_organization/$container_image_name:$tag"; then + podman manifest create "localhost/$container_image_organization/$container_image_name:$tag" + fi + + # Add archs to manifest + for i in "${!archs[@]}"; do + podman manifest add \ + "localhost/$container_image_organization/$container_image_name:$tag" \ + "containers-storage:ghcr.io/$container_image_organization/$container_image_name:cache-${archs[$i]}${variants[$i]}" + done + done - # Get version tag - container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \ - "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant") + podman image list - # Recreate tags - podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest" - podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" + # Push manifests + for tag in "${tags[@]}"; do + build_msg CONTAINER "Pushing manifest with tag: $tag" - podman push "docker.io/$container_image_organization/$container_image_name:latest" - podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" + podman manifest push \ + "localhost/$container_image_organization/$container_image_name:$tag" \ + "docker://docker.io/$container_image_organization/$container_image_name:$tag" + done ) dump_return $? } From 4314b0e0f26285e5c98d98ecca71c45b52ceb8e1 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Wed, 7 May 2025 13:15:25 +0200 Subject: [PATCH 11/12] cleanup preparation for initial review, moves all container stuff to lib_sxng_container.sh and reverts changes from fork CI testing --- .github/workflows/checker.yml | 1 - .github/workflows/container.yml | 33 ++- .github/workflows/documentation.yml | 4 +- .github/workflows/integration.yml | 119 +------- .github/workflows/l10n.yml | 4 +- Dockerfile | 2 - Dockerfile.compat | 8 +- Makefile | 7 +- manage | 404 +--------------------------- utils/lib_sxng_container.sh | 319 ++++++++++++++++++++++ 10 files changed, 362 insertions(+), 539 deletions(-) create mode 100755 utils/lib_sxng_container.sh diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index 4e900dd4c54..2de3c9896ce 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -43,5 +43,4 @@ jobs: run: make V=1 install - name: Search checker - continue-on-error: true run: make search.checker diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 9e73eba694b..1cf77827b3a 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -26,9 +26,7 @@ env: jobs: build: - # TODO: Uncomment - # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' - if: false + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' name: Build (${{ matrix.arch }}) runs-on: ${{ matrix.os }} strategy: @@ -49,6 +47,13 @@ jobs: # Organization GHCR packages: write + outputs: + version_string: ${{ steps.build.outputs.version_string }} + version_tag: ${{ steps.build.outputs.version_tag }} + docker_tag: ${{ steps.build.outputs.docker_tag }} + git_url: ${{ steps.build.outputs.git_url }} + git_branch: ${{ steps.build.outputs.git_branch }} + steps: - name: Setup Python uses: actions/setup-python@v5 @@ -88,9 +93,10 @@ jobs: password: "${{ secrets.GITHUB_TOKEN }}" - name: Build + id: build env: OVERRIDE_ARCH: "${{ matrix.arch }}" - run: make ci.container.build + run: make podman.build test: name: Test (${{ matrix.arch }}) @@ -134,16 +140,17 @@ jobs: - name: Test env: OVERRIDE_ARCH: "${{ matrix.arch }}" - GIT_URL: "${{ steps.build.outputs.git_url }}" - run: make ci.container.test + GIT_URL: "${{ needs.build.outputs.git_url }}" + run: make container.test release: - # TODO: Uncomment before merge - # if: github.repository_owner == 'searxng' - if: false + if: github.repository_owner == 'searxng' && github.ref_name == 'master' name: Release runs-on: ubuntu-24.04-arm - needs: test + needs: + - build + - test + steps: - if: env.DOCKERHUB_USERNAME != '' name: Checkout @@ -170,6 +177,6 @@ jobs: - if: env.DOCKERHUB_USERNAME != '' name: Release env: - GIT_URL: "${{ steps.build.outputs.git_url }}" - DOCKER_TAG: "${{ steps.build.outputs.docker_tag }}" - run: make ci.container.push + GIT_URL: "${{ needs.build.outputs.git_url }}" + DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}" + run: make container.push diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 713ee40e79b..6d9f2cac583 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -24,9 +24,7 @@ env: jobs: release: - # TODO: Uncomment - # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' - if: false + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' name: Release runs-on: ubuntu-24.04-arm permissions: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a64e7a07eef..52ce01663ad 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ on: - master pull_request: branches: - - container-gha + - master concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -23,8 +23,6 @@ env: jobs: test: - # TODO: Remove this - if: false name: Python ${{ matrix.python-version }} runs-on: ubuntu-24.04 strategy: @@ -61,11 +59,14 @@ jobs: run: make V=1 ci.test theme: - # TODO: Remove this - if: false name: Theme runs-on: ubuntu-24.04-arm steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 with: @@ -82,44 +83,6 @@ jobs: key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}" path: "./client/simple/node_modules/" - - name: Build - run: make themes.all - - # TODO: Remove this - container-build: - # TODO: Uncomment - # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' - name: Build (${{ matrix.arch }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - arch: amd64 - os: ubuntu-24.04 - emulation: false - - arch: arm64 - os: ubuntu-24.04-arm - emulation: false - - arch: armv7 - os: ubuntu-24.04-arm - emulation: true - - permissions: - # Organization GHCR - packages: write - - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: "false" - - name: Setup cache Python uses: actions/cache@v4 with: @@ -127,72 +90,8 @@ jobs: restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" path: "./local/" - - name: Setup cache container mounts - uses: actions/cache@v4 - with: - key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./Dockerfile*') }}" - restore-keys: "container-mounts-${{ matrix.arch }}-" - path: | - /var/tmp/buildah-cache/ - /var/tmp/buildah-cache-*/ - - - if: ${{ matrix.emulation }} - name: Setup QEMU - uses: docker/setup-qemu-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: "ghcr.io" - username: "${{ github.repository_owner }}" - password: "${{ secrets.GITHUB_TOKEN }}" + - name: Setup venv + run: make V=1 install - name: Build - env: - OVERRIDE_ARCH: "${{ matrix.arch }}" - run: make ci.container.build - - # TODO: Remove this - container-test: - name: Test (${{ matrix.arch }}) - runs-on: ${{ matrix.os }} - needs: container-build - strategy: - fail-fast: false - matrix: - include: - - arch: amd64 - os: ubuntu-24.04 - emulation: false - - arch: arm64 - os: ubuntu-24.04-arm - emulation: false - - arch: armv7 - os: ubuntu-24.04-arm - emulation: true - - permissions: - # Organization GHCR - packages: write - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: "false" - - - if: ${{ matrix.emulation }} - name: Setup QEMU - uses: docker/setup-qemu-action@v3 - - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: "ghcr.io" - username: "${{ github.repository_owner }}" - password: "${{ secrets.GITHUB_TOKEN }}" - - - name: Test - env: - OVERRIDE_ARCH: "${{ matrix.arch }}" - run: make ci.container.test + run: make themes.all diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml index 690bc7ed13d..9fa97c7f52f 100644 --- a/.github/workflows/l10n.yml +++ b/.github/workflows/l10n.yml @@ -26,9 +26,7 @@ env: jobs: update: - # TODO: Uncomment - # if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' - if: false + if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' name: Update runs-on: ubuntu-24.04-arm permissions: diff --git a/Dockerfile b/Dockerfile index f31a1183b8e..92dfb49394a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,8 +43,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ # healthcheck wget \ - # lxml (ARMv7) - libxslt1.1 \ # uwsgi libpcre3 \ libxml2 \ diff --git a/Dockerfile.compat b/Dockerfile.compat index 4b44d953775..471b8bfdd1f 100644 --- a/Dockerfile.compat +++ b/Dockerfile.compat @@ -16,7 +16,7 @@ WORKDIR /usr/local/searxng/ COPY ./requirements.txt ./requirements.txt -RUN --mount=type=cache,id=pip,target=$HOME/.cache/pip python -m venv ./venv \ +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ && pip install "uwsgi~=2.0" @@ -45,6 +45,10 @@ FROM docker.io/library/python:3.13-slim RUN apt-get update \ && apt-get install -y --no-install-recommends \ + # healthcheck + wget \ + # lxml (ARMv7) + libxslt1.1 \ # uwsgi libpcre3 \ libxml2 \ @@ -97,8 +101,6 @@ VOLUME $DATA_PATH EXPOSE 8080 -USER searxng:searxng - HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 ENTRYPOINT ["/usr/local/searxng/dockerfiles/docker-entrypoint.sh"] diff --git a/Makefile b/Makefile index 0b35621ccf3..8c20eed35a2 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,9 @@ test.shell: MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean +MANAGE += podman.build MANAGE += docker.build docker.buildx -MANAGE += container.build container.buildx +MANAGE += container.build container.test container.push MANAGE += gecko.driver MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean @@ -88,7 +89,6 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover MANAGE += themes.all themes.fix themes.test MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs -MANAGE += ci.container.build ci.container.test ci.container.push PHONY += $(MANAGE) @@ -97,9 +97,8 @@ $(MANAGE): # short hands of selected targets -PHONY += docs docker themes +PHONY += docs container themes docs: docs.html container: container.build -docker: container.build themes: themes.all diff --git a/manage b/manage index f0c53024ae8..3e431a4e750 100755 --- a/manage +++ b/manage @@ -11,6 +11,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" +# shellcheck source=utils/lib_sxng_container.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_container.sh" + # shellcheck source=utils/lib_sxng_data.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh" @@ -77,9 +80,6 @@ docs.: gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build -container. : - build : build container image - buildx : build container image with BuildKit gecko.driver: download & install geckodriver if not already installed (required for robot_tests) @@ -97,14 +97,11 @@ pyenv.: OK : test if virtualenv is OK format.: python : format Python code source using black -ci.: - container.: - build : build container image - release : push container images to remote registry EOF go.help node.help weblate.help + container.help data.help test.help themes.help @@ -144,132 +141,6 @@ docker.buildx() { container.buildx } -# Alias -docker.build() { - container.build -} - -container.buildx() { - container.build buildx -} - -container.build() { - pyenv.install - - local parch=${OVERRIDE_ARCH:-$(uname -m)} - local container_engine - local dockerfile - local arch - local variant - local platform - - # Setup arch specific - case $parch in - "X64" | "x86_64" | "amd64") - dockerfile="Dockerfile" - arch="amd64" - variant="" - platform="linux/$arch" - ;; - "ARM64" | "aarch64" | "arm64") - dockerfile="Dockerfile" - arch="arm64" - variant="" - platform="linux/$arch" - ;; - "ARMV7" | "armhf" | "armv7l" | "armv7") - dockerfile="Dockerfile.compat" - arch="arm" - variant="v7" - platform="linux/$arch/$variant" - ;; - *) - err_msg "Unsupported architecture; (PARCH=\"$parch\")" - exit 1 - ;; - esac - build_msg CONTAINER "Selected platform: $platform" - - # Check if podman or docker is installed - if command -v podman &>/dev/null; then - container_engine="podman" - elif command -v docker &>/dev/null; then - container_engine="docker" - else - die 1 "podman/docker is not installed" - fi - build_msg CONTAINER "Engine: $container_engine" - - # Check if git is installed - if ! command -v git &>/dev/null; then - die 1 "git is not installed" - fi - - ( - set -e - pyenv.activate - - # Check if it is a git repository - if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - - if ! git remote get-url origin 2>/dev/null; then - die 1 "there is no remote origin" - fi - - # This is a git repository - git update-index -q --refresh - python -m searx.version freeze - eval "$(python -m searx.version)" - - # Get the last git commit id - version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg CONTAINER "Last commit: $version_gitcommit" - - if [ "$container_engine" = "docker" ]; then - if [ "$1" = "buildx" ]; then - docker_builder="buildx build" - else - docker_builder="build" - warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback" - fi - - params_build_builder="$docker_builder --platform=$platform --target=builder" - params_build="$docker_builder --platform=$platform --squash" - else - params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" - params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" - fi - - # Define container image org/name - # shellcheck disable=SC2001 - container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" - container_image_name="searxng" - - build_msg CONTAINER "Building..." - - # shellcheck disable=SC2086 - "$container_engine" $params_build_builder \ - --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ - --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ - --tag="localhost/$container_image_organization/$container_image_name:builder" \ - --file="./$dockerfile" - - # shellcheck disable=SC2086 - "$container_engine" $params_build \ - --build-arg="GIT_URL=$GIT_URL" \ - --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ - --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ - --build-arg="LABEL_VCS_URL=$GIT_URL" \ - --tag="localhost/$container_image_organization/$container_image_name:latest" \ - --tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \ - --file="./$dockerfile" - ) - dump_return $? -} - # shellcheck disable=SC2119 gecko.driver() { pyenv.install @@ -378,273 +249,6 @@ docs.prebuild() { dump_return $? } -ci.container.build() { - if ! "$GITHUB_ACTIONS"; then - die 1 "This command is intended to be run in GitHub Actions" - fi - - pyenv.install - - local parch=${OVERRIDE_ARCH:-$(uname -m)} - local dockerfile - local arch - local variant - local platform - - # Setup arch specific - case $parch in - "X64" | "x86_64" | "amd64") - dockerfile="Dockerfile" - arch="amd64" - variant="" - platform="linux/$arch" - ;; - "ARM64" | "aarch64" | "arm64") - dockerfile="Dockerfile" - arch="arm64" - variant="" - platform="linux/$arch" - ;; - "ARMV7" | "armhf" | "armv7l" | "armv7") - dockerfile="Dockerfile.compat" - arch="arm" - variant="v7" - platform="linux/$arch/$variant" - ;; - *) - err_msg "Unsupported architecture; (PARCH=\"$parch\")" - exit 1 - ;; - esac - build_msg CONTAINER "Selected platform: $platform" - - # Check if podman is installed - if ! command -v podman &>/dev/null; then - die 1 "podman is not installed" - fi - - # Check if git is installed - if ! command -v git &>/dev/null; then - die 1 "git is not installed" - fi - - ( - set -e - pyenv.activate - - # Check if it is a git repository - if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - - if ! git remote get-url origin 2>/dev/null; then - die 1 "there is no remote origin" - fi - - # This is a git repository - git update-index -q --refresh - python -m searx.version freeze - eval "$(python -m searx.version)" - - # Get the last git commit id - version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg CONTAINER "Last commit: $version_gitcommit" - - # Define container image org/name - # shellcheck disable=SC2001 - container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" - container_image_name="searxng" - - build_msg CONTAINER "Building..." - - podman build --format=docker --platform="$platform" --target=builder --layers --identity-label=false \ - --cache-from="ghcr.io/$container_image_organization/cache" \ - --cache-to="ghcr.io/$container_image_organization/cache" \ - --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ - --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ - --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \ - --file="./$dockerfile" - - podman build --format=docker --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ - --cache-from="ghcr.io/$container_image_organization/cache" \ - --cache-to="ghcr.io/$container_image_organization/cache" \ - --build-arg="GIT_URL=$GIT_URL" \ - --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ - --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ - --build-arg="LABEL_VCS_URL=$GIT_URL" \ - --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \ - --file="./$dockerfile" - - podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" - - # From version.py - { - echo "version_string=$VERSION_STRING" - echo "version_tag=$VERSION_TAG" - echo "docker_tag=$DOCKER_TAG" - echo "git_url=$GIT_URL" - echo "git_branch=$GIT_BRANCH" - } >>"$GITHUB_OUTPUT" - ) - dump_return $? -} - -ci.container.test() { - if ! "$GITHUB_ACTIONS"; then - die 1 "This command is intended to be run in GitHub Actions" - fi - - local parch=${OVERRIDE_ARCH:-$(uname -m)} - local arch - local variant - local platform - - # Setup arch specific - case $parch in - "X64" | "x86_64" | "amd64") - arch="amd64" - variant="" - platform="linux/$arch" - ;; - "ARM64" | "aarch64" | "arm64") - arch="arm64" - variant="" - platform="linux/$arch" - ;; - "ARMV7" | "armhf" | "armv7l" | "armv7") - arch="arm" - variant="v7" - platform="linux/$arch/$variant" - ;; - *) - err_msg "Unsupported architecture; (PARCH=\"$parch\")" - exit 1 - ;; - esac - build_msg CONTAINER "Selected platform: $platform" - - # Check if podman is installed - if ! command -v podman &>/dev/null; then - die 1 "podman is not installed" - fi - - ( - set -e - - # Define container image org/name - # shellcheck disable=SC2001 - container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" - container_image_name="searxng" - - podman pull "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" - - name="$container_image_name-$(date +%N)" - - podman create --name="$name" --rm --timeout=60 --network="host" \ - "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" >/dev/null - - podman start "$name" >/dev/null - podman logs -f "$name" & - pid_logs=$! - - # Wait until container is ready - sleep 5 - - curl -vf --max-time 5 "http://localhost:8080/healthz" - - kill $pid_logs &>/dev/null || true - podman stop "$name" >/dev/null - ) - dump_return $? -} - -ci.container.push() { - if ! "$GITHUB_ACTIONS"; then - die 1 "This command is intended to be run in GitHub Actions" - fi - - # Architectures to release - local release_archs=("amd64" "arm64" "armv7") - - local archs=() - local variants=() - local platforms=() - - for arch in "${release_archs[@]}"; do - case $arch in - "X64" | "x86_64" | "amd64") - archs+=("amd64") - variants+=("") - platforms+=("linux/${archs[-1]}") - ;; - "ARM64" | "aarch64" | "arm64") - archs+=("arm64") - variants+=("") - platforms+=("linux/${archs[-1]}") - ;; - "ARMV7" | "armv7" | "armhf" | "arm") - archs+=("arm") - variants+=("v7") - platforms+=("linux/${archs[-1]}/${variants[-1]}") - ;; - *) - err_msg "Unsupported architecture; (ARCH=\"${arch[-1]}\")" - exit 1 - ;; - esac - done - - # Check if podman is installed - if ! command -v podman &>/dev/null; then - die 1 "podman is not installed" - fi - - ( - set -e - - # Define container image org/name - # shellcheck disable=SC2001 - container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" - container_image_name="searxng" - - # Pull archs - for i in "${!archs[@]}"; do - podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-${archs[$i]}${variants[$i]}" - done - - # Tags to release - tags=("latest") - tags+=("$DOCKER_TAG") - - # Create manifests - for tag in "${tags[@]}"; do - if ! podman manifest exists "localhost/$container_image_organization/$container_image_name:$tag"; then - podman manifest create "localhost/$container_image_organization/$container_image_name:$tag" - fi - - # Add archs to manifest - for i in "${!archs[@]}"; do - podman manifest add \ - "localhost/$container_image_organization/$container_image_name:$tag" \ - "containers-storage:ghcr.io/$container_image_organization/$container_image_name:cache-${archs[$i]}${variants[$i]}" - done - done - - podman image list - - # Push manifests - for tag in "${tags[@]}"; do - build_msg CONTAINER "Pushing manifest with tag: $tag" - - podman manifest push \ - "localhost/$container_image_organization/$container_image_name:$tag" \ - "docker://docker.io/$container_image_organization/$container_image_name:$tag" - done - ) - dump_return $? -} - # shellcheck disable=SC2119 main() { local _type diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh new file mode 100755 index 00000000000..325f1a7e70b --- /dev/null +++ b/utils/lib_sxng_container.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +container.help() { + cat </dev/null; then + container_engine="docker" + else + die 1 "Docker is not installed" + fi + elif [ "$1" = "podman" ]; then + if command -v podman &>/dev/null; then + container_engine="podman" + else + die 1 "Podman is not installed" + fi + else + # If no explicit engine is passed, prioritize podman over docker + if command -v podman &>/dev/null; then + container_engine="podman" + elif command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "Podman/Docker is not installed" + fi + fi + info_msg "Selected engine: $container_engine" + + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "Git is not installed" + fi + + ( + set -e + pyenv.activate + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + + if ! git remote get-url origin &>/dev/null; then + die 1 "There is no remote origin" + fi + + # This is a git repository + git update-index -q --refresh + python -m searx.version freeze + eval "$(python -m searx.version)" + + info_msg "Set \$VERSION_STRING: $VERSION_STRING" + info_msg "Set \$VERSION_TAG: $VERSION_TAG" + info_msg "Set \$DOCKER_TAG: $DOCKER_TAG" + info_msg "Set \$GIT_URL: $GIT_URL" + info_msg "Set \$GIT_BRANCH: $GIT_BRANCH" + + if [ "$container_engine" = "podman" ]; then + params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" + else + params_build_builder="build --platform=$platform --target=builder" + params_build="build --platform=$platform --squash" + fi + + if [ "$GITHUB_ACTIONS" = "true" ]; then + params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + params_build+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + + # Tags + params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + else + # Tags + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest" + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG" + fi + + # shellcheck disable=SC2086 + "$container_engine" $params_build_builder \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ + --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \ + --file="./$dockerfile" \ + . + build_msg CONTAINER "Image \"builder\" built" + + # shellcheck disable=SC2086 + "$container_engine" $params_build \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --file="./$dockerfile" \ + . + build_msg CONTAINER "Image built" + + if [ "$GITHUB_ACTIONS" = "true" ]; then + "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + # Output to GHA + { + echo "version_string=$VERSION_STRING" + echo "version_tag=$VERSION_TAG" + echo "docker_tag=$DOCKER_TAG" + echo "git_url=$GIT_URL" + echo "git_branch=$GIT_BRANCH" + } >>"$GITHUB_OUTPUT" + fi + ) + dump_return $? +} + +container.test() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -e + + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + name="$CONTAINER_IMAGE_NAME-$(date +%N)" + + podman create --name="$name" --rm --timeout=60 --network="host" \ + "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null + + podman start "$name" >/dev/null + podman logs -f "$name" & + pid_logs=$! + + # Wait until container is ready + sleep 5 + + curl -vf --max-time 5 "http://localhost:8080/healthz" + + kill $pid_logs &>/dev/null || true + podman stop "$name" >/dev/null + ) + dump_return $? +} + +container.push() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + # Architectures to release + local release_archs=("amd64" "arm64" "armv7") + + local archs=() + local variants=() + local platforms=() + + for arch in "${release_archs[@]}"; do + case $arch in + "X64" | "x86_64" | "amd64") + archs+=("amd64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARM64" | "aarch64" | "arm64") + archs+=("arm64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARMV7" | "armv7" | "armhf" | "arm") + archs+=("arm") + variants+=("v7") + platforms+=("linux/${archs[-1]}/${variants[-1]}") + ;; + *) + err_msg "Unsupported architecture; (ARCH=\"${arch[-1]}\")" + exit 1 + ;; + esac + done + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -e + + # Pull archs + for i in "${!archs[@]}"; do + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + + # Tags to release + tags=("latest") + tags+=("$DOCKER_TAG") + + # Create manifests + for tag in "${tags[@]}"; do + if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then + podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + fi + + # Add archs to manifest + for i in "${!archs[@]}"; do + podman manifest add \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + done + + podman image list + + # Push manifests + for tag in "${tags[@]}"; do + build_msg CONTAINER "Pushing manifest with tag: $tag" + + podman manifest push \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "docker://docker.io/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + done + ) + dump_return $? +} + +# Alias +podman.build() { + container.build podman +} + +# Alias +docker.build() { + container.build docker +} + +# Alias +docker.buildx() { + container.build docker +} From af5a3dbc329a4d5fa4693f6b6d2bd5b6b6fc7ba5 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Wed, 7 May 2025 23:40:23 +0200 Subject: [PATCH 12/12] copilot review changes https://github.com/searxng/searxng/pull/4707#discussion_r2078529275 https://github.com/searxng/searxng/pull/4707#discussion_r2078529280 From Copilot review, manually tested --- utils/lib_sxng_container.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh index 325f1a7e70b..f6ffd5d2e2e 100755 --- a/utils/lib_sxng_container.sh +++ b/utils/lib_sxng_container.sh @@ -159,7 +159,7 @@ container.build() { } container.test() { - if ! "$GITHUB_ACTIONS"; then + if [ "$GITHUB_ACTIONS" != "true" ]; then die 1 "This command is intended to be run in GitHub Actions" fi @@ -186,7 +186,7 @@ container.test() { platform="linux/$arch/$variant" ;; *) - err_msg "Unsupported architecture; (PARCH=\"$parch\")" + err_msg "Unsupported architecture; $parch" exit 1 ;; esac @@ -223,7 +223,7 @@ container.test() { } container.push() { - if ! "$GITHUB_ACTIONS"; then + if [ "$GITHUB_ACTIONS" != "true" ]; then die 1 "This command is intended to be run in GitHub Actions" fi @@ -252,7 +252,7 @@ container.push() { platforms+=("linux/${archs[-1]}/${variants[-1]}") ;; *) - err_msg "Unsupported architecture; (ARCH=\"${arch[-1]}\")" + err_msg "Unsupported architecture; $arch" exit 1 ;; esac