diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index d0074712d3b..2de3c9896ce 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -1,31 +1,46 @@ -name: "Checker" -on: # yamllint disable-line rule:truthy +--- +name: Checker + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "0 4 * * 5" - workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" jobs: - checker: - name: Checker - runs-on: ubuntu-24.04 + search: + name: Search + runs-on: ubuntu-24.04-arm steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" - - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup cache Python + uses: actions/cache@v4 with: - python-version: '3.13' - architecture: 'x64' + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local" - - name: Install Python dependencies - run: | - make V=1 install + - name: Setup venv + run: make V=1 install - - name: Checker - run: | - make search.checker + - name: Search checker + run: make search.checker diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 00000000000..1cf77827b3a --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,182 @@ +--- +name: Container + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + # Organization GHCR + packages: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + build: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + outputs: + version_string: ${{ steps.build.outputs.version_string }} + version_tag: ${{ steps.build.outputs.version_tag }} + docker_tag: ${{ steps.build.outputs.docker_tag }} + git_url: ${{ steps.build.outputs.git_url }} + git_branch: ${{ steps.build.outputs.git_branch }} + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + id: build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make podman.build + + test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + GIT_URL: "${{ needs.build.outputs.git_url }}" + run: make container.test + + release: + if: github.repository_owner == 'searxng' && github.ref_name == 'master' + name: Release + runs-on: ubuntu-24.04-arm + needs: + - build + - test + + steps: + - if: env.DOCKERHUB_USERNAME != '' + name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: "docker.io" + username: "${{ env.DOCKERHUB_USERNAME }}" + password: "${{ secrets.DOCKERHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Release + env: + GIT_URL: "${{ needs.build.outputs.git_url }}" + DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}" + run: make container.push diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 39893127d5a..f8c7eeccab1 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -1,14 +1,27 @@ -name: "Update searx.data" -on: # yamllint disable-line rule:truthy +--- +name: Update searx.data + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "59 23 28 * *" - workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" jobs: - updateData: - name: Update data - ${{ matrix.fetch }} - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng'}} + data: + if: github.repository_owner == 'searxng' + name: ${{ matrix.fetch }} + runs-on: ubuntu-24.04-arm strategy: fail-fast: false matrix: @@ -20,48 +33,47 @@ jobs: - update_engine_traits.py - update_wikidata_units.py - update_engine_descriptions.py + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" - - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup cache Python + uses: actions/cache@v4 with: - python-version: '3.12' - architecture: 'x64' + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" - - name: Install Python dependencies - run: | - make V=1 install + - name: Setup venv + run: make V=1 install - name: Fetch data - env: - FETCH_SCRIPT: ./searxng_extra/update/${{ matrix.fetch }} - run: | - V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT" + run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}" - - name: Create Pull Request + - name: Create PR id: cpr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: - commit-message: '[data] update searx.data - ${{ matrix.fetch }}' - committer: searxng-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: update_data_${{ matrix.fetch }} - delete-branch: true - draft: false - title: '[data] update searx.data - ${{ matrix.fetch }}' - body: | - update searx.data - ${{ matrix.fetch }} + author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" + committer: "searxng-bot " + title: "[data] update searx.data - ${{ matrix.fetch }}" + commit-message: "[data] update searx.data - ${{ matrix.fetch }}" + branch: "update_data_${{ matrix.fetch }}" + delete-branch: "true" + draft: "false" + signoff: "false" labels: | data - - name: Check outputs + - name: Display information run: | echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000000..6d9f2cac583 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,67 @@ +--- +name: Documentation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + release: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + name: Release + runs-on: ubuntu-24.04-arm + permissions: + # for JamesIves/github-pages-deploy-action to push + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Build documentation + run: make V=1 docs.clean docs.html + + - name: Release + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: "dist/docs" + branch: "gh-pages" + commit-message: "[doc] build from commit ${{ github.sha }}" + # Automatically remove deleted files from the deploy branch + clean: "true" + single-commit: "true" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ddc0498f762..52ce01663ad 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,179 +1,97 @@ +--- name: Integration -on: # yamllint disable-line rule:truthy +# yamllint disable-line rule:truthy +on: + workflow_dispatch: push: - branches: ["master"] + branches: + - master pull_request: - branches: ["master"] + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false permissions: contents: read +env: + PYTHON_VERSION: "3.13" + jobs: - python: + test: name: Python ${{ matrix.python-version }} runs-on: ubuntu-24.04 strategy: matrix: - os: [ubuntu-24.04] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - - name: Run tests - run: make V=1 ci.test + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" - themes: - name: Themes - runs-on: ubuntu-24.04 steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Ubuntu packages - run: sudo ./utils/searxng.sh install buildhost - - name: Set up Python + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' - - name: Build themes - run: make themes.all + python-version: "${{ matrix.python-version }}" - documentation: - name: Documentation - runs-on: ubuntu-24.04 - permissions: - contents: write # for JamesIves/github-pages-deploy-action to push changes in repo - steps: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' - persist-credentials: false - - name: Install Ubuntu packages - run: sudo ./utils/searxng.sh install buildhost - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python + persist-credentials: "false" + + - name: Setup cache Python uses: actions/cache@v4 with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: Build documentation - run: | - make V=1 docs.clean docs.html - - name: Deploy - if: github.ref == 'refs/heads/master' - uses: JamesIves/github-pages-deploy-action@3.7.1 - with: - GITHUB_TOKEN: ${{ github.token }} - BRANCH: gh-pages - FOLDER: dist/docs - CLEAN: true # Automatically remove deleted files from the deploy branch - SINGLE_COMMIT: true - COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' - - babel: - name: Update translations branch - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} - needs: - - python - - themes - - documentation - permissions: - contents: write # for make V=1 weblate.push.translations + key: "python-${{ matrix.python-version }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ matrix.python-version }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Run tests + run: make V=1 ci.test + + theme: + name: Theme + runs-on: ubuntu-24.04-arm steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - - name: Set up Python + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: weblate & git setup - env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} - run: | - mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - name: Update transations - id: update - run: | - make V=1 weblate.push.translations - - dockers: - name: Docker - if: github.ref == 'refs/heads/master' - needs: - - python - - themes - - documentation - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - runs-on: ubuntu-24.04 - steps: + python-version: "${{ env.PYTHON_VERSION }}" + - name: Checkout - if: env.DOCKERHUB_USERNAME != null uses: actions/checkout@v4 with: - # make sure "make docker.push" can get the git history - fetch-depth: '0' - - name: Set up Python - uses: actions/setup-python@v5 + persist-credentials: "false" + + - name: Setup Node.js + uses: actions/setup-node@v4 with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python + node-version-file: "./.nvmrc" + + - name: Setup cache Node.js uses: actions/cache@v4 with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: Set up QEMU - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - if: env.DOCKERHUB_USERNAME != null - uses: docker/login-action@v1 + key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}" + path: "./client/simple/node_modules/" + + - name: Setup cache Python + uses: actions/cache@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - if: env.DOCKERHUB_USERNAME != null - run: make -e GIT_URL=$(git remote get-url origin) docker.buildx + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Build + run: make themes.all diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml new file mode 100644 index 00000000000..9fa97c7f52f --- /dev/null +++ b/.github/workflows/l10n.yml @@ -0,0 +1,134 @@ +--- +name: Translation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + schedule: + - cron: "05 07 * * 5" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + update: + if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + name: Update + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.push.translations" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Update translations + run: make V=1 weblate.push.translations + + pr: + if: | + github.repository_owner == 'searxng' + && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') + name: Pull Request + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.translations.commit" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Merge and push translation updates + run: make V=1 weblate.translations.commit + + - name: Create PR + id: cpr + uses: peter-evans/create-pull-request@v7 + with: + author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" + committer: "searxng-bot " + title: "[l10n] update translations from Weblate" + commit-message: "[l10n] update translations from Weblate" + branch: "translations_update" + delete-branch: "true" + draft: "false" + signoff: "false" + labels: | + translation + + - name: Display information + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 249db305b64..3a435e199be 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,28 +1,44 @@ -name: "Security checks" -on: # yamllint disable-line rule:truthy +--- +name: Security + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "42 05 * * *" - workflow_dispatch: + # TODO: Remove this + pull_request: + branches: + - container-gha + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read jobs: - dockers: - name: Trivy ${{ matrix.image }} - runs-on: ubuntu-24.04 + container: + name: Container + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master + - name: Run Trivy scanner + uses: aquasecurity/trivy-action@0.30.0 with: - image-ref: 'searxng/searxng:latest' - ignore-unfixed: false - vuln-type: 'os,library' - severity: 'UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL' - format: 'sarif' - output: 'trivy-results.sarif' + image-ref: "docker.io/searxng/searxng:latest" + vuln-type: "os,library" + severity: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" + ignore-unfixed: "false" + format: "sarif" + output: "./trivy-results.sarif" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + - name: Upload SARIFs + uses: github/codeql-action/upload-sarif@v3 with: - sarif_file: 'trivy-results.sarif' + sarif_file: "./trivy-results.sarif" diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml deleted file mode 100644 index 85e141e7f88..00000000000 --- a/.github/workflows/translations-update.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Update translations" -on: # yamllint disable-line rule:truthy - schedule: - - cron: "05 07 * * 5" - workflow_dispatch: - -jobs: - babel: - name: "create PR for additions from weblate" - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: weblate & git setup - env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} - run: | - mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - name: Merge and push transation updates - run: | - make V=1 weblate.translations.commit - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - commit-message: '[l10n] update translations from Weblate' - committer: searxng-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: translations_update - delete-branch: true - draft: false - title: '[l10n] update translations from Weblate' - body: | - update translations from Weblate - labels: | - translation diff --git a/Dockerfile b/Dockerfile index 9aeb2821475..92dfb49394a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ brotli \ - # lxml - libxml2-dev \ - libxslt1-dev \ - zlib1g-dev \ # uwsgi libpcre3-dev \ && rm -rf /var/lib/apt/lists/* @@ -16,8 +12,7 @@ WORKDIR /usr/local/searxng/ COPY ./requirements.txt ./requirements.txt -# Readd on #4707 "--mount=type=cache,id=pip,target=/root/.cache/pip" -RUN python -m venv ./venv \ +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ && pip install "uwsgi~=2.0" @@ -48,8 +43,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ # healthcheck wget \ - # lxml (ARMv7) - libxslt1.1 \ # uwsgi libpcre3 \ libxml2 \ diff --git a/Dockerfile.compat b/Dockerfile.compat new file mode 100644 index 00000000000..471b8bfdd1f --- /dev/null +++ b/Dockerfile.compat @@ -0,0 +1,106 @@ +FROM docker.io/library/python:3.13-slim AS builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + brotli \ + # lxml + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + # uwsgi + libpcre3-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/local/searxng/ + +COPY ./requirements.txt ./requirements.txt + +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ + && . ./venv/bin/activate \ + && pip install -r requirements.txt \ + && pip install "uwsgi~=2.0" + +COPY ./searx/ ./searx/ + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 + +RUN python -m compileall -q searx \ + && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ + && touch -c --date=@$TIMESTAMP_UWSGI ./dockerfiles/uwsgi.ini \ + && find /usr/local/searxng/searx/static \ + \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ + -type f -exec gzip -9 -k {} + -exec brotli --best {} + + +ARG SEARXNG_UID=977 +ARG SEARXNG_GID=977 + +RUN grep -m1 root /etc/group > /tmp/.searxng.group \ + && grep -m1 root /etc/passwd > /tmp/.searxng.passwd \ + && echo "searxng:x:$SEARXNG_GID:" >> /tmp/.searxng.group \ + && echo "searxng:x:$SEARXNG_UID:$SEARXNG_GID:searxng:/usr/local/searxng:/bin/bash" >> /tmp/.searxng.passwd + +FROM docker.io/library/python:3.13-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # healthcheck + wget \ + # lxml (ARMv7) + libxslt1.1 \ + # uwsgi + libpcre3 \ + libxml2 \ + mailcap \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=root:root --from=builder /tmp/.searxng.passwd /etc/passwd +COPY --chown=root:root --from=builder /tmp/.searxng.group /etc/group + +ARG LABEL_DATE="0001-01-01T00:00:00Z" +ARG GIT_URL="unspecified" +ARG SEARXNG_GIT_VERSION="unspecified" +ARG LABEL_VCS_REF="unspecified" +ARG LABEL_VCS_URL="unspecified" + +WORKDIR /usr/local/searxng/ + +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ +COPY --chown=searxng:searxng ./dockerfiles/ ./dockerfiles/ + +LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ + org.opencontainers.image.created=$LABEL_DATE \ + org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ + org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later" \ + org.opencontainers.image.revision=$LABEL_VCS_REF \ + org.opencontainers.image.source=$LABEL_VCS_URL \ + org.opencontainers.image.title="searxng" \ + org.opencontainers.image.url=$LABEL_VCS_URL \ + org.opencontainers.image.version=$SEARXNG_GIT_VERSION + +ENV CONFIG_PATH=/etc/searxng \ + DATA_PATH=/var/cache/searxng + +ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ + INSTANCE_NAME=searxng \ + AUTOCOMPLETE="" \ + BASE_URL="" \ + BIND_ADDRESS=[::]:8080 \ + MORTY_KEY="" \ + MORTY_URL="" \ + SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ + UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ + UWSGI_WORKERS=%k \ + UWSGI_THREADS=4 + +VOLUME $CONFIG_PATH +VOLUME $DATA_PATH + +EXPOSE 8080 + +HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 + +ENTRYPOINT ["/usr/local/searxng/dockerfiles/docker-entrypoint.sh"] diff --git a/Makefile b/Makefile index c1c0671495d..8c20eed35a2 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,9 @@ test.shell: MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean -MANAGE += docker.build docker.push docker.buildx +MANAGE += podman.build +MANAGE += docker.build docker.buildx +MANAGE += container.build container.test container.push MANAGE += gecko.driver MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean @@ -95,8 +97,8 @@ $(MANAGE): # short hands of selected targets -PHONY += docs docker themes +PHONY += docs container themes docs: docs.html -docker: docker.build +container: container.build themes: themes.all diff --git a/manage b/manage index 61bc68b748c..3e431a4e750 100755 --- a/manage +++ b/manage @@ -11,6 +11,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" +# shellcheck source=utils/lib_sxng_container.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_container.sh" + # shellcheck source=utils/lib_sxng_data.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh" @@ -60,7 +63,7 @@ while IFS= read -r line; do if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then YAMLLINT_FILES+=("$line") fi -done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" +done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" RST_FILES=( 'README.rst' @@ -77,9 +80,6 @@ docs.: gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build -docker.: - build : build docker image - push : build and push docker image gecko.driver: download & install geckodriver if not already installed (required for robot_tests) @@ -101,6 +101,7 @@ EOF go.help node.help weblate.help + container.help data.help test.help themes.help @@ -112,7 +113,6 @@ environment ... EOF } - if [ "$VERBOSE" = "1" ]; then SPHINX_VERBOSE="-v" PYLINT_VERBOSE="-v" @@ -125,99 +125,20 @@ webapp.run() { local parent_proc="$$" ( if [ "${LIVE_THEME}" ]; then - ( themes.live "${LIVE_THEME}" ) + (themes.live "${LIVE_THEME}") kill $parent_proc fi - )& + ) & ( sleep 3 xdg-open http://127.0.0.1:8888/ - )& + ) & SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp } -docker.push() { - docker.build push -} - +# Alias docker.buildx() { - docker.build buildx -} - -# shellcheck disable=SC2119 -docker.build() { - pyenv.install - - local SEARXNG_GIT_VERSION - local VERSION_GITCOMMIT - local GITHUB_USER - local SEARXNG_IMAGE_NAME - local BUILD - - build_msg DOCKER build - # run installation in a subprocess and activate pyenv - - # See https://www.shellcheck.net/wiki/SC1001 and others .. - # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 - ( set -e - pyenv.activate - - # Check if it is a git repository - if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - if [ ! -x "$(which git)" ]; then - die 1 "git is not installed" - fi - - if ! git remote get-url origin 2> /dev/null; then - die 1 "there is no remote origin" - fi - - # This is a git repository - git update-index -q --refresh - python -m searx.version freeze - eval "$(python -m searx.version)" - - # Get the last git commit id - VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" - - # define the docker image name - GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') - SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" - - BUILD="build" - if [ "$1" = "buildx" ]; then - # buildx includes the push option - CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" - BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" - shift - fi - build_msg DOCKER "Build command: ${BUILD}" - - # build Docker image - build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" - # shellcheck disable=SC2086 - docker $BUILD \ - --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ - --build-arg GIT_URL="${GIT_URL}" \ - --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ - --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ - --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ - --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ - --build-arg LABEL_VCS_URL="${GIT_URL}" \ - --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ - --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ - -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . - - if [ "$1" = "push" ]; then - docker push "${SEARXNG_IMAGE_NAME}:latest" - docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" - fi - ) - dump_return $? + container.buildx } # shellcheck disable=SC2119 @@ -226,10 +147,11 @@ gecko.driver() { build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv - ( set -e + ( + set -e pyenv.activate - INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" + INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" set +e if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then build_msg INSTALL "geckodriver already installed" @@ -237,13 +159,13 @@ gecko.driver() { fi PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in - "linux 32bit" | "linux2 32bit") ARCH="linux32";; - "linux 64bit" | "linux2 64bit") ARCH="linux64";; - "windows 32 bit") ARCH="win32";; - "windows 64 bit") ARCH="win64";; - "mac 64bit") ARCH="macos";; + "linux 32bit" | "linux2 32bit") ARCH="linux32" ;; + "linux 64bit" | "linux2 64bit") ARCH="linux64" ;; + "windows 32 bit") ARCH="win32" ;; + "windows 64 bit") ARCH="win64" ;; + "mac 64bit") ARCH="macos" ;; esac - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz" build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" @@ -258,13 +180,14 @@ gecko.driver() { py.build() { build_msg BUILD "python package ${PYDIST}" pyenv.cmd python setup.py \ - sdist -d "${PYDIST}" \ - bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" + sdist -d "${PYDIST}" \ + bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv - ( set -e + ( + set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info @@ -275,22 +198,22 @@ py.clean() { } pyenv.check() { - cat < OK') EOF } pyenv.install() { - if ! pyenv.OK; then - py.clean > /dev/null + py.clean >/dev/null fi - if pyenv.install.OK > /dev/null; then + if pyenv.install.OK >/dev/null; then return 0 fi - ( set -e + ( + set -e pyenv build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}" @@ -303,8 +226,8 @@ pyenv.install() { pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" - pyenv.cmd python setup.py develop --uninstall 2>&1 \ - | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + pyenv.cmd python setup.py develop --uninstall 2>&1 | + prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } @@ -320,7 +243,7 @@ docs.prebuild() { set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" - ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" + ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst" pyenv.cmd searxng_extra/docs_prebuild ) dump_return $? @@ -328,9 +251,9 @@ docs.prebuild() { # shellcheck disable=SC2119 main() { - local _type - local cmd="$1"; shift + local cmd="$1" + shift if [ "$cmd" == "" ]; then help @@ -339,22 +262,25 @@ main() { fi case "$cmd" in - --getenv) var="$1"; echo "${!var}";; - --help) help;; - --*) - help - err_msg "unknown option $cmd" + --getenv) + var="$1" + echo "${!var}" + ;; + --help) help ;; + --*) + help + err_msg "unknown option $cmd" + return 42 + ;; + *) + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + err_msg "unknown command: $cmd / use --help" return 42 - ;; - *) - _type="$(type -t "$cmd")" - if [ "$_type" != 'function' ]; then - err_msg "unknown command: $cmd / use --help" - return 42 - else - "$cmd" "$@" - fi - ;; + else + "$cmd" "$@" + fi + ;; esac } diff --git a/searx/version.py b/searx/version.py index d2013808ba4..565cc7e7abd 100644 --- a/searx/version.py +++ b/searx/version.py @@ -41,6 +41,12 @@ def subprocess_run(args, **kwargs): def get_git_url_and_branch(): + # handle GHA directly + if "GITHUB_REPOSITORY" in os.environ and "GITHUB_REF_NAME" in os.environ: + git_url = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}" + git_branch = os.environ["GITHUB_REF_NAME"] + return git_url, git_branch + try: ref = subprocess_run("git rev-parse --abbrev-ref @{upstream}") except subprocess.CalledProcessError: diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh new file mode 100755 index 00000000000..f6ffd5d2e2e --- /dev/null +++ b/utils/lib_sxng_container.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +container.help() { + cat </dev/null; then + container_engine="docker" + else + die 1 "Docker is not installed" + fi + elif [ "$1" = "podman" ]; then + if command -v podman &>/dev/null; then + container_engine="podman" + else + die 1 "Podman is not installed" + fi + else + # If no explicit engine is passed, prioritize podman over docker + if command -v podman &>/dev/null; then + container_engine="podman" + elif command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "Podman/Docker is not installed" + fi + fi + info_msg "Selected engine: $container_engine" + + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "Git is not installed" + fi + + ( + set -e + pyenv.activate + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + + if ! git remote get-url origin &>/dev/null; then + die 1 "There is no remote origin" + fi + + # This is a git repository + git update-index -q --refresh + python -m searx.version freeze + eval "$(python -m searx.version)" + + info_msg "Set \$VERSION_STRING: $VERSION_STRING" + info_msg "Set \$VERSION_TAG: $VERSION_TAG" + info_msg "Set \$DOCKER_TAG: $DOCKER_TAG" + info_msg "Set \$GIT_URL: $GIT_URL" + info_msg "Set \$GIT_BRANCH: $GIT_BRANCH" + + if [ "$container_engine" = "podman" ]; then + params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" + else + params_build_builder="build --platform=$platform --target=builder" + params_build="build --platform=$platform --squash" + fi + + if [ "$GITHUB_ACTIONS" = "true" ]; then + params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + params_build+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + + # Tags + params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + else + # Tags + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest" + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG" + fi + + # shellcheck disable=SC2086 + "$container_engine" $params_build_builder \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ + --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \ + --file="./$dockerfile" \ + . + build_msg CONTAINER "Image \"builder\" built" + + # shellcheck disable=SC2086 + "$container_engine" $params_build \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --file="./$dockerfile" \ + . + build_msg CONTAINER "Image built" + + if [ "$GITHUB_ACTIONS" = "true" ]; then + "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + # Output to GHA + { + echo "version_string=$VERSION_STRING" + echo "version_tag=$VERSION_TAG" + echo "docker_tag=$DOCKER_TAG" + echo "git_url=$GIT_URL" + echo "git_branch=$GIT_BRANCH" + } >>"$GITHUB_OUTPUT" + fi + ) + dump_return $? +} + +container.test() { + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; $parch" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -e + + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + name="$CONTAINER_IMAGE_NAME-$(date +%N)" + + podman create --name="$name" --rm --timeout=60 --network="host" \ + "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null + + podman start "$name" >/dev/null + podman logs -f "$name" & + pid_logs=$! + + # Wait until container is ready + sleep 5 + + curl -vf --max-time 5 "http://localhost:8080/healthz" + + kill $pid_logs &>/dev/null || true + podman stop "$name" >/dev/null + ) + dump_return $? +} + +container.push() { + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + # Architectures to release + local release_archs=("amd64" "arm64" "armv7") + + local archs=() + local variants=() + local platforms=() + + for arch in "${release_archs[@]}"; do + case $arch in + "X64" | "x86_64" | "amd64") + archs+=("amd64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARM64" | "aarch64" | "arm64") + archs+=("arm64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARMV7" | "armv7" | "armhf" | "arm") + archs+=("arm") + variants+=("v7") + platforms+=("linux/${archs[-1]}/${variants[-1]}") + ;; + *) + err_msg "Unsupported architecture; $arch" + exit 1 + ;; + esac + done + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -e + + # Pull archs + for i in "${!archs[@]}"; do + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + + # Tags to release + tags=("latest") + tags+=("$DOCKER_TAG") + + # Create manifests + for tag in "${tags[@]}"; do + if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then + podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + fi + + # Add archs to manifest + for i in "${!archs[@]}"; do + podman manifest add \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + done + + podman image list + + # Push manifests + for tag in "${tags[@]}"; do + build_msg CONTAINER "Pushing manifest with tag: $tag" + + podman manifest push \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "docker://docker.io/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + done + ) + dump_return $? +} + +# Alias +podman.build() { + container.build podman +} + +# Alias +docker.build() { + container.build docker +} + +# Alias +docker.buildx() { + container.build docker +}