[CI] Add support for building alpine-linux / musl-libc based wheels. #2691
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: [push, pull_request, workflow_dispatch] | |
| # Cancel previous runs, i.e. run only latest push. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| # For macos, at least 10.13 is required | |
| # to avoid issues and since the runners are macos-13 and macos-14: | |
| # -> use 13.6, which is Venture from 2022 and 14.0 on the arm runners. | |
| jobs: | |
| build_wheels: | |
| name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| # macos-14 (which is macos-latest) is ARM only. macos-13 is the latest intel runner. | |
| # libunwind from brew has 13.6 as requirement right now, update the 13 runners accordingly. | |
| os: [ ubuntu-latest, macos-13, macos-14 ] | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| include: | |
| - os: ubuntu-latest | |
| python-version: "3.9" | |
| cibw-build: "cp39-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.10" | |
| cibw-build: "cp310-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.11" | |
| cibw-build: "cp311-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.12" | |
| cibw-build: "cp312-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.13" | |
| cibw-build: "cp313-manylinux_x86_64" | |
| - os: macos-13 | |
| python-version: "3.9" | |
| cibw-build: "cp39-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.10" | |
| cibw-build: "cp310-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.11" | |
| cibw-build: "cp311-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.12" | |
| cibw-build: "cp312-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.13" | |
| cibw-build: "cp313-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-14 | |
| python-version: "3.9" | |
| cibw-build: "cp39-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.10" | |
| cibw-build: "cp310-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.11" | |
| cibw-build: "cp311-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.12" | |
| cibw-build: "cp312-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.13" | |
| cibw-build: "cp313-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install macOS dependencies | |
| if: runner.os == 'macOS' | |
| run: | | |
| set -euo pipefail | |
| echo "=== Starting macOS dependency installation ===" | |
| # Install dependencies with comprehensive verification | |
| bash ./scripts/macos/brew_dependencies.sh | |
| bash ./scripts/macos/install_antlr4_cpp_runtime.sh | |
| bash ./scripts/macos/install_aws-sdk-cpp.sh | |
| # Set up Java environment | |
| echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile | |
| export PATH="/usr/local/opt/openjdk@11/bin:$PATH" | |
| # Update environment variables for the build | |
| export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" | |
| export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" | |
| export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" | |
| # Run comprehensive build environment test | |
| echo "=== Running build environment test ===" | |
| bash ./scripts/macos/test_build_environment.sh | |
| echo "=== macOS dependency installation completed ===" | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install cibuildwheel | |
| # by default use the latest version, if there are issues use 3.1.4. | |
| run: python -m pip install cibuildwheel | |
| - name: Build wheels | |
| run: python -m cibuildwheel --output-dir wheelhouse | |
| env: | |
| # configure cibuildwheel to build native archs ('auto'), and some | |
| # emulated ones | |
| CIBW_ARCHS_LINUX: native | |
| CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" | |
| CIBW_BUILD: ${{ matrix.cibw-build }} | |
| # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory | |
| # the OpenSSL3 lib is stored under /usr/local/lib64. | |
| CIBW_ENVIRONMENT_LINUX: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/opt/lib" | |
| # Requires macOS 10.13 at least to build because of C++17 features. | |
| # To avoid issues, simply use 13.6 for now. | |
| # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. | |
| # For github actions, $HOME is /Users/runner/ | |
| CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" | |
| # ensure build requirements are available during wheel building | |
| CIBW_BUILD_REQUIRES: "cloudpickle numpy" | |
| # run all python tests to make sure wheels are not defunct | |
| CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter cloudpickle" | |
| # Use following test command when segfaults happen to better pinpoint: | |
| # python3 -X faulthandler -m pytest -p no:faulthandler | |
| # else can use pytest ... | |
| # use 3min timeout per test and print top 25 slowest tests | |
| CIBW_TEST_COMMAND: "cd {project} && python3 -X faulthandler -m pytest -p no:faulthandler tuplex/python/tests -v --timeout 600 --durations 25" | |
| - name: reorganize files | |
| run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./.github/scripts/test_pypi.sh ./wheelhouse | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.os }}-${{ matrix.cibw-build }} | |
| path: | | |
| ./wheelhouse/*.whl | |
| # Note: when using download-artifact, use | |
| # - uses: actions/download-artifact@v4 | |
| # with: | |
| # path: dist | |
| # merge-multiple: true | |
| # # Requires 4.1 |