[CI] Add support for building alpine-linux / musl-libc based wheels. #2681
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: [push, pull_request, workflow_dispatch] | |
| # Cancel previous runs, i.e. run only latest push. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| # For macos, at least 10.13 is required | |
| # to avoid issues and since the runners are macos-13 and macos-14: | |
| # -> use 13.6, which is Venture from 2022 and 14.0 on the arm runners. | |
| jobs: | |
| build_wheels: | |
| name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| # macos-14 (which is macos-latest) is ARM only. macos-13 is the latest intel runner. | |
| # libunwind from brew has 13.6 as requirement right now, update the 13 runners accordingly. | |
| os: [ ubuntu-latest, macos-13, macos-14 ] | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| include: | |
| - os: ubuntu-latest | |
| python-version: "3.9" | |
| cibw-build: "cp39-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.10" | |
| cibw-build: "cp310-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.11" | |
| cibw-build: "cp311-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.12" | |
| cibw-build: "cp312-manylinux_x86_64" | |
| - os: ubuntu-latest | |
| python-version: "3.13" | |
| cibw-build: "cp313-manylinux_x86_64" | |
| - os: macos-13 | |
| python-version: "3.9" | |
| cibw-build: "cp39-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.10" | |
| cibw-build: "cp310-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.11" | |
| cibw-build: "cp311-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.12" | |
| cibw-build: "cp312-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-13 | |
| python-version: "3.13" | |
| cibw-build: "cp313-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| - os: macos-14 | |
| python-version: "3.9" | |
| cibw-build: "cp39-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.10" | |
| cibw-build: "cp310-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.11" | |
| cibw-build: "cp311-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.12" | |
| cibw-build: "cp312-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-14 | |
| python-version: "3.13" | |
| cibw-build: "cp313-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # disable cache for now. | |
| #- name: Cache brew dependencies | |
| # if: runner.os == 'macOS' | |
| # uses: actions/cache@v4.2.1 | |
| # with: | |
| # # Paths to cache: | |
| # # /usr/local/Homebrew - installation folder of Homebrew | |
| # # /usr/local/Cellar - installation folder of Homebrew formulae | |
| # # /usr/local/Frameworks, /usr/local/bin, /usr/local/opt - contain (links to) binaries installed by Homebrew formulae | |
| # path: | | |
| # /usr/local/Homebrew | |
| # /usr/local/Cellar | |
| # /usr/local/Frameworks | |
| # /usr/local/bin | |
| # /usr/local/opt | |
| # # macos13 runners are x86, macos14 are arm. --> use os therefore as cache key. | |
| # key: ${{ matrix.os }}-build-cache-${{ hashFiles('./scripts/macos/brew_dependencies.sh') }}-v2 | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install cibuildwheel | |
| # by default use the latest version, if there are issues use 3.1.4. | |
| run: python -m pip install cibuildwheel | |
| - name: Test build environment (macOS only) | |
| if: runner.os == 'macOS' | |
| run: | | |
| echo "=== Pre-build environment test ===" | |
| bash ./scripts/macos/test_build_environment.sh | |
| echo "=== Pre-build test completed ===" | |
| - name: Build wheels | |
| run: python -m cibuildwheel --output-dir wheelhouse | |
| env: | |
| # configure cibuildwheel to build native archs ('auto'), and some | |
| # emulated ones | |
| CIBW_ARCHS_LINUX: native | |
| CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" | |
| CIBW_BUILD: ${{ matrix.cibw-build }} | |
| # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. | |
| CIBW_BEFORE_ALL_MACOS: | | |
| set -euxo pipefail | |
| echo "=== Starting macOS dependency installation ===" | |
| # Install dependencies with comprehensive verification | |
| bash ./scripts/macos/install_antlr4_cpp_runtime.sh | |
| bash ./scripts/macos/brew_dependencies.sh | |
| bash ./scripts/macos/install_aws-sdk-cpp.sh | |
| # Set up Java environment | |
| echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile | |
| export PATH="/usr/local/opt/openjdk@11/bin:$PATH" | |
| # Update environment variables for the build | |
| export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" | |
| export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" | |
| export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" | |
| # Run comprehensive build environment test | |
| echo "=== Running build environment test ===" | |
| bash ./scripts/macos/test_build_environment.sh | |
| # Final verification | |
| echo "=== Final verification ===" | |
| echo "Protobuf version: $(protoc --version)" | |
| echo "Protobuf cmake files:" | |
| find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -5 | |
| echo "CMake version: $(cmake --version | head -1)" | |
| echo "Java version: $(java -version 2>&1 | head -1)" | |
| echo "=== macOS dependency installation completed ===" | |
| # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory | |
| # the OpenSSL3 lib is stored under /usr/local/lib64. | |
| CIBW_ENVIRONMENT_LINUX: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/opt/lib" | |
| # Requires macOS 10.13 at least to build because of C++17 features. | |
| # To avoid issues, simply use 13.6 for now. | |
| # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. | |
| # For github actions, $HOME is /Users/runner/ | |
| CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DLLVM_ROOT=/opt/homebrew/Cellar/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" | |
| # ensure build requirements are available during wheel building | |
| CIBW_BUILD_REQUIRES: "cloudpickle numpy" | |
| # run all python tests to make sure wheels are not defunct | |
| CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter cloudpickle" | |
| # Use following test command when segfaults happen to better pinpoint: | |
| # python3 -X faulthandler -m pytest -p no:faulthandler | |
| # else can use pytest ... | |
| # use 3min timeout per test and print top 25 slowest tests | |
| CIBW_TEST_COMMAND: "cd {project} && python3 -X faulthandler -m pytest -p no:faulthandler tuplex/python/tests -v --timeout 600 --durations 25" | |
| - name: reorganize files | |
| run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./.github/scripts/test_pypi.sh ./wheelhouse | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.os }}-${{ matrix.cibw-build }} | |
| path: | | |
| ./wheelhouse/*.whl | |
| # Note: when using download-artifact, use | |
| # - uses: actions/download-artifact@v4 | |
| # with: | |
| # path: dist | |
| # merge-multiple: true | |
| # # Requires 4.1 |