[CI] Add support for building alpine-linux / musl-libc based wheels. #2704
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: [push, pull_request, workflow_dispatch] | |
| # Cancel previous runs, i.e. run only latest push. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| # For macos, at least 10.13 is required | |
| # to avoid issues and since the runners are macos-13 and macos-14: | |
| # -> use 13.6, which is Venture from 2022 and 14.0 on the arm runners. | |
| jobs: | |
| build_wheels: | |
| name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: macos-14 | |
| python-version: "3.13" | |
| cibw-build: "cp313-macosx_arm64" | |
| macos-target: "14.0" | |
| arch: "arm64" | |
| - os: macos-13 | |
| python-version: "3.13" | |
| cibw-build: "cp313-macosx_x86_64" | |
| macos-target: "13.6" | |
| arch: "x86_64" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install macOS dependencies | |
| if: runner.os == 'macOS' | |
| run: | | |
| set -euo pipefail | |
| echo "=== Starting macOS dependency installation ===" | |
| echo "Current PATH: $PATH" | |
| echo "Current architecture: $(uname -m)" | |
| echo "macOS version: $(sw_vers -productVersion)" | |
| echo "Current working directory: $(pwd)" | |
| echo "User: $(whoami)" | |
| echo "Home directory: $HOME" | |
| echo "Brew cellar directory: $(brew --cellar)" | |
| echo "Brew prefix directory: $(brew --prefix)" | |
| # Create /opt directory with correct permissions | |
| USER=$(whoami) | |
| sudo mkdir -p /opt | |
| sudo chown $USER /opt | |
| touch /opt/test | |
| # Handle different Homebrew cellar directories between Apple Silicon and x86_64 | |
| # Apple Silicon: /opt/homebrew/Cellar | |
| # Intel x86_64: /usr/local/Cellar | |
| HOMEBREW_PREFIX="$(brew --prefix)" | |
| HOMEBREW_CELLAR="$(brew --cellar)" | |
| ARCH="$(uname -m)" | |
| echo "Homebrew prefix: $HOMEBREW_PREFIX" | |
| echo "Homebrew cellar: $HOMEBREW_CELLAR" | |
| echo "Architecture: $ARCH" | |
| # Ensure /opt/homebrew exists and points to the correct location | |
| if [[ "$ARCH" == "arm64" ]]; then | |
| # Apple Silicon - Homebrew should already be in /opt/homebrew | |
| echo "Apple Silicon detected - using native /opt/homebrew" | |
| if [ ! -d /opt/homebrew ]; then | |
| echo "ERROR: /opt/homebrew not found on Apple Silicon!" | |
| exit 1 | |
| fi | |
| else | |
| # Intel x86_64 - Homebrew is in /usr/local, create symlink to /opt/homebrew | |
| echo "Intel x86_64 detected - creating /opt/homebrew symlink" | |
| if [ -L /opt/homebrew ]; then | |
| echo "/opt/homebrew already exists as a symlink" | |
| elif [ -d /opt/homebrew ]; then | |
| echo "Removing existing /opt/homebrew directory" | |
| sudo rm -rf /opt/homebrew | |
| fi | |
| # Create symlink from /opt/homebrew to actual Homebrew prefix | |
| sudo ln -s "$HOMEBREW_PREFIX" /opt/homebrew | |
| echo "Created symlink: /opt/homebrew -> $HOMEBREW_PREFIX" | |
| fi | |
| # Verify the setup | |
| echo "Verifying Homebrew setup:" | |
| echo " /opt/homebrew exists: $([ -e /opt/homebrew ] && echo 'YES' || echo 'NO')" | |
| echo " /opt/homebrew/bin exists: $([ -d /opt/homebrew/bin ] && echo 'YES' || echo 'NO')" | |
| echo " /opt/homebrew/lib exists: $([ -d /opt/homebrew/lib ] && echo 'YES' || echo 'NO')" | |
| echo " /opt/homebrew/Cellar exists: $([ -d /opt/homebrew/Cellar ] && echo 'YES' || echo 'NO')" | |
| # Install dependencies with comprehensive verification | |
| PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh | |
| PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh | |
| PREFIX=/opt bash ./scripts/macos/install_aws-sdk-cpp.sh | |
| # Set up Java environment | |
| echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile | |
| export PATH="/usr/local/opt/openjdk@11/bin:$PATH" | |
| # Update environment variables for the build (handle unbound variables with set -u) | |
| export PATH="/opt/homebrew/bin:/opt/bin:/usr/local/bin:${PATH:-}" | |
| export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/opt/lib/pkgconfig:/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH:-}" | |
| export CMAKE_PREFIX_PATH="/opt/homebrew:/opt:/usr/local:${CMAKE_PREFIX_PATH:-}" | |
| echo "=== macOS dependency installation completed ===" | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install cibuildwheel | |
| # by default use the latest version, if there are issues use 3.1.4. | |
| run: python -m pip install cibuildwheel | |
| - name: Build wheels | |
| run: python -m cibuildwheel --output-dir wheelhouse | |
| env: | |
| # configure cibuildwheel to build native archs ('auto'), and some | |
| # emulated ones | |
| CIBW_ARCHS_LINUX: native | |
| CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" | |
| CIBW_BUILD: ${{ matrix.cibw-build }} | |
| # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory | |
| # the OpenSSL3 lib is stored under /usr/local/lib64. | |
| CIBW_ENVIRONMENT_LINUX: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/opt/lib" | |
| # Requires macOS 10.13 at least to build because of C++17 features. | |
| # To avoid issues, simply use 13.6 for now. | |
| # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. | |
| # For github actions, $HOME is /Users/runner/ | |
| CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" | |
| # ensure build requirements are available during wheel building | |
| CIBW_BUILD_REQUIRES: "cloudpickle numpy" | |
| # run all python tests to make sure wheels are not defunct | |
| CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter cloudpickle" | |
| # Use following test command when segfaults happen to better pinpoint: | |
| # python3 -X faulthandler -m pytest -p no:faulthandler | |
| # else can use pytest ... | |
| # use 3min timeout per test and print top 25 slowest tests | |
| CIBW_TEST_COMMAND: "cd {project} && python3 -X faulthandler -m pytest -p no:faulthandler tuplex/python/tests -v --timeout 600 --durations 25" | |
| - name: reorganize files | |
| run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./.github/scripts/test_pypi.sh ./wheelhouse | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.os }}-${{ matrix.cibw-build }} | |
| path: | | |
| ./wheelhouse/*.whl | |
| # Note: when using download-artifact, use | |
| # - uses: actions/download-artifact@v4 | |
| # with: | |
| # path: dist | |
| # merge-multiple: true | |
| # # Requires 4.1 |