diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f4dc95937..df5a9d663 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,96 +17,99 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-14 (which is macos-latest) is ARM only. macos-13 is the latest intel runner. - # libunwind from brew has 13.6 as requirement right now, update the 13 runners accordingly. - os: [ ubuntu-latest, macos-13, macos-14 ] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] include: - - os: ubuntu-latest - python-version: "3.9" - cibw-build: "cp39-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.10" - cibw-build: "cp310-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.11" - cibw-build: "cp311-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.12" - cibw-build: "cp312-manylinux_x86_64" - - os: ubuntu-latest + - os: macos-14 python-version: "3.13" - cibw-build: "cp313-manylinux_x86_64" - - os: macos-13 - python-version: "3.9" - cibw-build: "cp39-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.10" - cibw-build: "cp310-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.11" - cibw-build: "cp311-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.12" - cibw-build: "cp312-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" + cibw-build: "cp313-macosx_arm64" + macos-target: "14.0" + arch: "arm64" - os: macos-13 python-version: "3.13" cibw-build: "cp313-macosx_x86_64" macos-target: "13.6" arch: "x86_64" - - os: macos-14 - python-version: "3.9" - cibw-build: "cp39-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.10" - cibw-build: "cp310-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.11" - cibw-build: "cp311-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.12" - cibw-build: "cp312-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.13" - cibw-build: "cp313-macosx_arm64" - macos-target: "14.0" - arch: "arm64" steps: - - uses: actions/checkout@v4 - - # disable cache for now. - #- name: Cache brew dependencies - # if: runner.os == 'macOS' - # uses: actions/cache@v4.2.1 - # with: - # # Paths to cache: - # # /usr/local/Homebrew - installation folder of Homebrew - # # /usr/local/Cellar - installation folder of Homebrew formulae - # # /usr/local/Frameworks, /usr/local/bin, /usr/local/opt - contain (links to) binaries installed by Homebrew formulae - # path: | - # /usr/local/Homebrew - # /usr/local/Cellar - # /usr/local/Frameworks - # /usr/local/bin - # /usr/local/opt - # # macos13 runners are x86, macos14 are arm. --> use os therefore as cache key. - # key: ${{ matrix.os }}-build-cache-${{ hashFiles('./scripts/macos/brew_dependencies.sh') }}-v2 + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install macOS dependencies + if: runner.os == 'macOS' + run: | + set -euo pipefail + echo "=== Starting macOS dependency installation ===" + echo "Current PATH: $PATH" + echo "Current architecture: $(uname -m)" + echo "macOS version: $(sw_vers -productVersion)" + echo "Current working directory: $(pwd)" + echo "User: $(whoami)" + echo "Home directory: $HOME" + echo "Brew cellar directory: $(brew --cellar)" + echo "Brew prefix directory: $(brew --prefix)" + + # Create /opt directory with correct permissions + USER=$(whoami) + sudo mkdir -p /opt + sudo chown $USER /opt + touch /opt/test + + # Handle different Homebrew cellar directories between Apple Silicon and x86_64 + # Apple Silicon: /opt/homebrew/Cellar + # Intel x86_64: /usr/local/Cellar + + HOMEBREW_PREFIX="$(brew --prefix)" + HOMEBREW_CELLAR="$(brew --cellar)" + ARCH="$(uname -m)" + + echo "Homebrew prefix: $HOMEBREW_PREFIX" + echo "Homebrew cellar: $HOMEBREW_CELLAR" + echo "Architecture: $ARCH" + + # Ensure /opt/homebrew exists and points to the correct location + if [[ "$ARCH" == "arm64" ]]; then + # Apple Silicon - Homebrew should already be in /opt/homebrew + echo "Apple Silicon detected - using native /opt/homebrew" + if [ ! -d /opt/homebrew ]; then + echo "ERROR: /opt/homebrew not found on Apple Silicon!" + exit 1 + fi + else + # Intel x86_64 - Homebrew is in /usr/local, create symlink to /opt/homebrew + echo "Intel x86_64 detected - creating /opt/homebrew symlink" + if [ -L /opt/homebrew ]; then + echo "/opt/homebrew already exists as a symlink" + elif [ -d /opt/homebrew ]; then + echo "Removing existing /opt/homebrew directory" + sudo rm -rf /opt/homebrew + fi + + # Create symlink from /opt/homebrew to actual Homebrew prefix + sudo ln -s "$HOMEBREW_PREFIX" /opt/homebrew + echo "Created symlink: /opt/homebrew -> $HOMEBREW_PREFIX" + fi + + # Verify the setup + echo "Verifying Homebrew setup:" + echo " /opt/homebrew exists: $([ -e /opt/homebrew ] && echo 'YES' || echo 'NO')" + echo " /opt/homebrew/bin exists: $([ -d /opt/homebrew/bin ] && echo 'YES' || echo 'NO')" + echo " /opt/homebrew/lib exists: $([ -d /opt/homebrew/lib ] && echo 'YES' || echo 'NO')" + echo " /opt/homebrew/Cellar exists: $([ -d /opt/homebrew/Cellar ] && echo 'YES' || echo 'NO')" + + + # Install dependencies with comprehensive verification + PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh + PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh + PREFIX=/opt bash ./scripts/macos/install_aws-sdk-cpp.sh + + # Set up Java environment + echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile + export PATH="/usr/local/opt/openjdk@11/bin:$PATH" + + # Update environment variables for the build (handle unbound variables with set -u) + export PATH="/opt/homebrew/bin:/opt/bin:/usr/local/bin:${PATH:-}" + export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/opt/lib/pkgconfig:/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + export CMAKE_PREFIX_PATH="/opt/homebrew:/opt:/usr/local:${CMAKE_PREFIX_PATH:-}" + + echo "=== macOS dependency installation completed ===" - name: Setup python uses: actions/setup-python@v5 @@ -114,6 +117,7 @@ jobs: python-version: '3.11' - name: Install cibuildwheel + # by default use the latest version, if there are issues use 3.1.4. run: python -m pip install cibuildwheel==2.22.0 - name: Build wheels @@ -125,8 +129,6 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" CIBW_BUILD: ${{ matrix.cibw-build }} - # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory # the OpenSSL3 lib is stored under /usr/local/lib64. @@ -136,10 +138,10 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" - + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib -DLLVM_ROOT=/opt/homebrew/opt/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + # run all python tests to make sure wheels are not defunct - CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter" + CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter cloudpickle" # Use following test command when segfaults happen to better pinpoint: # python3 -X faulthandler -m pytest -p no:faulthandler diff --git a/pyproject.toml b/pyproject.toml index aefc4e5dc..fdc917f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,21 @@ [build-system] requires = [ - "setuptools>=42", - "pip>9", + "setuptools", + "pip", "wheel", "cloudpickle", "numpy", - "ninja; sys_platform != 'win32'", + "ninja;sys_platform!='win32'", "cmake>=3.25", - "delocate; sys.platform == 'darwin'", - "auditwheel; sys.platform == 'linux'", + "delocate;sys.platform=='darwin'", + "auditwheel;sys.platform=='linux'", "requests" ] build-backend = "setuptools.build_meta" + +# Ensure cloudpickle and numpy are installed as build-time dependencies when building the wheel. +[tool.cibuildwheel] +before-build = [ + "pip install --upgrade pip setuptools wheel", + "pip install cloudpickle numpy" +] diff --git a/scripts/alpine/README.md b/scripts/alpine/README.md new file mode 100644 index 000000000..7be927775 --- /dev/null +++ b/scripts/alpine/README.md @@ -0,0 +1,5 @@ +Building a minimal alpine image with Tuplex. + +Images like python:3.11-alpine use muslc and are extremtly small. However, building for them is not trivial. + +This folder here holds a script to create whl files to be used in a python:3.11-alpine image. \ No newline at end of file diff --git a/scripts/alpine/builder/Dockerfile b/scripts/alpine/builder/Dockerfile new file mode 100644 index 000000000..52971e5fb --- /dev/null +++ b/scripts/alpine/builder/Dockerfile @@ -0,0 +1,12 @@ +FROM tuplex/musl:latest + +# mount tuplex in /code. + +# mount wheelhouse in /wheelhouse. + +WORKDIR /work + +COPY build.sh /work/build.sh +RUN chmod +x /work/build.sh + +ENTRYPOINT ["/work/build.sh"] diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh new file mode 100644 index 000000000..e7330f53b --- /dev/null +++ b/scripts/alpine/builder/build.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +# Alpine wheel builder script +# This script will be executed when the container starts +# Usage: build.sh [--test-only] + +set -e + +# Check for --test-only parameter +TEST_ONLY=false +if [ "$1" = "--test-only" ]; then + TEST_ONLY=true + echo "Test-only mode: will skip wheel building and test existing wheel" +fi + +echo "Alpine wheel builder container started" + +# Use Python 3.11 from the tuplex/musl image +PYTHON="/opt/_internal/cpython-3.11.13/bin/python3.11" +PIP="/opt/_internal/cpython-3.11.13/bin/pip" + +echo "Using Python: $PYTHON" +echo "Python version: $($PYTHON --version)" + + # Check if the project root is mounted + if [ ! -f "/code/setup.py" ]; then + echo "Error: /code/setup.py not found. Make sure the project root is mounted." + exit 1 + fi + + # Check if wheelhouse directory is mounted + if [ ! -d "/wheelhouse" ]; then + echo "Error: /wheelhouse directory not found. Make sure the wheelhouse is mounted." + exit 1 + fi + + # Create a writable build directory in /work (since /code is read-only) + echo "Setting up writable build directory in /work..." + mkdir -p /work/build + +if [ "$TEST_ONLY" = false ]; then + echo "Installing required dependencies..." + $PIP install setuptools wheel cloudpickle numpy attrs dill pluggy py pygments six wcwidth astor prompt_toolkit jedi PyYAML psutil iso8601 + + export LLVM_ROOT_DIR="/opt/llvm-16.0.6" + export LLVM_CONFIG="/opt/llvm-16.0.6/bin/llvm-config" + export PATH="/opt/llvm-16.0.6/bin:$PATH" + + echo "Building wheel for tuplex..." + + export CMAKE_ARGS=" + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_WITH_AWS=OFF \ + -DBUILD_WITH_ORC=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSKIP_AWS_TESTS=ON \ + -DBUILD_FOR_CI=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ + -DCMAKE_CXX_FLAGS=\"-O3 -DNDEBUG\" \ + -DCMAKE_C_FLAGS=\"-O3 -DNDEBUG\" \ + -DCMAKE_SHARED_LINKER_FLAGS=\"-Wl,--export-dynamic -Wl,--no-as-needed\" \ + -DLLVM_ROOT=/opt/llvm-16.0.6 \ + -DLLVM_ROOT_DIR=/opt/llvm-16.0.6 \ + -DLLVM_CONFIG=/opt/llvm-16.0.6/bin/llvm-config \ + -DPython3_EXECUTABLE=/opt/_internal/cpython-3.11.13/bin/python3.11 \ + -DPython3_ROOT_DIR=/opt/_internal/cpython-3.11.13 \ + -DPython3_LIBRARY=/opt/_internal/cpython-3.11.13/lib/libpython3.11.so \ + -DPython3_INCLUDE_DIR=/opt/python/cp311-cp311/include/python3.11 \ + -DPYTHON3_VERSION=3.11 \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + " + + # Build the wheel using setup.py + echo "Building wheel with setup.py..." + # Exclude history server from the wheel to reduce size and dependencies + echo "Setting TUPLEX_INCLUDE_HISTORYSERVER=False to exclude history server" + export TUPLEX_INCLUDE_HISTORYSERVER=False + echo "Environment variable TUPLEX_INCLUDE_HISTORYSERVER is set to: $TUPLEX_INCLUDE_HISTORYSERVER" + + # Change to the source directory and build with writable build directory + cd /code + export TMPDIR=/work/tmp + mkdir -p /work/tmp + CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py build --build-base /work/build bdist_wheel + + # Find the built wheel + WHEEL_FILE=$(find dist/ -name "*.whl" | head -1) + + if [ -z "$WHEEL_FILE" ]; then + echo "Error: No wheel file found in dist/ directory" + exit 1 + fi + + echo "Built wheel: $WHEEL_FILE" + + check_pyinit_tuplex_symbol() { + # Check that the symbol PyInit_tuplex is present in the tuplex .so + echo "Checking for PyInit_tuplex symbol in the built .so..." + + # Find the .so file in the wheel + local so_file + so_file=$(unzip -l "$WHEEL_FILE" | grep '\.so$' | awk '{print $4}' | head -1) + if [ -z "$so_file" ]; then + echo "Error: No .so file found in the wheel." + exit 1 + fi + + # Extract the .so file to a temp location + local tmp_so_dir + tmp_so_dir=$(mktemp -d) + unzip -j "$WHEEL_FILE" "$so_file" -d "$tmp_so_dir" >/dev/null + + local so_path="$tmp_so_dir/$(basename "$so_file")" + + if [ ! -f "$so_path" ]; then + echo "Error: Failed to extract .so file from wheel." + exit 1 + fi + + # Check for PyInit_tuplex symbol + if ! nm -D "$so_path" 2>/dev/null | grep -q 'PyInit_tuplex'; then + echo "Error: PyInit_tuplex symbol not found in $so_path" + exit 1 + else + echo "✓ PyInit_tuplex symbol found in $so_path" + fi + + # Clean up temp .so extraction directory + rm -rf "$tmp_so_dir" + } + + check_pyinit_tuplex_symbol + + # Use auditwheel to repair/delocate the wheel + echo "Repairing wheel with auditwheel..." + $PIP install auditwheel + auditwheel repair "$WHEEL_FILE" -w /wheelhouse + + # Find the repaired wheel + REPAIRED_WHEEL=$(find /wheelhouse -name "*.whl" | head -1) + + if [ -z "$REPAIRED_WHEEL" ]; then + echo "Error: No repaired wheel found in /wheelhouse" + exit 1 + fi + + echo "Successfully built and repaired wheel: $REPAIRED_WHEEL" + echo "Wheel stored in: /wheelhouse" +else + echo "Skipping wheel building (test-only mode)" + # Find the existing repaired wheel + REPAIRED_WHEEL=$(find /wheelhouse -name "*.whl" | head -1) + + if [ -z "$REPAIRED_WHEEL" ]; then + echo "Error: No wheel file found in /wheelhouse for testing" + exit 1 + fi + + echo "Found existing wheel for testing: $REPAIRED_WHEEL" +fi + + + + +echo "==============================" +echo "Testing built wheel in a fresh virtual environment..." +echo "==============================" + +# Create a temporary directory for the venv +VENV_DIR=$(mktemp -d) +echo "Created temporary venv directory: $VENV_DIR" + +# Create a new virtual environment +$PYTHON -m venv "$VENV_DIR/venv" +source "$VENV_DIR/venv/bin/activate" + +# Upgrade pip and install wheel +pip install --upgrade pip wheel + +# Install the repaired wheel +pip install "$REPAIRED_WHEEL" + +# Install test dependencies (pytest, etc.) +pip install pytest + +# Run the tests in tuplex/python/tests +echo "Running tests in tuplex/python/tests..." +pytest /code/tuplex/python/tests + +TEST_EXIT_CODE=$? + +# Deactivate and remove the virtual environment +deactivate +rm -rf "$VENV_DIR" + +if [ $TEST_EXIT_CODE -eq 0 ]; then + echo "All tests passed successfully!" +else + echo "Some tests failed. Exit code: $TEST_EXIT_CODE" + exit $TEST_EXIT_CODE +fi + diff --git a/scripts/alpine/example/Dockerfile b/scripts/alpine/example/Dockerfile new file mode 100644 index 000000000..f8f0533a2 --- /dev/null +++ b/scripts/alpine/example/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-alpine + diff --git a/scripts/alpine/make-wheels.sh b/scripts/alpine/make-wheels.sh new file mode 100755 index 000000000..bed5d940e --- /dev/null +++ b/scripts/alpine/make-wheels.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# Script to build Tuplex wheels for musl-based Alpine Linux images +# This script checks for the required tuplex/musl Docker image and provides +# instructions if it doesn't exist. +# Usage: make-wheels.sh [--test-only] + +set -e + +# Check for --test-only parameter +TEST_ONLY=false +if [ "$1" = "--test-only" ]; then + TEST_ONLY=true + echo "Test-only mode: will test existing wheel without rebuilding" +fi + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "Checking for tuplex/musl Docker image..." + +# Check if the tuplex/musl image exists locally +if ! docker image inspect tuplex/musl >/dev/null 2>&1; then + echo -e "${RED}Error: tuplex/musl Docker image not found locally.${NC}" + echo "" + echo -e "${YELLOW}To build the required tuplex/musl image, please run:${NC}" + echo "" + echo " docker build -t tuplex/musl -f scripts/docker/musllinux/Dockerfile ." + echo "" + echo "This will create the musl-based image needed for building Alpine-compatible wheels." + echo "" + echo -e "${YELLOW}Note:${NC} The build process may take some time as it needs to compile" + echo "Tuplex and all its dependencies for the musl libc environment." + echo "" + exit 1 +fi + +echo -e "${GREEN}✓ tuplex/musl Docker image found locally.${NC}" +echo "Proceeding with wheel building..." + + + +# Build the tuplex/alpine-wheel-builder image based on scripts/alpine/builder/Dockerfile +echo "Building tuplex/alpine-wheel-builder Docker image..." +docker build -t tuplex/alpine-wheel-builder -f "$SCRIPT_DIR/builder/Dockerfile" "$SCRIPT_DIR/builder" + +echo -e "${GREEN}✓ tuplex/alpine-wheel-builder Docker image built successfully.${NC}" + +# Prepare Docker mount options +# Mount the entire project root to /code so setup.py and other files are accessible +# Note: Must be writable as setup.py needs to create directories during build +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")/.." +MOUNT_OPTS="-v $PROJECT_ROOT:/code:ro" + +echo "Launching tuplex/alpine-wheel-builder container with tuplex/ mounted..." + +# Ensure wheelhouse/ exists in the project root directory +if [ ! -d "$SCRIPT_DIR/wheelhouse" ]; then + echo "Creating wheelhouse/ directory for output wheels..." + mkdir -p "$SCRIPT_DIR/wheelhouse" +fi + +# Add wheelhouse mount to Docker options +MOUNT_OPTS="$MOUNT_OPTS -v $SCRIPT_DIR/wheelhouse:/wheelhouse" + + +# You can add --rm to auto-remove the container after exit +if [ "$TEST_ONLY" = true ]; then + echo "Running container in test-only mode..." + docker run -it --rm \ + $MOUNT_OPTS \ + --name tuplex-alpine-wheel-builder \ + tuplex/alpine-wheel-builder --test-only +else + echo "Running container in build mode..." + docker run -it --rm \ + $MOUNT_OPTS \ + --name tuplex-alpine-wheel-builder \ + tuplex/alpine-wheel-builder +fi + +echo "Container execution completed." diff --git a/scripts/build_linux_musl_wheels.sh b/scripts/build_linux_musl_wheels.sh new file mode 100755 index 000000000..7575ad280 --- /dev/null +++ b/scripts/build_linux_musl_wheels.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# (c) 2017 - 2023 Tuplex team +# this script invokes the cibuildwheel process with necessary env variables to build musl wheels for linux/docker +# builds wheels for python 3.11+ with musl support +# supports building for amd64, arm64, or both architectures + +# check from where script is invoked +CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +# Parse command line arguments +TARGET_ARCH="" +while [[ $# -gt 0 ]]; do + case $1 in + --arch) + TARGET_ARCH="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --arch ARCH Target architecture: amd64, arm64, or both (default: native)" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 # Build for native architecture" + echo " $0 --arch amd64 # Build for AMD64 only" + echo " $0 --arch arm64 # Build for ARM64 only" + echo " $0 --arch both # Build for both architectures" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Determine target architecture +if [[ -z "$TARGET_ARCH" ]]; then + # Auto-detect native architecture + NATIVE_ARCH=$(uname -m) + case "$NATIVE_ARCH" in + x86_64) + TARGET_ARCH="amd64" + ;; + aarch64|arm64) + TARGET_ARCH="arm64" + ;; + *) + echo "Unsupported native architecture: $NATIVE_ARCH" + echo "Please specify --arch explicitly (amd64, arm64, or both)" + exit 1 + ;; + esac + echo "No architecture specified, using native architecture: $TARGET_ARCH" +fi + +# Validate architecture choice +case "$TARGET_ARCH" in + amd64|arm64|both) + echo "Target architecture: $TARGET_ARCH" + ;; + *) + echo "Invalid architecture: $TARGET_ARCH" + echo "Supported architectures: amd64, arm64, both" + exit 1 + ;; +esac + +echo "Executing musl buildwheel script located in $CWD" +pushd $CWD > /dev/null +cd .. + +# delete dir if exists +rm -rf wheelhouse +# delete in tree build files +rm -rf tuplex/python/tuplex/libexec/tuplex*.so + +# Check if musl Docker image exists, build if not +echo "Checking for tuplex/musl Docker image..." +if ! docker image inspect tuplex/musl:latest >/dev/null 2>&1; then + echo "tuplex/musl:latest image not found." + echo "" + case "$TARGET_ARCH" in + both) + echo "To build the tuplex/musl Docker image for both architectures, run:" + echo " cd scripts/docker/musllinux" + echo " ./build.sh --platform both --buildx" + ;; + amd64) + echo "To build the tuplex/musl Docker image for AMD64, run:" + echo " cd scripts/docker/musllinux" + echo " ./build.sh --platform amd64" + ;; + arm64) + echo "To build the tuplex/musl Docker image for ARM64, run:" + echo " cd scripts/docker/musllinux" + echo " ./build.sh --platform arm64" + ;; + esac + echo "" + echo "After building the image, re-run this script." + exit 1 +else + echo "tuplex/musl:latest image found" + if [[ "$TARGET_ARCH" == "both" ]]; then + echo "Note: For multi-architecture builds, ensure the image supports both x86_64 and aarch64" + fi +fi + +# CIBUILDWHEEL CONFIGURATION +export CIBUILDWHEEL=1 +export TUPLEX_BUILD_ALL=0 + +# Set architecture-specific configuration +case "$TARGET_ARCH" in + amd64) + export CIBW_ARCHS_LINUX="x86_64" + export CIBW_MUSLLINUX_X86_64_IMAGE='tuplex/musl:latest' + ;; + arm64) + export CIBW_ARCHS_LINUX="aarch64" + export CIBW_MUSLLINUX_AARCH64_IMAGE='tuplex/musl:latest' + ;; + both) + export CIBW_ARCHS_LINUX="x86_64 aarch64" + export CIBW_MUSLLINUX_X86_64_IMAGE='tuplex/musl:latest' + export CIBW_MUSLLINUX_AARCH64_IMAGE='tuplex/musl:latest' + ;; +esac + +# check whether lambda zip was build and stored in build-lambda +TUPLEX_LAMBDA_ZIP=${TUPLEX_LAMBDA_ZIP:-build-lambda/tplxlam.zip} + +echo "work dir is: $(pwd)" +if [[ -f "${TUPLEX_LAMBDA_ZIP}" ]]; then + echo "Found lambda runner ${TUPLEX_LAMBDA_ZIP}, adding to package" + mkdir -p tuplex/other + cp ${TUPLEX_LAMBDA_ZIP} tuplex/other/tplxlam.zip +fi + +# add to environment, e.g. TUPLEX_BUILD_TYPE=tsan to force a tsan build. Release is the default mode +# Note: The tuplex/musl image uses /opt as the installation prefix for both architectures +export CIBW_ENVIRONMENT="CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/opt/lib:/opt/lib64" + +# Build only musllinux wheels for Python 3.11+ +export CIBW_BUILD="cp3{11,12,13}-*" +# Note: CIBW_ARCHS_LINUX is already set above based on TARGET_ARCH + +# Only build musllinux wheels (skip manylinux) +export CIBW_SKIP="*-manylinux_*" + +export CIBW_BUILD_VERBOSITY=3 +export CIBW_PROJECT_REQUIRES_PYTHON=">=3.11" + +# Set platform tag for musl wheels +export CIBW_PLATFORM_LINUX="musllinux_1_2" + +echo "Building musl wheels with the following configuration:" +echo " - Image: tuplex/musl:latest" +echo " - Python versions: 3.11, 3.12, 3.13" +case "$TARGET_ARCH" in + amd64) + echo " - Architecture: x86_64 (AMD64)" + ;; + arm64) + echo " - Architecture: aarch64 (ARM64)" + ;; + both) + echo " - Architectures: x86_64 (AMD64), aarch64 (ARM64)" + ;; +esac +echo " - Platform: musllinux_1_2" +echo " - Skip: manylinux wheels" +echo "" + +cibuildwheel --platform linux . + +popd > /dev/null + +echo "Done building musl wheels!" +echo "Check the wheelhouse directory for the generated .whl files" diff --git a/scripts/docker/Dockerfile.alpine.minimal b/scripts/docker/Dockerfile.alpine.minimal new file mode 100644 index 000000000..7adb48882 --- /dev/null +++ b/scripts/docker/Dockerfile.alpine.minimal @@ -0,0 +1,11 @@ +FROM python:3.11-alpine + +RUN apk update +RUN apk add gcc musl-dev build-base linux-headers + +WORKDIR /code +COPY tuplex-0.3.7-cp311-cp311-musllinux_1_2_aarch64.whl /code/wheelhouse/ + +RUN python3 -m pip install /code/wheelhouse/tuplex-0.3.7-cp311-cp311-musllinux_1_2_aarch64.whl + + diff --git a/scripts/docker/benchmark/Dockerfile b/scripts/docker/benchmark/Dockerfile index 4188a4aee..a1b86d2f6 100644 --- a/scripts/docker/benchmark/Dockerfile +++ b/scripts/docker/benchmark/Dockerfile @@ -2,7 +2,6 @@ # (c) 2017 - 2023 L.Spiegelberg, R. Yesantharao FROM ubuntu:20.04 -MAINTAINER Leonhard Spiegelberg "leonhard_spiegelberg@brown.edu" # Use /tmp folder to build everything... ENV DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/musllinux/.dockerignore b/scripts/docker/musllinux/.dockerignore new file mode 100644 index 000000000..83cc17311 --- /dev/null +++ b/scripts/docker/musllinux/.dockerignore @@ -0,0 +1,41 @@ +# Git and version control +.git +.gitignore +.gitattributes + +# Documentation +README.md +*.md + +# Test files +test_*.sh + +# Original monolithic script (no longer needed) +install_requirements.sh + +# Docker files (avoid recursive copying) +Dockerfile* +.dockerignore + +# Build artifacts and temporary files +build/ +*.o +*.so +*.a +*.log + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile new file mode 100644 index 000000000..653ce3588 --- /dev/null +++ b/scripts/docker/musllinux/Dockerfile @@ -0,0 +1,148 @@ +# Build musllinux (Alpine) compatible version. +# You can create a multiplatform docker image via +# docker buildx build --platform linux/amd64,linux/arm64 -t tuplex/musl . + +# BUILD INSTRUCTIONS: +# =================== +# +# Single platform builds: +# ----------------------- +# For AMD64 (x86_64) systems: +# docker build --platform linux/amd64 -t tuplex/musl:amd64 . +# +# For ARM64 (aarch64) systems (like Apple Silicon): +# docker build --platform linux/arm64 -t tuplex/musl:arm64 . +# +# Multi-platform build (requires docker buildx): +# --------------------------------------------- +# docker buildx build --platform linux/amd64,linux/arm64 -t tuplex/musl:latest . +# +# To enable buildx (if not already enabled): +# docker buildx create --use +# +# To see available platforms: +# docker buildx inspect --bootstrap + +# Set platform-specific base image +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +# Use appropriate base image based on target platform +FROM quay.io/pypa/musllinux_1_2_x86_64:latest AS base-amd64 +FROM quay.io/pypa/musllinux_1_2_aarch64:latest AS base-arm64 + +# Select the appropriate base image +FROM base-${TARGETPLATFORM#linux/} + +# docker build -t tuplex/musl . +# docker run -it tuplex/musl bash + +# Installation prefix - all dependencies will be installed to this directory +ARG PREFIX=/opt + +# Version arguments for dependencies +ARG BOOST_VERSION=1.88.0 +ARG LLVM_VERSION=16.0.6 +ARG AWSSDK_CPP_VERSION=1.11.524 +ARG ANTLR4_VERSION=4.13.1 +ARG YAML_CPP_VERSION=0.8.0 +ARG AWS_LAMBDA_CPP_VERSION=0.2.10 +ARG PCRE2_VERSION=10.45 +ARG PROTOBUF_VERSION=32.0 +ARG CELERO_VERSION=2.8.3 +ARG SNAPPY_VERSION=1.2.2 +ARG ZLIB_VERSION=2.1.3 +ARG CMAKE_VERSION=3.27.5 + +# Update package index +RUN apk update + +# Install basic dependencies needed for the scripts +RUN apk add --no-cache \ + bash \ + wget \ + curl \ + git \ + make \ + gcc \ + g++ \ + cmake \ + ninja \ + python3 \ + py3-pip + +# Create necessary directories +RUN mkdir -p /opt/sbin /opt/bin /opt/lib /opt/include + +# Copy all installation scripts +COPY install_alpine.sh install_zlib.sh install_llvm.sh install_boost.sh \ + install_googletest.sh install_snappy.sh install_zstd.sh install_yamlcpp.sh install_celero.sh \ + install_antlr.sh install_pcre2.sh install_protobuf.sh install_aws_sdk.sh \ + /tmp/ + +# Make all scripts executable +RUN chmod +x /tmp/install_*.sh + +# Set working directory +WORKDIR /opt + +# Layer 1: Install Alpine packages and CMake (base system dependencies) +RUN /tmp/install_alpine.sh + +# Layer 2: Install zlib-ng (required by AWS SDK) +RUN /tmp/install_zlib.sh + +# Layer 3: Install testing and compression libraries +RUN /tmp/install_googletest.sh + +# Layer 4: Install snappy compression library +RUN /tmp/install_snappy.sh + +# Layer 5: Install zstd compression library +RUN /tmp/install_zstd.sh + +# Layer 6: Install parsing and data libraries +RUN /tmp/install_yamlcpp.sh + +# Layer 7: Install celero benchmarking library +RUN /tmp/install_celero.sh + +# Layer 8: Install parsing and regex libraries +RUN /tmp/install_antlr.sh + +# Layer 9: Install parsing and regex library +RUN /tmp/install_pcre2.sh + +# Layer 10: Install Protocol Buffers +RUN /tmp/install_protobuf.sh + +# Layer 11: Install AWS SDK C++ and Lambda runtime +RUN /tmp/install_aws_sdk.sh + +# Layer 12: Install Boost (C++ libraries) +RUN /tmp/install_boost.sh + +# Layer 13: Install LLVM (compiler infrastructure) (this will take a while) +RUN /tmp/install_llvm.sh + +# Clean up installation scripts +RUN rm -f /tmp/install_*.sh + +# Clean up downloaded temporary files and build artifacts +RUN rm -rf /tmp/tuplex-downloads + +# Fix CMake issue by using the working pipx-installed version +RUN ln -sf /opt/_internal/pipx/venvs/cmake/bin/cmake /opt/bin/cmake + +# Set environment variables +ENV PATH="/opt/bin:/opt/sbin:$PATH" +ENV LD_LIBRARY_PATH="/opt/lib:/opt/lib64:$LD_LIBRARY_PATH" +ENV PKG_CONFIG_PATH="/opt/lib/pkgconfig:$PKG_CONFIG_PATH" + +RUN mkdir -p /code + +WORKDIR /code + +# Default command +CMD ["/bin/bash"] + diff --git a/scripts/docker/musllinux/README.md b/scripts/docker/musllinux/README.md new file mode 100644 index 000000000..875b73686 --- /dev/null +++ b/scripts/docker/musllinux/README.md @@ -0,0 +1,193 @@ +# Tuplex Dependencies Installation Scripts + +This directory contains modular installation scripts for building Tuplex on Alpine Linux. Each script installs a specific dependency, allowing for flexible and optimized Docker builds. + +## Scripts Overview + +### Base System Dependencies +- **`install_alpine.sh`** - Installs Alpine Linux packages and CMake +- **`install_zlib.sh`** - Installs zlib-ng compression library + +### Core Compiler Infrastructure +- **`install_llvm.sh`** - Installs LLVM and Clang compiler +- **`install_boost.sh`** - Installs Boost C++ libraries + +### Testing and Benchmarking +- **`install_googletest.sh`** - Installs Google Test framework +- **`install_celero.sh`** - Installs Celero benchmarking library + +### Data Processing Libraries +- **`install_snappy.sh`** - Installs Snappy compression library +- **`install_yamlcpp.sh`** - Installs YAML-CPP parser +- **`install_protobuf.sh`** - Installs Protocol Buffers + +### Parsing and Regex +- **`install_antlr.sh`** - Installs ANTLR4 parser generator +- **`install_pcre2.sh`** - Installs PCRE2 regex library + +### Cloud Services +- **`install_aws_sdk.sh`** - Installs AWS SDK C++ and Lambda runtime + +### Master Script +- **`install_all.sh`** - Runs all scripts in the correct dependency order + +## Usage + +### Individual Installation +Each script can be run independently: + +```bash +# Install just LLVM +./install_llvm.sh + +# Install with custom version +LLVM_VERSION=17.0.0 ./install_llvm.sh + +# Install with custom prefix +PREFIX=/usr/local ./install_boost.sh +``` + +### Complete Installation +Use the master script to install everything: + +```bash +./install_all.sh +``` + +## Docker Build + +### Quick Start +Use the provided build script for easy Docker builds: + +```bash +# Build for current platform +./build.sh + +# Build for specific platform +./build.sh --platform amd64 +./build.sh --platform arm64 + +# Build multi-platform image +./build.sh --platform both --buildx + +# Build and push to registry +./build.sh --platform both --buildx --push + +# Build with custom installation prefix +./build.sh --prefix /usr/local +``` + +### Manual Docker Build +Build manually with Docker: + +```bash +# Single platform build +docker build --platform linux/amd64 -t tuplex/musl:amd64 . + +# Multi-platform build (requires docker buildx) +docker buildx build --platform linux/amd64,linux/arm64 -t tuplex/musl:latest . +``` + +### Docker Build Optimization +For optimal Docker layer caching, the Dockerfile is structured with separate layers: + +```dockerfile +# Base system +RUN /tmp/install_alpine.sh + +# Core dependencies +RUN /tmp/install_llvm.sh && /tmp/install_boost.sh + +# Libraries +RUN /tmp/install_zlib.sh && /tmp/install_googletest.sh && /tmp/install_snappy.sh + +# Parsing libraries +RUN /tmp/install_yamlcpp.sh && /tmp/install_celero.sh && /tmp/install_antlr.sh + +# System libraries +RUN /tmp/install_pcre2.sh && /tmp/install_protobuf.sh + +# Cloud services +RUN /tmp/install_aws_sdk.sh +``` + +### Customizing Versions and Installation Path +Override dependency versions and installation path using Docker build arguments: + +```bash +# Custom versions +docker build \ + --build-arg BOOST_VERSION=1.89.0 \ + --build-arg LLVM_VERSION=17.0.0 \ + --build-arg AWSSDK_CPP_VERSION=1.12.0 \ + -t tuplex/musl:custom . + +# Custom installation prefix +docker build \ + --build-arg PREFIX=/usr/local \ + -t tuplex/musl:custom-prefix . + +# Both custom versions and prefix +docker build \ + --build-arg PREFIX=/usr/local \ + --build-arg BOOST_VERSION=1.89.0 \ + --build-arg LLVM_VERSION=17.0.0 \ + -t tuplex/musl:custom-all . +``` + +## Environment Variables + +All scripts support the following environment variables: + +- **`PREFIX`** - Installation prefix (default: `/opt`) +- **`WORKDIR`** - Working directory for downloads (default: `/tmp`) +- **`CC`** - C compiler (default: `gcc`) +- **`CXX`** - C++ compiler (default: `g++`) +- **`CPU_COUNT`** - Number of CPU cores for parallel builds (auto-detected) + +### Version-Specific Variables +- **`LLVM_VERSION`** - LLVM version (default: `16.0.6`) +- **`BOOST_VERSION`** - Boost version (default: `1.88.0`) +- **`AWSSDK_CPP_VERSION`** - AWS SDK version (default: `1.11.524`) +- **`ANTLR4_VERSION`** - ANTLR4 version (default: `4.13.1`) +- **`YAML_CPP_VERSION`** - YAML-CPP version (default: `0.8.0`) +- **`CELERO_VERSION`** - Celero version (default: `2.8.3`) +- **`PCRE2_VERSION`** - PCRE2 version (default: `10.45`) +- **`PROTOBUF_VERSION`** - Protocol Buffers version (default: `32.0`) +- **`SNAPPY_VERSION`** - Snappy version (default: `1.2.2`) +- **`ZLIB_VERSION`** - zlib-ng version (default: `2.1.3`) +- **`CMAKE_VERSION`** - CMake version (default: `3.27.5`) + +## Dependencies Order + +The installation order is important due to dependencies: + +1. **Alpine packages + CMake** - Base system and build tools +2. **zlib-ng** - Required by AWS SDK +3. **LLVM** - Compiler infrastructure +4. **Boost** - C++ libraries +5. **Google Test** - Testing framework +6. **Snappy** - Compression library +7. **YAML-CPP** - YAML parsing +8. **Celero** - Benchmarking +9. **ANTLR4** - Parser generator +10. **PCRE2** - Regex library +11. **Protocol Buffers** - Serialization +12. **AWS SDK** - Cloud services (depends on zlib-ng) + +## Requirements + +- Alpine Linux +- Root privileges (run with `sudo` or as root) +- Internet connection for downloading source code +- Sufficient disk space for builds + +## Notes + +- All scripts use `set -euxo pipefail` for strict error handling +- Scripts automatically detect CPU count for parallel builds +- Downloads are cleaned up after installation to save space +- All libraries are built with `-fPIC` for position-independent code +- Most libraries are built as static libraries for better portability +- Docker builds are optimized with separate layers for better caching +- Multi-platform builds are supported via docker buildx diff --git a/scripts/docker/musllinux/build.sh b/scripts/docker/musllinux/build.sh new file mode 100755 index 000000000..69f647788 --- /dev/null +++ b/scripts/docker/musllinux/build.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Build script for Tuplex musllinux Docker image +# This script demonstrates how to build with different configurations + +set -e + +# Default values +PLATFORM="" +TAG="tuplex/musl" +PREFIX="/opt" +PUSH=false +BUILDX=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --platform) + PLATFORM="$2" + shift 2 + ;; + --tag) + TAG="$2" + shift 2 + ;; + --prefix) + PREFIX="$2" + shift 2 + ;; + --push) + PUSH=true + shift + ;; + --buildx) + BUILDX=true + shift + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --platform PLATFORM Target platform (amd64, arm64, or both)" + echo " --tag TAG Docker image tag (default: tuplex/musl)" + echo " --prefix PREFIX Installation prefix (default: /opt)" + echo " --push Push image to registry after build" + echo " --buildx Use docker buildx for multi-platform builds" + echo " --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo "Building Tuplex musllinux Docker image..." +echo "Tag: $TAG" +echo "Prefix: $PREFIX" +echo "Platform: ${PLATFORM:-auto-detect}" +echo "Buildx: $BUILDX" +echo "Push: $PUSH" +echo "" + +# Function to build single platform +build_single() { + local platform=$1 + local tag_suffix=$2 + + echo "Building for platform: $platform" + docker build --platform linux/$platform \ + --build-arg PREFIX=$PREFIX \ + --build-arg BOOST_VERSION=1.88.0 \ + --build-arg LLVM_VERSION=16.0.6 \ + --build-arg AWSSDK_CPP_VERSION=1.11.524 \ + --build-arg ANTLR4_VERSION=4.13.1 \ + --build-arg YAML_CPP_VERSION=0.8.0 \ + --build-arg AWS_LAMBDA_CPP_VERSION=0.2.10 \ + --build-arg PCRE2_VERSION=10.45 \ + --build-arg PROTOBUF_VERSION=32.0 \ + --build-arg CELERO_VERSION=2.8.3 \ + --build-arg SNAPPY_VERSION=1.2.2 \ + --build-arg ZLIB_VERSION=2.1.3 \ + --build-arg CMAKE_VERSION=3.27.5 \ + -t $TAG:$tag_suffix . + + if [ "$PUSH" = true ]; then + echo "Pushing $TAG:$tag_suffix..." + docker push $TAG:$tag_suffix + fi +} + +# Function to build multi-platform +build_multi() { + echo "Building multi-platform image..." + + # Enable buildx if requested + if [ "$BUILDX" = true ]; then + docker buildx create --use --name tuplex-builder || true + fi + + docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg PREFIX=$PREFIX \ + --build-arg BOOST_VERSION=1.88.0 \ + --build-arg LLVM_VERSION=16.0.6 \ + --build-arg AWSSDK_CPP_VERSION=1.11.524 \ + --build-arg ANTLR4_VERSION=4.13.1 \ + --build-arg YAML_CPP_VERSION=0.8.0 \ + --build-arg AWS_LAMBDA_CPP_VERSION=0.2.10 \ + --build-arg PCRE2_VERSION=10.45 \ + --build-arg PROTOBUF_VERSION=32.0 \ + --build-arg CELERO_VERSION=2.8.3 \ + --build-arg SNAPPY_VERSION=1.2.2 \ + --build-arg ZLIB_VERSION=2.1.3 \ + --build-arg CMAKE_VERSION=3.27.5 \ + -t $TAG:latest . + + if [ "$PUSH" = true ]; then + echo "Pushing multi-platform image..." + docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg BOOST_VERSION=1.88.0 \ + --build-arg LLVM_VERSION=16.0.6 \ + --build-arg AWSSDK_CPP_VERSION=1.11.524 \ + --build-arg ANTLR4_VERSION=4.13.1 \ + --build-arg YAML_CPP_VERSION=0.8.0 \ + --build-arg AWS_LAMBDA_CPP_VERSION=0.2.10 \ + --build-arg PCRE2_VERSION=10.45 \ + --build-arg PROTOBUF_VERSION=32.0 \ + --build-arg CELERO_VERSION=2.8.3 \ + --build-arg SNAPPY_VERSION=1.2.2 \ + --build-arg ZLIB_VERSION=2.1.3 \ + --build-arg CMAKE_VERSION=3.27.5 \ + -t $TAG:latest --push . + fi +} + +# Main build logic +if [ -z "$PLATFORM" ]; then + # Auto-detect platform + if [ "$BUILDX" = true ]; then + build_multi + else + # Build for current platform + CURRENT_ARCH=$(uname -m) + if [ "$CURRENT_ARCH" = "x86_64" ]; then + build_single "amd64" "latest" + elif [ "$CURRENT_ARCH" = "aarch64" ] || [ "$CURRENT_ARCH" = "arm64" ]; then + build_single "arm64" "latest" + else + echo "Unsupported architecture: $CURRENT_ARCH" + exit 1 + fi + fi +else + case $PLATFORM in + "amd64") + build_single "amd64" "amd64" + ;; + "arm64") + build_single "arm64" "arm64" + ;; + "both"|"multi") + build_multi + ;; + *) + echo "Unsupported platform: $PLATFORM" + echo "Supported platforms: amd64, arm64, both" + exit 1 + ;; + esac +fi + +echo "" +echo "Build completed successfully!" +echo "Image: $TAG" + +# Show available images +echo "" +echo "Available images:" +docker images | grep $TAG || echo "No images found with tag $TAG" diff --git a/scripts/docker/musllinux/install_all.sh b/scripts/docker/musllinux/install_all.sh new file mode 100755 index 000000000..6dbf099a1 --- /dev/null +++ b/scripts/docker/musllinux/install_all.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Master script that installs all tuplex dependencies in the correct order. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Starting installation of all Tuplex dependencies" + +# Create base directories +echo ">> Creating base directories" +mkdir -p $PREFIX && chmod 0755 $PREFIX +mkdir -p $PREFIX/sbin +mkdir -p $PREFIX/bin +mkdir -p $PREFIX/share +mkdir -p $PREFIX/include +mkdir -p $PREFIX/lib + +echo ">> Files will be downloaded to ${WORKDIR}/tuplex-downloads" +DOWNLOAD_DIR=$WORKDIR/tuplex-downloads +mkdir -p $DOWNLOAD_DIR + +# Set environment variables +export CC=${CC:-gcc} +export CXX=${CXX:-g++} +export PATH=$PREFIX/bin:$PATH + +# 1. Install Alpine packages and CMake (base system dependencies) +echo ">> Step 1: Installing Alpine packages and CMake" +./install_alpine.sh + +# 2. Install zlib-ng (required by AWS SDK) +echo ">> Step 2: Installing zlib-ng" +./install_zlib.sh + +# 3. Install LLVM (compiler infrastructure) +echo ">> Step 3: Installing LLVM" +./install_llvm.sh + +# 4. Install Boost (C++ libraries) +echo ">> Step 4: Installing Boost" +./install_boost.sh + +# 5. Install Google Test (testing framework) +echo ">> Step 5: Installing Google Test" +./install_googletest.sh + +# 6. Install Snappy (compression library) +echo ">> Step 6: Installing Snappy" +./install_snappy.sh + +# 7. Install YAML-CPP (YAML parsing) +echo ">> Step 7: Installing YAML-CPP" +./install_yamlcpp.sh + +# 8. Install Celero (benchmarking) +echo ">> Step 8: Installing Celero" +./install_celero.sh + +# 9. Install ANTLR4 (parser generator) +echo ">> Step 9: Installing ANTLR4" +./install_antlr.sh + +# 10. Install PCRE2 (regex library) +echo ">> Step 10: Installing PCRE2" +./install_pcre2.sh + +# 11. Install Protocol Buffers (serialization) +echo ">> Step 11: Installing Protocol Buffers" +./install_protobuf.sh + +# 12. Install AWS SDK C++ and Lambda runtime +echo ">> Step 12: Installing AWS SDK C++ and Lambda runtime" +./install_aws_sdk.sh + +# Clean up downloads +echo ">> Cleaning up download directory" +rm -rf ${DOWNLOAD_DIR} + +echo ">> All Tuplex dependencies installed successfully to ${PREFIX}" + +# Verify that key tools are available +echo ">> Verifying installation..." +ls -la /opt/bin/ +echo ">> PATH: $PATH" +echo ">> CMake version:" +/opt/bin/cmake --version || echo "CMake not found in /opt/bin" +echo ">> Python version:" +python3 --version || echo "Python3 not found" + +echo "-- Done, all Tuplex requirements installed to /opt --" diff --git a/scripts/docker/musllinux/install_alpine.sh b/scripts/docker/musllinux/install_alpine.sh new file mode 100755 index 000000000..91468f453 --- /dev/null +++ b/scripts/docker/musllinux/install_alpine.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Alpine Linux packages and CMake required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +CMAKE_VERSION=${CMAKE_VERSION:-3.27.5} + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Alpine Linux packages and CMake" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p $DOWNLOAD_DIR + +echo ">> Installing apk dependencies" +apk update + +apk add --no-cache \ + autoconf automake libtool \ + curl libxml2-dev vim build-base \ + openssl-dev zlib-dev ncurses-dev \ + readline-dev sqlite-dev \ + bzip2-dev expat-dev xz-dev \ + tk-dev libffi-dev wget git \ + curl-dev python3-dev py3-pip \ + openjdk11 ninja \ + linux-headers musl-dev \ + openssh file-dev + +echo ">> Installing recent cmake" +# fetch recent cmake & install - detect architecture +ARCH=$(uname -m) +if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + CMAKE_ARCH="aarch64" +else + CMAKE_ARCH="x86_64" +fi +URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz +mkdir -p ${DOWNLOAD_DIR}/cmake && cd ${DOWNLOAD_DIR}/cmake && + curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + tar -v -zxf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + rm -f cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + cd cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} && + cp -rp bin/* ${PREFIX}/bin/ && + cp -rp share/* ${PREFIX}/share/ && + cd / && rm -rf ${DOWNLOAD_DIR}/cmake + +export PATH=$PREFIX/bin:$PATH +cmake --version + +echo ">> Alpine Linux packages and CMake installation completed successfully" diff --git a/scripts/docker/musllinux/install_antlr.sh b/scripts/docker/musllinux/install_antlr.sh new file mode 100755 index 000000000..1197a1442 --- /dev/null +++ b/scripts/docker/musllinux/install_antlr.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs ANTLR4 dependencies required to build tuplex on Alpine Linux. +# +# KNOWN ISSUE: The ANTLR C++ runtime build may fail due to CMake version +# compatibility issues with its embedded Google Test dependency. This script +# will fail explicitly if the build doesn't succeed. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +ANTLR4_VERSION=${ANTLR4_VERSION:-4.13.1} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# Cleanup function +cleanup() { + if [ -d "${DOWNLOAD_DIR}/antlr" ]; then + echo ">> Cleaning up on exit..." + rm -rf "${DOWNLOAD_DIR}/antlr" + fi +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing ANTLR4 ${ANTLR4_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr + +echo ">> Downloading ANTLR JAR file..." +curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar +cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ +echo ">> ANTLR JAR file installed successfully" + +echo ">> Downloading ANTLR C++ runtime source..." +curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip +unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime +rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip + +echo ">> Building ANTLR C++ runtime..." +cd antlr4-cpp-runtime +echo ">> Removing existing build directory..." +rm -rf build +echo ">> Creating fresh build directory..." +mkdir -p build +echo ">> Entering build directory..." +cd build +echo ">> Current working directory: $(pwd)" + +echo ">> Running cmake configuration..." +echo ">> Note: ANTLR C++ runtime has embedded Google Test with CMake compatibility issues" +echo ">> Attempting to configure with policy overrides..." +echo ">> Current directory: $(pwd)" +echo ">> Running cmake command..." +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. +echo ">> Cmake exit code: $?" +if [ $? -ne 0 ]; then + echo ">> ERROR: cmake configuration failed" + echo ">> This is likely due to CMake version compatibility issues with embedded Google Test" + echo ">> ANTLR C++ runtime requires CMake 3.15+ but has embedded dependencies requiring older versions" + echo ">> Consider using a pre-built ANTLR C++ runtime package instead of building from source" + exit 1 +fi +echo ">> Cmake configuration completed successfully" + +echo ">> Compiling ANTLR C++ runtime..." +echo ">> Current directory: $(pwd)" +echo ">> Checking for Makefile..." +ls -la Makefile* || echo ">> No Makefile found!" +echo ">> Running make command..." +make -j ${CPU_COUNT} +echo ">> Make exit code: $?" +if [ $? -ne 0 ]; then + echo ">> ERROR: make compilation failed" + echo ">> This indicates that the cmake configuration did not generate proper build files" + echo ">> The ANTLR C++ runtime build is not working properly" + exit 1 +fi +echo ">> Make compilation completed successfully" + +echo ">> Installing ANTLR C++ runtime..." +make install +if [ $? -ne 0 ]; then + echo ">> ERROR: make install failed" + exit 1 +fi + +echo ">> Verifying ANTLR C++ runtime installation..." +if ! ls -la ${PREFIX}/lib/libantlr4-runtime* >/dev/null 2>&1; then + echo ">> ERROR: ANTLR C++ runtime library not found after installation" + echo ">> Expected files in ${PREFIX}/lib/:" + ls -la ${PREFIX}/lib/ | grep -i antlr || echo ">> No antlr files found in lib directory" + exit 1 +fi + +if ! ls -la ${PREFIX}/include/antlr4-runtime/ >/dev/null 2>&1; then + echo ">> ERROR: ANTLR C++ runtime headers not found after installation" + echo ">> Expected directory: ${PREFIX}/include/antlr4-runtime/" + echo ">> Contents of ${PREFIX}/include/:" + ls -la ${PREFIX}/include/ || echo ">> Include directory not found" + exit 1 +fi + +echo ">> ANTLR C++ runtime verification successful" + +# Clean up build artifacts +echo ">> Cleaning up build artifacts..." +cd ${DOWNLOAD_DIR} +rm -rf antlr + +echo ">> ANTLR4 ${ANTLR4_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_aws_sdk.sh b/scripts/docker/musllinux/install_aws_sdk.sh new file mode 100755 index 000000000..61f1004d0 --- /dev/null +++ b/scripts/docker/musllinux/install_aws_sdk.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs AWS SDK C++ and AWS Lambda C++ runtime dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +AWSSDK_CPP_VERSION=${AWSSDK_CPP_VERSION:-1.11.524} +AWS_LAMBDA_CPP_VERSION=${AWS_LAMBDA_CPP_VERSION:-0.2.10} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing AWS SDK C++ ${AWSSDK_CPP_VERSION} and AWS Lambda C++ runtime ${AWS_LAMBDA_CPP_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/aws + +# Install AWS SDK C++ +echo ">> Installing AWS SDK C++" +cd ${DOWNLOAD_DIR}/aws +git clone --progress --verbose --recurse-submodules https://github.com/aws/aws-sdk-cpp.git +cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. +make -j ${CPU_COUNT} +make install + +# Installing AWS Lambda C++ runtime +echo ">> Installing AWS Lambda C++ runtime" +cd ${DOWNLOAD_DIR}/aws +git clone https://github.com/awslabs/aws-lambda-cpp.git +cd aws-lambda-cpp +git fetch && git fetch --tags +git checkout v${AWS_LAMBDA_CPP_VERSION} +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. +make -j${CPU_COUNT} && make install + +echo ">> AWS SDK C++ ${AWSSDK_CPP_VERSION} and AWS Lambda C++ runtime ${AWS_LAMBDA_CPP_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_boost.sh b/scripts/docker/musllinux/install_boost.sh new file mode 100755 index 000000000..6750c7458 --- /dev/null +++ b/scripts/docker/musllinux/install_boost.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Boost dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +BOOST_VERSION=${BOOST_VERSION:-1.88.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get Python version for Boost installation path +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") +BOOST_INSTALL_DIR="${PREFIX}/boost/python${PYTHON_VERSION}" + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Boost ${BOOST_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p "${DOWNLOAD_DIR}/boost" + +# create underscored version, e.g. 1.79.0 -> 1_79_0 +boost_underscored_version=$(echo "${BOOST_VERSION}" | tr . _) + +# Download and build Boost including boost python +cd "${DOWNLOAD_DIR}/boost" +curl -L -O "https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz" +tar xf "boost-${BOOST_VERSION}-b2-nodocs.tar.gz" +cd "${DOWNLOAD_DIR}/boost/boost-${BOOST_VERSION}" +./bootstrap.sh --with-python="${PYTHON_EXECUTABLE}" --prefix="${BOOST_INSTALL_DIR}" --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" +./b2 cxxflags="-fPIC" link=static -j "${CPU_COUNT}" +./b2 cxxflags="-fPIC" link=static install +sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' "${BOOST_INSTALL_DIR}/include/boost/thread/pthread/thread_data.hpp" + +cd "${DOWNLOAD_DIR}" +rm -rf "${DOWNLOAD_DIR}/boost" + +echo ">> Boost ${BOOST_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_celero.sh b/scripts/docker/musllinux/install_celero.sh new file mode 100755 index 000000000..dc8551c74 --- /dev/null +++ b/scripts/docker/musllinux/install_celero.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Celero dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +CELERO_VERSION=${CELERO_VERSION:-2.8.3} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Celero ${CELERO_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/celero && cd ${DOWNLOAD_DIR}/celero \ +&& git clone https://github.com/DigitalInBlue/Celero.git celero && cd celero \ +&& git checkout tags/v${CELERO_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC -std=c++11" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Celero ${CELERO_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_googletest.sh b/scripts/docker/musllinux/install_googletest.sh new file mode 100755 index 000000000..d7917e2db --- /dev/null +++ b/scripts/docker/musllinux/install_googletest.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Google Test dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +GOOGLETEST_VERSION=${GOOGLETEST_VERSION:-1.14.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Google Test ${GOOGLETEST_VERSION}" + +git clone https://github.com/google/googletest.git -b v${GOOGLETEST_VERSION} && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install + +echo ">> Google Test ${GOOGLETEST_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_llvm.sh b/scripts/docker/musllinux/install_llvm.sh new file mode 100755 index 000000000..cd9c73f61 --- /dev/null +++ b/scripts/docker/musllinux/install_llvm.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs LLVM dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +LLVM_VERSION=${LLVM_VERSION:-16.0.6} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing LLVM ${LLVM_VERSION}" + +# Parse LLVM version components +LLVM_MAJOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f1` +LLVM_MINOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f2` +LLVM_MAJMIN_VERSION="${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}" + +# list of targets available to build: AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SystemZ;VE;WebAssembly;X86;XCore +# in order to cross-compile, should use targets: + +echo ">> building LLVM ${LLVM_VERSION}" +LLVM_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz +CLANG_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/clang-${LLVM_VERSION}.src.tar.xz +# required when LLVM version > 15 +LLVM_CMAKE_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cmake-${LLVM_VERSION}.src.tar.xz + +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) +echo ">> Building dependencies for ${PYTHON_VERSION}" + +echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}" +LLVM_WORKDIR=${WORKDIR}/tuplex-downloads/llvm${LLVM_VERSION} +mkdir -p ${LLVM_WORKDIR} +cd "${LLVM_WORKDIR}" || exit 1 + +wget ${LLVM_URL} && tar xf llvm-${LLVM_VERSION}.src.tar.xz +wget ${CLANG_URL} && tar xf clang-${LLVM_VERSION}.src.tar.xz && mv clang-${LLVM_VERSION}.src llvm-${LLVM_VERSION}.src/../clang + +if [ $LLVM_MAJOR_VERSION -ge 15 ]; then + wget ${LLVM_CMAKE_URL} && tar xf cmake-${LLVM_VERSION}.src.tar.xz && mv cmake-${LLVM_VERSION}.src cmake +fi + +mkdir -p llvm-${LLVM_VERSION}.src/build && cd llvm-${LLVM_VERSION}.src/build + +cmake -GNinja -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DCMAKE_INSTALL_PREFIX=${PREFIX}/llvm-${LLVM_VERSION} .. +ninja install +cd /tmp + +echo ">> LLVM ${LLVM_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_pcre2.sh b/scripts/docker/musllinux/install_pcre2.sh new file mode 100755 index 000000000..b77419292 --- /dev/null +++ b/scripts/docker/musllinux/install_pcre2.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs PCRE2 dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PCRE2_VERSION=${PCRE2_VERSION:-10.45} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing PCRE2 ${PCRE2_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/pcre2 && cd ${DOWNLOAD_DIR}/pcre2 \ +&& curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PCRE2_VERSION}/pcre2-${PCRE2_VERSION}.zip \ +&& unzip pcre2-${PCRE2_VERSION}.zip \ +&& rm pcre2-${PCRE2_VERSION}.zip \ +&& cd pcre2-${PCRE2_VERSION} \ +&& ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \ +&& make -j${CPU_COUNT} && make install + +echo ">> PCRE2 ${PCRE2_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_protobuf.sh b/scripts/docker/musllinux/install_protobuf.sh new file mode 100755 index 000000000..ceb2ca443 --- /dev/null +++ b/scripts/docker/musllinux/install_protobuf.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Protocol Buffers dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PROTOBUF_VERSION=${PROTOBUF_VERSION:-32.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Protocol Buffers ${PROTOBUF_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/protobuf && cd ${DOWNLOAD_DIR}/protobuf \ +&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install + +echo ">> Protocol Buffers ${PROTOBUF_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh new file mode 100644 index 000000000..320b3069c --- /dev/null +++ b/scripts/docker/musllinux/install_requirements.sh @@ -0,0 +1,297 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs all tuplex dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +CMAKE_VERSION="3.27.5" +BOOST_VERSION="1.88.0" +LLVM_VERSION="16.0.6" +AWSSDK_CPP_VERSION=1.11.524 +ANTLR4_VERSION=4.13.1 +YAML_CPP_VERSION=0.8.0 +AWS_LAMBDA_CPP_VERSION=0.2.10 +PCRE2_VERSION=10.45 +PROTOBUF_VERSION=32.0 +CELERO_VERSION=2.8.3 +SNAPPY_VERSION=1.2.2 +CC=gcc +CXX=g++ + +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +PYTHON_VERSION=$(echo $(python3 --version) | cut -d ' ' -f2) +PYTHON_MAJMIN_VERSION=${PYTHON_VERSION%.*} +echo ">> Installing dependencies for Python version ${PYTHON_VERSION}" + +version() { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +# --- helper functions --- +# Parse the major, minor and patch versions. +# You use it like this: +# semver="3.4.5+xyz" +# a=($(parse_semver "$semver")) +# major=${a[0]} +# minor=${a[1]} +# patch=${a[2]} +# printf "%-32s %4d %4d %4d\n" "$semver" $major $minor $patch +parse_semver() { + token="$1" + major=0 + minor=0 + patch=0 + + if echo "$token" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' >/dev/null 2>&1 ; then + # It has the correct syntax. + n=$(echo "$token" | tr -cd '0-9.' | tr '.' ' ') + major=$(echo "$n" | cut -d' ' -f1) + minor=$(echo "$n" | cut -d' ' -f2) + patch=$(echo "$n" | cut -d' ' -f3) + fi + + echo "$major $minor $patch" +} + +install_llvm() { + LLVM_VERSION=$1 + LLVM_MAJOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f1` + LLVM_MINOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f2` + LLVM_MAJMIN_VERSION="${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}" + + # list of targets available to build: AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SystemZ;VE;WebAssembly;X86;XCore + # in order to cross-compile, should use targets: + + + echo ">> building LLVM ${LLVM_VERSION}" + LLVM_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz + CLANG_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/clang-${LLVM_VERSION}.src.tar.xz + # required when LLVM version > 15 + LLVM_CMAKE_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cmake-${LLVM_VERSION}.src.tar.xz + + PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} + PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" + PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) + echo ">> Building dependencies for ${PYTHON_VERSION}" + + echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}}" + LLVM_WORKDIR=${DOWNLOAD_DIR}/llvm${LLVM_VERSION} + mkdir -p ${LLVM_WORKDIR} + cd "${LLVM_WORKDIR}" || exit 1 + + wget ${LLVM_URL} && tar xf llvm-${LLVM_VERSION}.src.tar.xz + wget ${CLANG_URL} && tar xf clang-${LLVM_VERSION}.src.tar.xz && mv clang-${LLVM_VERSION}.src llvm-${LLVM_VERSION}.src/../clang + + if [ $LLVM_MAJOR_VERSION -ge 15 ]; then + wget ${LLVM_CMAKE_URL} && tar xf cmake-${LLVM_VERSION}.src.tar.xz && mv cmake-${LLVM_VERSION}.src cmake + fi + + mkdir -p llvm-${LLVM_VERSION}.src/build && cd llvm-${LLVM_VERSION}.src/build + + cmake -GNinja -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm-${LLVM_VERSION} .. + ninja install + cd /tmp +} + +echo ">> Installing packages into ${PREFIX}" +mkdir -p $PREFIX && chmod 0755 $PREFIX +mkdir -p $PREFIX/sbin +mkdir -p $PREFIX/bin +mkdir -p $PREFIX/share +mkdir -p $PREFIX/include +mkdir -p $PREFIX/lib + +echo ">> Files will be downloaded to ${WORKDIR}/tuplex-downloads" +DOWNLOAD_DIR=$WORKDIR/tuplex-downloads +mkdir -p $DOWNLOAD_DIR + +PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) +echo ">> Building dependencies for ${PYTHON_VERSION}" +echo ">> Installing all build dependencies for Tuplex under Alpine Linux" + +echo ">> Installing apk dependencies" +apk update + +apk add --no-cache \ + autoconf automake libtool \ + curl libxml2-dev vim build-base \ + openssl-dev zlib-dev ncurses-dev \ + readline-dev sqlite-dev \ + bzip2-dev expat-dev xz-dev \ + tk-dev libffi-dev wget git \ + curl-dev python3-dev py3-pip \ + openjdk11 ninja \ + linux-headers musl-dev \ + openssh file-dev + +export CC=${CC} +export CXX=${CXX} + +echo ">> Installing recent cmake" +# fetch recent cmake & install - detect architecture +ARCH=$(uname -m) +if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + CMAKE_ARCH="aarch64" +else + CMAKE_ARCH="x86_64" +fi +URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz +mkdir -p ${DOWNLOAD_DIR}/cmake && cd ${DOWNLOAD_DIR}/cmake && + curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + tar -v -zxf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + rm -f cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + cd cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} && + cp -rp bin/* ${PREFIX}/bin/ && + cp -rp share/* ${PREFIX}/share/ && + cd / && rm -rf ${DOWNLOAD_DIR}/cmake + +export PATH=$PREFIX/bin:$PATH +cmake --version + + +echo ">> Installing Boost" +install_boost() { + local boost_version="${1:-$BOOST_VERSION}" + local download_dir="${2:-$DOWNLOAD_DIR}" + local prefix="${3:-$PREFIX}" + local python_executable="${4:-$PYTHON_EXECUTABLE}" + local cpu_count="${5:-$CPU_COUNT}" + + echo ">> Installing Boost ${boost_version}" + mkdir -p "${download_dir}/boost" + + # create underscored version, e.g. 1.79.0 -> 1_79_0 + local boost_underscored_version + boost_underscored_version=$(echo "${boost_version}" | tr . _) + + # Download and build Boost including boost python + cd "${download_dir}/boost" + curl -L -O "https://github.com/boostorg/boost/releases/download/boost-${boost_version}/boost-${boost_version}-b2-nodocs.tar.gz" + tar xf "boost-${boost_version}-b2-nodocs.tar.gz" + cd "${download_dir}/boost/boost-${boost_version}" + ./bootstrap.sh --with-python="${python_executable}" --prefix="${prefix}" --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" + ./b2 cxxflags="-fPIC" link=static -j "${cpu_count}" + ./b2 cxxflags="-fPIC" link=static install + sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' "${prefix}/include/boost/thread/pthread/thread_data.hpp" + + cd "${download_dir}" + rm -rf "${download_dir}/boost" +} + +install_boost "$BOOST_VERSION" "$DOWNLOAD_DIR" "$PREFIX" "$PYTHON_EXECUTABLE" "$CPU_COUNT" + +## -- install llvm +install_llvm $LLVM_VERSION + + +echo ">>> Installing tuplex dependencies." +# install recent zlib version (1.2.11) fork from cloudflare +# https://github.com/aws/aws-graviton-getting-started#zlib-on-linux +LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64:$LD_LIBRARY_PATH + +# Cloudflare fork is too old +#mkdir -p $WORKDIR/zlib && cd $WORKDIR && git clone https://github.com/cloudflare/zlib.git && cd zlib && ./configure --prefix=$PREFIX && make -j ${CPU_COUNT} && make install + +# note that zlib defines Z_NULL=0 whereas zlib-ng defines it as NULL, patch aws sdk accordingly +git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/2.1.3 && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON .. && make -j ${CPU_COUNT} && make install + +git clone https://github.com/google/googletest.git -b v1.14.0 && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release .. && make -j ${CPU_COUNT} && make install + +# build snappy as static lib +git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install + +# SSH setup not needed for Docker build - ssh-keyscan not available +mkdir -p /root/.ssh/ && + touch /root/.ssh/known_hosts && + ssh-keyscan github.com >> /root/.ssh/known_hosts + + +echo ">> Installing YAMLCPP" +mkdir -p ${DOWNLOAD_DIR}/yamlcpp && cd ${DOWNLOAD_DIR}/yamlcpp \ +&& git clone https://github.com/jbeder/yaml-cpp.git yaml-cpp \ +&& cd yaml-cpp \ +&& git checkout tags/${YAML_CPP_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Installing Celero" +mkdir -p ${DOWNLOAD_DIR}/celero && cd ${DOWNLOAD_DIR}/celero \ +&& git clone https://github.com/DigitalInBlue/Celero.git celero && cd celero \ +&& git checkout tags/v${CELERO_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC -std=c++11" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Installing ANTLR" +mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr \ +&& curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \ +&& cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ \ +&& curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime \ +&& rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& cd antlr4-cpp-runtime \ +&& mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT}&& make install + +echo ">> Installing AWS SDK" +# Note the z-lib patch here. +mkdir -p ${DOWNLOAD_DIR}/aws && cd ${DOWNLOAD_DIR}/aws \ +&& git clone --progress --verbose --recurse-submodules https://github.com/aws/aws-sdk-cpp.git \ +&& cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT} \ +&& make install + +# Installing AWS Lambda C++ runtime. +cd ${DOWNLOAD_DIR}/aws \ +&& git clone https://github.com/awslabs/aws-lambda-cpp.git \ +&& cd aws-lambda-cpp \ +&& git fetch && git fetch --tags \ +&& git checkout v${AWS_LAMBDA_CPP_VERSION} \ +&& mkdir build \ +&& cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j${CPU_COUNT} && make install + +echo ">> Installing PCRE2" +mkdir -p ${DOWNLOAD_DIR}/pcre2 && cd ${DOWNLOAD_DIR}/pcre2 \ +&& curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PCRE2_VERSION}/pcre2-${PCRE2_VERSION}.zip \ +&& unzip pcre2-${PCRE2_VERSION}.zip \ +&& rm pcre2-${PCRE2_VERSION}.zip \ +&& cd pcre2-${PCRE2_VERSION} \ +&& ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \ +&& make -j${CPU_COUNT} && make install + +echo ">> Installing protobuf" +mkdir -p ${DOWNLOAD_DIR}/protobuf && cd ${DOWNLOAD_DIR}/protobuf \ +&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j ${CPU_COUNT} && make install + + +# delete workdir (downloads dir) to clean up space +rm -rf ${DOWNLOAD_DIR} + +echo "-- Done, all Tuplex requirements installed to /opt --" + +# Verify that key tools are available +echo ">> Verifying installation..." +ls -la /opt/bin/ +echo ">> PATH: $PATH" +echo ">> CMake version:" +/opt/bin/cmake --version || echo "CMake not found in /opt/bin" +echo ">> Python version:" +python3 --version || echo "Python3 not found" diff --git a/scripts/docker/musllinux/install_snappy.sh b/scripts/docker/musllinux/install_snappy.sh new file mode 100755 index 000000000..c6c8437d1 --- /dev/null +++ b/scripts/docker/musllinux/install_snappy.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Snappy dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +SNAPPY_VERSION=${SNAPPY_VERSION:-1.2.2} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Snappy ${SNAPPY_VERSION}" + +# build snappy as static lib +git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install + +echo ">> Snappy ${SNAPPY_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_yamlcpp.sh b/scripts/docker/musllinux/install_yamlcpp.sh new file mode 100755 index 000000000..efe7367e3 --- /dev/null +++ b/scripts/docker/musllinux/install_yamlcpp.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs YAML-CPP dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +YAML_CPP_VERSION=${YAML_CPP_VERSION:-0.8.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing YAML-CPP ${YAML_CPP_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/yamlcpp && cd ${DOWNLOAD_DIR}/yamlcpp \ +&& git clone https://github.com/jbeder/yaml-cpp.git yaml-cpp \ +&& cd yaml-cpp \ +&& git checkout tags/${YAML_CPP_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> YAML-CPP ${YAML_CPP_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_zlib.sh b/scripts/docker/musllinux/install_zlib.sh new file mode 100755 index 000000000..adfb67bdc --- /dev/null +++ b/scripts/docker/musllinux/install_zlib.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs zlib-ng dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +ZLIB_VERSION=${ZLIB_VERSION:-2.1.3} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing zlib-ng ${ZLIB_VERSION}" + +# Set LD_LIBRARY_PATH +LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64:$LD_LIBRARY_PATH + +# note that zlib defines Z_NULL=0 whereas zlib-ng defines it as NULL, patch aws sdk accordingly +git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/${ZLIB_VERSION} && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install + +echo ">> zlib-ng ${ZLIB_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_zstd.sh b/scripts/docker/musllinux/install_zstd.sh new file mode 100755 index 000000000..cdca31446 --- /dev/null +++ b/scripts/docker/musllinux/install_zstd.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs zstd dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +ZSTD_VERSION=${ZSTD_VERSION:-1.5.6} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing zstd ${ZSTD_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/zstd && cd ${DOWNLOAD_DIR}/zstd \ +&& curl -LO https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz \ +&& tar -xzf zstd-${ZSTD_VERSION}.tar.gz \ +&& rm zstd-${ZSTD_VERSION}.tar.gz \ +&& cd zstd-${ZSTD_VERSION} \ +&& make -j${CPU_COUNT} CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC" \ +&& make install PREFIX=${PREFIX} \ +&& make install PREFIX=${PREFIX} MOREFLAGS="-fPIC" + +echo ">> zstd ${ZSTD_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/protobuf-musl-compatibility.patch b/scripts/docker/musllinux/protobuf-musl-compatibility.patch new file mode 100644 index 000000000..657ba8fb3 --- /dev/null +++ b/scripts/docker/musllinux/protobuf-musl-compatibility.patch @@ -0,0 +1,74 @@ +diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h ++++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +@@ -23,6 +23,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + namespace absl { + ABSL_NAMESPACE_BEGIN + namespace base_internal { +diff --git a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc ++++ b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +@@ -20,6 +20,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + #include "absl/base/internal/low_level_alloc.h" + + #include +diff --git a/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc b/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc ++++ b/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +@@ -20,6 +20,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + #include "absl/strings/internal/ostringstream.h" + + #include +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1234567..abcdefg 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -7,6 +7,12 @@ + cmake_minimum_required(VERSION 3.5) + project(protobuf VERSION 24.3.0 LANGUAGES C CXX) + ++# Add musl libc compatibility flags ++if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ++ add_compile_definitions(_LARGEFILE64_SOURCE) ++ add_compile_definitions(_FILE_OFFSET_BITS=64) ++endif() ++ + # Set C++ standard + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/scripts/docker/musllinux/protobuf-musl-fix.patch b/scripts/docker/musllinux/protobuf-musl-fix.patch new file mode 100644 index 000000000..826bf9017 --- /dev/null +++ b/scripts/docker/musllinux/protobuf-musl-fix.patch @@ -0,0 +1,19 @@ +diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h ++++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +@@ -23,6 +23,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + namespace absl { + ABSL_NAMESPACE_BEGIN + namespace base_internal { diff --git a/scripts/docker/musllinux/test_docker_build.sh b/scripts/docker/musllinux/test_docker_build.sh new file mode 100755 index 000000000..25ac6b0fc --- /dev/null +++ b/scripts/docker/musllinux/test_docker_build.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Test script to verify Docker build works with new modular scripts + +set -e + +echo "Testing Docker build with new modular scripts..." +echo "================================================" + +# Check if Docker is available +if ! command -v docker &> /dev/null; then + echo "❌ Docker is not installed or not in PATH" + exit 1 +fi + +echo "✅ Docker is available" + +# Check if all required scripts exist +REQUIRED_SCRIPTS=( + "install_alpine.sh" + "install_zlib.sh" + "install_llvm.sh" + "install_boost.sh" + "install_googletest.sh" + "install_snappy.sh" + "install_yamlcpp.sh" + "install_celero.sh" + "install_antlr.sh" + "install_pcre2.sh" + "install_protobuf.sh" + "install_aws_sdk.sh" +) + +echo "Checking required scripts..." +for script in "${REQUIRED_SCRIPTS[@]}"; do + if [ -f "$script" ]; then + echo "✅ $script exists" + else + echo "❌ $script missing" + exit 1 + fi +done + +# Check if scripts are executable +echo "Checking script permissions..." +for script in "${REQUIRED_SCRIPTS[@]}"; do + if [ -x "$script" ]; then + echo "✅ $script is executable" + else + echo "❌ $script is not executable" + exit 1 + fi +done + +# Check if Dockerfile exists +if [ -f "Dockerfile" ]; then + echo "✅ Dockerfile exists" +else + echo "❌ Dockerfile missing" + exit 1 +fi + +# Check if .dockerignore exists +if [ -f ".dockerignore" ]; then + echo "✅ .dockerignore exists" +else + echo "❌ .dockerignore missing" + exit 1 +fi + +# Check if build script exists +if [ -f "build.sh" ]; then + echo "✅ build.sh exists" + if [ -x "build.sh" ]; then + echo "✅ build.sh is executable" + else + echo "❌ build.sh is not executable" + exit 1 + fi +else + echo "❌ build.sh missing" + exit 1 +fi + +echo "" +echo "All checks passed! 🎉" +echo "" +echo "You can now build the Docker image using:" +echo " ./build.sh # Build for current platform" +echo " ./build.sh --platform amd64 # Build for AMD64" +echo " ./build.sh --platform arm64 # Build for ARM64" +echo " ./build.sh --platform both # Build multi-platform" +echo "" +echo "Or build manually with:" +echo " docker build -t tuplex/musl ." +echo "" +echo "The new modular approach provides:" +echo " ✅ Better Docker layer caching" +echo " ✅ Easier debugging and maintenance" +echo " ✅ Flexible version customization" +echo " ✅ Individual dependency installation" +echo " ✅ Multi-platform build support" diff --git a/scripts/docker/musllinux/test_prefix_config.sh b/scripts/docker/musllinux/test_prefix_config.sh new file mode 100755 index 000000000..8dd74b1af --- /dev/null +++ b/scripts/docker/musllinux/test_prefix_config.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Test script to verify PREFIX variable configuration in all installation scripts + +set -e + +echo "Testing PREFIX variable configuration in installation scripts..." +echo "=============================================================" + +# Test PREFIX variable definition +echo "1. Checking PREFIX variable definition..." +for script in install_*.sh; do + if [ "$script" = "install_all.sh" ]; then + continue # Skip the master script + fi + + if grep -q "PREFIX=\${PREFIX:-/opt}" "$script"; then + echo "✅ $script: PREFIX variable properly defined" + else + echo "❌ $script: PREFIX variable missing or incorrectly defined" + exit 1 + fi +done + +echo "" +echo "2. Checking cmake installations use -DCMAKE_INSTALL_PREFIX..." +CMAKE_SCRIPTS=( + "install_zlib.sh" + "install_googletest.sh" + "install_snappy.sh" + "install_yamlcpp.sh" + "install_celero.sh" + "install_antlr.sh" + "install_protobuf.sh" + "install_aws_sdk.sh" +) + +for script in "${CMAKE_SCRIPTS[@]}"; do + if grep -q "CMAKE_INSTALL_PREFIX.*PREFIX" "$script"; then + echo "✅ $script: cmake uses PREFIX variable" + else + echo "❌ $script: cmake missing PREFIX variable" + exit 1 + fi +done + +echo "" +echo "3. Checking configure installations use --prefix..." +CONFIGURE_SCRIPTS=( + "install_pcre2.sh" +) + +for script in "${CONFIGURE_SCRIPTS[@]}"; do + if grep -q "prefix.*PREFIX" "$script"; then + echo "✅ $script: configure uses PREFIX variable" + else + echo "❌ $script: configure missing PREFIX variable" + exit 1 + fi +done + +echo "" +echo "4. Checking bootstrap installations use --prefix..." +BOOTSTRAP_SCRIPTS=( + "install_boost.sh" +) + +for script in "${BOOTSTRAP_SCRIPTS[@]}"; do + if grep -q "prefix.*PREFIX" "$script"; then + echo "✅ $script: bootstrap uses PREFIX variable" + else + echo "❌ $script: bootstrap missing PREFIX variable" + exit 1 + fi +done + +echo "" +echo "5. Checking LLVM installation path..." +if grep -q "CMAKE_INSTALL_PREFIX.*PREFIX.*llvm" install_llvm.sh; then + echo "✅ install_llvm.sh: Uses PREFIX variable for LLVM installation" +else + echo "❌ install_llvm.sh: LLVM installation path not using PREFIX variable" + exit 1 +fi + +echo "" +echo "All PREFIX variable configurations are correct! 🎉" +echo "" +echo "Summary of installation methods:" +echo " ✅ cmake: -DCMAKE_INSTALL_PREFIX=\${PREFIX}" +echo " ✅ configure: --prefix=\${PREFIX}" +echo " ✅ bootstrap: --prefix=\"\${PREFIX}\"" +echo " ✅ LLVM: \${PREFIX}/llvm-\${LLVM_VERSION}" +echo "" +echo "All scripts will now install to the directory specified by the PREFIX" +echo "environment variable (default: /opt) instead of hardcoded paths." diff --git a/scripts/docker/musllinux/test_protobuf_patched.sh b/scripts/docker/musllinux/test_protobuf_patched.sh new file mode 100644 index 000000000..2bb6e02e0 --- /dev/null +++ b/scripts/docker/musllinux/test_protobuf_patched.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +echo "=== Testing Protobuf 24.3 with Musl Compatibility Patch ===" + +# Step 1: Clone protobuf +echo "1. Cloning protobuf..." +git clone -b v24.3 https://github.com/protocolbuffers/protobuf.git /tmp/protobuf-test +cd /tmp/protobuf-test + +echo "2. Updating submodules..." +git submodule update --init --recursive + +echo "3. Applying musl compatibility patch..." +# Create the patch file inline +cat > /tmp/musl-fix.patch << 'EOF' +diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h ++++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +@@ -23,6 +23,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + namespace absl { + ABSL_NAMESPACE_BEGIN + namespace base_internal { +diff --git a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc ++++ b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +@@ -20,6 +20,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + #include "absl/base/internal/low_level_alloc.h" + + #include +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1234567..abcdefg 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -7,6 +7,12 @@ + cmake_minimum_required(VERSION 3.5) + project(protobuf VERSION 24.3.0 LANGUAGES C CXX) + ++# Add musl libc compatibility flags ++if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ++ add_compile_definitions(_LARGEFILE64_SOURCE) ++ add_compile_definitions(_FILE_OFFSET_BITS=64) ++endif() ++ + # Set C++ standard + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +EOF + +# Apply the patch +patch -p1 < /tmp/musl-fix.patch + +echo "4. Building protobuf with patch..." +mkdir build +cd build +/opt/bin/cmake -DCMAKE_CXX_FLAGS="-fPIC -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. +make -j4 + +echo "5. Installing protobuf..." +make install + +echo "6. Testing protoc..." +protoc --version + +echo "7. Creating test project..." +cd /tmp +mkdir proto-test +cd proto-test + +echo "8. Creating test proto file..." +cat > test.proto << 'EOF' +syntax = "proto3"; +message TestMessage { + string text = 1; + int32 number = 2; +} +EOF + +echo "9. Compiling proto file..." +protoc --cpp_out=. test.proto + +echo "10. Creating test program..." +cat > main.cpp << 'EOF' +#include +#include "test.pb.h" + +int main() { + TestMessage msg; + msg.set_text("Hello Protobuf!"); + msg.set_number(42); + + std::cout << "Text: " << msg.text() << std::endl; + std::cout << "Number: " << msg.number() << std::endl; + + return 0; +} +EOF + +echo "11. Building test program..." +g++ -std=c++17 main.cpp test.pb.cc -lprotobuf -o test_program + +echo "12. Running test..." +./test_program + +echo "=== Protobuf test with musl patch completed successfully! ===" diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index ac2a5512c..167caa7e9 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -2,11 +2,186 @@ # This script installs all required dependencies via brew # for instructions on how to install brew, visit https://brew.sh/ +set -euo pipefail -# brew doesn't provide llvm@16 bottle anymore for big sur, but python3.8 only works with big sur tags. use llvm@15 instead -brew install openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf +# Function to verify a command exists and is working +verify_command() { + local cmd="$1" + local expected_output="$2" + echo "Verifying $cmd..." + + if command -v "$cmd" >/dev/null 2>&1; then + if [ -n "$expected_output" ]; then + local actual_output + actual_output=$($cmd --version 2>/dev/null || echo "version check failed") + echo "$cmd found: $actual_output" + else + echo "$cmd found" + fi + return 0 + else + echo "ERROR: $cmd not found in PATH" + return 1 + fi +} -# link (when e.g. used from restoring cache) -brew link --overwrite cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil +# Function to verify library exists +verify_library() { + local lib_name="$1" + local search_paths="$2" + echo "Verifying $lib_name library..." + + # Expand all paths including wildcards + local expanded_paths="" + for path in $search_paths; do + if [[ "$path" == *"*"* ]]; then + # Expand wildcards + for expanded_path in $path; do + if [ -d "$expanded_path" ]; then + expanded_paths="$expanded_paths $expanded_path" + fi + done + else + expanded_paths="$expanded_paths $path" + fi + done + + # Search in all expanded paths + for path in $expanded_paths; do + echo " Searching in: $path" + # Check if directory exists before searching + if [ ! -d "$path" ]; then + echo " Directory does not exist, skipping" + continue + fi + # Use a temporary variable to capture output and check length + local found_files + found_files=$(find "$path" -name "*${lib_name}*" -type f 2>/dev/null) + echo " Found $(echo "$found_files" | wc -l) files" + if [ -n "$found_files" ]; then + echo "$lib_name found in $path" + echo "$found_files" | head -3 + return 0 + fi + done + + echo "ERROR: $lib_name not found in any of: $search_paths" + return 1 +} -echo "Done!" \ No newline at end of file +# Function to verify cmake package +verify_cmake_package() { + local package_name="$1" + echo "Verifying CMake package: $package_name" + + # Get Homebrew paths dynamically + local homebrew_prefix="${HOMEBREW_PREFIX:-$(brew --prefix)}" + local homebrew_cellar="${HOMEBREW_CELLAR:-$(brew --cellar)}" + + # Search in standard locations and Cellar paths + # Use find to get all CMake directories + local expanded_paths="$homebrew_prefix/lib/cmake /usr/local/lib/cmake" + + # Add Cellar paths using find to get actual paths + cellar_cmake_paths=$(find "$homebrew_cellar" -name "cmake" -type d -path "*/lib/cmake" 2>/dev/null) + if [ -n "$cellar_cmake_paths" ]; then + expanded_paths="$expanded_paths $cellar_cmake_paths" + fi + + # Search in all expanded paths + for path in $expanded_paths; do + # Check if directory exists before searching + if [ ! -d "$path" ]; then + continue + fi + # Use a temporary variable to capture output and check length + local found_packages + found_packages=$(find "$path" -name "*${package_name}*" 2>/dev/null) + if [ -n "$found_packages" ]; then + echo "CMake package $package_name found" + echo "$found_packages" | head -3 + return 0 + fi + done + + echo "ERROR: CMake package $package_name not found" + return 1 +} + +# Detect Homebrew paths dynamically +HOMEBREW_PREFIX="$(brew --prefix)" +HOMEBREW_CELLAR="$(brew --cellar)" +ARCH="$(uname -m)" + +echo "Detected Homebrew configuration:" +echo " Architecture: $ARCH" +echo " Homebrew prefix: $HOMEBREW_PREFIX" +echo " Homebrew cellar: $HOMEBREW_CELLAR" + +# Update brew first +echo "Updating brew..." +brew update --quiet 2>/dev/null || echo "Warning: brew update failed" + +# Install dependencies with better error handling +echo "Installing dependencies..." + +# Install core dependencies first +CORE_DEPENDENCIES="llvm@15 openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf protobuf boost" + +for dep in $CORE_DEPENDENCIES; do + # Check if package is already installed + if brew list "$dep" &>/dev/null; then + version=$(brew list --versions "$dep" | awk '{print $2}') + echo "$dep $version already installed, skipping." + else + if brew install "$dep" 2>/dev/null; then + version=$(brew list --versions "$dep" | awk '{print $2}') + echo "$dep $version installed successfully." + else + echo "ERROR: Failed to install $dep" + echo "Attempting to reinstall $dep with --force ..." + if brew reinstall -f "$dep" 2>/dev/null; then + version=$(brew list --versions "$dep" | awk '{print $2}') + echo "$dep $version reinstalled successfully." + else + echo "ERROR: Failed to reinstall $dep, continuing..." + fi + fi + fi +done + +# Update PATH to include brew binaries +echo "Updating PATH..." +export PATH="$HOMEBREW_PREFIX/bin:/usr/local/bin:$PATH" +echo "Updated PATH: $PATH" + +# Comprehensive verification +echo "=== Verifying installations ===" + +# Verify essential commands +verify_command "protoc" "libprotoc" +verify_command "cmake" "cmake version" +verify_command "java" "openjdk" +verify_command "pkg-config" "pkg-config" + +# Verify protobuf specifically +echo "=== Protobuf verification ===" +if verify_command "protoc" "libprotoc"; then + echo "Protobuf version: $(protoc --version)" + echo "Protobuf include path: $(pkg-config --variable=includedir protobuf 2>/dev/null || echo 'not found')" + echo "Protobuf lib path: $(pkg-config --variable=libdir protobuf 2>/dev/null || echo 'not found')" +else + echo "ERROR: Protobuf verification failed" + echo "Searching for protobuf installations..." + find "$HOMEBREW_PREFIX" /usr/local -name "*protobuf*" 2>/dev/null | head -10 + exit 1 +fi + +# Verify libraries using dynamic paths +verify_library "protobuf" "$HOMEBREW_PREFIX/lib /usr/local/lib $HOMEBREW_CELLAR/protobuf/*/lib" +verify_library "boost" "$HOMEBREW_PREFIX/lib /usr/local/lib $HOMEBREW_CELLAR/boost/*/lib" +verify_library "llvm" "$HOMEBREW_PREFIX/lib /usr/local/lib $HOMEBREW_CELLAR/llvm@15/*/lib" + +# Verify cmake packages +verify_cmake_package "protobuf" +verify_cmake_package "boost" \ No newline at end of file diff --git a/scripts/macos/install_antlr4_cpp_runtime.sh b/scripts/macos/install_antlr4_cpp_runtime.sh index 777a83495..1355e9590 100644 --- a/scripts/macos/install_antlr4_cpp_runtime.sh +++ b/scripts/macos/install_antlr4_cpp_runtime.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This is a script to install the antlr4 runtime. -set -euxo pipefail +set -euo pipefail # Specify here target location. PREFIX=${PREFIX:-/usr/local} diff --git a/scripts/macos/install_aws-sdk-cpp.sh b/scripts/macos/install_aws-sdk-cpp.sh index bdf7fadd9..04c8a86cd 100755 --- a/scripts/macos/install_aws-sdk-cpp.sh +++ b/scripts/macos/install_aws-sdk-cpp.sh @@ -1,42 +1,133 @@ #!/usr/bin/env bash -set -euxo pipefail +set -euo pipefail PREFIX=${PREFIX:-/usr/local} AWSSDK_CPP_VERSION=1.11.524 # need at least 1.11.267 because of pyarrow bugs... -# check if dir exists (i.e. restored from cache, then skip) +# Check if AWS SDK is already installed if [ -d "${PREFIX}/include/aws" ]; then - echo ">> Skip aws-sdk-cpp compile from source, already exists." + echo "AWS SDK C++ already installed, skipping." exit 0 fi -echo ">> installing AWS SDK from source" -CPU_CORES=$(sysctl -n hw.physicalcpu) - -# if macOS is 10.x -> use this as minimum -MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" +echo "Installing AWS SDK C++ v${AWSSDK_CPP_VERSION} from source..." +# Get system info +CPU_CORES=$(sysctl -n hw.physicalcpu) MACOS_VERSION=$(sw_vers -productVersion) -echo "-- processing on MacOS ${MACOS_VERSION}" -function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } - MACOS_VERSION_MAJOR=$(echo "$MACOS_VERSION" | awk -F \. {'print $1'}) -if (( $MACOS_VERSION_MAJOR >= 11 )); then - echo "-- Newer MacOS detected (>=11.0), using more recent base target." - echo "-- Using minimum target ${MACOS_VERSION_MAJOR}.0" + +# Determine minimum deployment target +if (( MACOS_VERSION_MAJOR >= 11 )); then MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_VERSION_MAJOR}.0" + echo " Using macOS ${MACOS_VERSION_MAJOR}.0 as minimum target" else - # keep as is - echo "defaulting build to use as minimum target ${MINIMUM_TARGET}" + MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" + echo " Using macOS 10.13 as minimum target" +fi + +# Create temporary directory +TEMP_DIR="/tmp/aws-sdk-cpp-$$" +mkdir -p "$TEMP_DIR" +cd "$TEMP_DIR" + +# Cleanup function +cleanup() { + echo "Cleaning up temporary files..." + cd /tmp + rm -rf "$TEMP_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +echo "Cloning AWS SDK C++ repository..." +if ! git clone --recurse-submodules --quiet https://github.com/aws/aws-sdk-cpp.git; then + echo "ERROR: Failed to clone AWS SDK C++ repository" + exit 1 +fi + +cd aws-sdk-cpp + +echo "Checking out version ${AWSSDK_CPP_VERSION}..." +if ! git checkout --quiet "tags/${AWSSDK_CPP_VERSION}"; then + echo "ERROR: Failed to checkout version ${AWSSDK_CPP_VERSION}" + exit 1 +fi + +echo "Configuring build..." +mkdir -p build +cd build + +CMAKE_ARGS=( + ${MINIMUM_TARGET} + "-DCMAKE_INSTALL_PREFIX=${PREFIX}" + "-DCMAKE_BUILD_TYPE=Release" + "-DUSE_OPENSSL=ON" + "-DENABLE_TESTING=OFF" + "-DENABLE_UNITY_BUILD=ON" + "-DCPP_STANDARD=17" + "-DBUILD_SHARED_LIBS=OFF" + "-DBUILD_ONLY=s3;core;lambda;transfer" + ".." +) + +if ! cmake "${CMAKE_ARGS[@]}" >/dev/null 2>&1; then + echo "ERROR: CMake configuration failed" + echo "CMake arguments: ${CMAKE_ARGS[*]}" + exit 1 +fi + +echo "Building AWS SDK C++ (using ${CPU_CORES} cores)..." +echo " This may take several minutes. Building in progress..." + +# Function to show progress dots +show_progress() { + local pid=$1 + local dots=0 + while kill -0 "$pid" 2>/dev/null; do + sleep 10 + dots=$((dots + 1)) + if [ $dots -eq 1 ]; then + echo " Still building... (10s elapsed)" + elif [ $dots -eq 3 ]; then + echo " Still building... (30s elapsed)" + elif [ $dots -eq 6 ]; then + echo " Still building... (1m elapsed)" + elif [ $dots -eq 12 ]; then + echo " Still building... (2m elapsed)" + elif [ $dots -eq 18 ]; then + echo " Still building... (3m elapsed)" + elif [ $dots -eq 30 ]; then + echo " Still building... (5m elapsed)" + elif [ $((dots % 12)) -eq 0 ]; then + echo " Still building... ($((dots * 10 / 60))m elapsed)" + fi + done +} + +# Start the build in background and show progress +make -j"${CPU_CORES}" >/dev/null 2>&1 & +BUILD_PID=$! + +# Show progress while building +show_progress $BUILD_PID + +# Wait for build to complete and check result +wait $BUILD_PID +BUILD_EXIT_CODE=$? + +if [ $BUILD_EXIT_CODE -ne 0 ]; then + echo "ERROR: Build failed" + exit 1 +fi + +echo " Build completed successfully!" + +echo "Installing AWS SDK C++..." +if ! make install >/dev/null 2>&1; then + echo "ERROR: Installation failed" + exit 1 fi -cd /tmp && - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git && - cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && mkdir build && pushd build && - cmake ${MINIMUM_TARGET} -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;core;lambda;transfer" .. && - make -j${CPU_CORES} && - make install && - popd && - cd - || echo ">> error: AWS SDK failed" +echo "AWS SDK C++ v${AWSSDK_CPP_VERSION} installed successfully!" diff --git a/scripts/macos/user-config.jam b/scripts/macos/user-config.jam deleted file mode 100644 index c6f2a5431..000000000 --- a/scripts/macos/user-config.jam +++ /dev/null @@ -1,4 +0,0 @@ -using python : 3.6 : /Library/Frameworks/Python.framework/Versions/3.6/bin/python3.6 : /Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/ : /Library/Frameworks/Python.framework/Versions/3.6/lib ; -using python : 3.7 : /Library/Frameworks/Python.framework/Versions/3.7/bin/python3.7 : /Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/ : /Library/Frameworks/Python.framework/Versions/3.7/lib ; -using python : 3.8 : /Library/Frameworks/Python.framework/Versions/3.8/bin/python3.8 : /Library/Frameworks/Python.framework/Versions/3.8/include/python3.8/ : /Library/Frameworks/Python.framework/Versions/3.8/lib ; -using python : 3.9 : /Library/Frameworks/Python.framework/Versions/3.9/bin/python3.9 : /Library/Frameworks/Python.framework/Versions/3.9/include/python3.9/ : /Library/Frameworks/Python.framework/Versions/3.9/lib ; diff --git a/scripts/ubuntu2004/Dockerfile b/scripts/ubuntu2004/Dockerfile index 20f4a72b1..597e497fa 100644 --- a/scripts/ubuntu2004/Dockerfile +++ b/scripts/ubuntu2004/Dockerfile @@ -3,8 +3,6 @@ FROM ubuntu:20.04 # build using docker build -t tuplex/ubuntu:20.04 . -MAINTAINER Leonhard Spiegelberg "leonhard@brown.edu" - ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /opt/sbin diff --git a/scripts/ubuntu2204/Dockerfile b/scripts/ubuntu2204/Dockerfile index d977f8b0c..9d9974690 100644 --- a/scripts/ubuntu2204/Dockerfile +++ b/scripts/ubuntu2204/Dockerfile @@ -3,8 +3,6 @@ FROM ubuntu:22.04 # build using docker build -t tuplex/ubuntu:22.04 . -MAINTAINER Leonhard Spiegelberg "leonhard@brown.edu" - ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /opt/sbin diff --git a/setup.py b/setup.py index 3d26dada7..8a62b52f0 100644 --- a/setup.py +++ b/setup.py @@ -117,50 +117,107 @@ def run_command(cmd, cwd, env): if in_google_colab(): logging.debug('Building dependencies for Google Colab environment') - install_dependencies = [ - 'urllib3!=1.25.0,!=1.25.1,>=1.21.1', - 'folium>=0.2.1' - 'requests', - 'attrs>=19.2.0', - 'dill>=0.2.7.1', - 'pluggy', - 'py>=1.5.2', - 'pygments>=2.4.1', - 'six>=1.11.0', - 'wcwidth>=0.1.7', - 'astor', - 'prompt_toolkit', - 'jedi', - "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", - "cloudpickle>=2.1.0;python_version=='3.10'", - "cloudpickle>=2.2.1;python_version>='3.11'", - 'PyYAML>=3.13', - 'psutil', - 'pymongo', - 'iso8601' - ] + # Check if history server should be included + include_historyserver = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') + + if include_historyserver: + install_dependencies = [ + 'urllib3!=1.25.0,!=1.25.1,>=1.21.1', + 'folium>=0.2.1' + 'requests', + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'pymongo', + 'iso8601' + ] + else: + # Exclude webui dependencies when history server is not included + # Note: iso8601 is still needed by core tuplex code, not just history server + install_dependencies = [ + 'urllib3!=1.25.0,!=1.25.1,>=1.21.1', + 'folium>=0.2.1' + 'requests', + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'iso8601' + ] else: logging.debug('Building dependencies for non Colab environment') - install_dependencies = webui_dependencies + [ - 'attrs>=19.2.0', - 'dill>=0.2.7.1', - 'pluggy', - 'py>=1.5.2', - 'pygments>=2.4.1', - 'six>=1.11.0', - 'wcwidth>=0.1.7', - 'astor', - 'prompt_toolkit', - 'jedi', - "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", - "cloudpickle>=2.1.0;python_version=='3.10'", - "cloudpickle>=2.2.1;python_version>='3.11'", - 'PyYAML>=3.13', - 'psutil', - 'pymongo', - 'iso8601' - ] + aws_lambda_dependencies + # Check if history server should be included + include_historyserver = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') + + if include_historyserver: + install_dependencies = webui_dependencies + [ + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'pymongo', + 'iso8601' + ] + aws_lambda_dependencies + else: + # Exclude webui dependencies when history server is not included + # Note: iso8601 is still needed by core tuplex code, not just history server + install_dependencies = [ + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'iso8601' + ] + aws_lambda_dependencies def ninja_installed(): # check whether ninja is on the path @@ -658,7 +715,18 @@ def read_readme(): def reorg_historyserver(): """ reorganize historyserver to become part of pip package. + Can be disabled by setting TUPLEX_INCLUDE_HISTORYSERVER=False """ + # Check if history server should be included + env_value = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True') + include_historyserver = env_value.lower() in ('true', '1', 'yes', 'on') + + logging.info(f'TUPLEX_INCLUDE_HISTORYSERVER={env_value}, include_historyserver={include_historyserver}') + + if not include_historyserver: + logging.info('History server excluded from package (TUPLEX_INCLUDE_HISTORYSERVER=False)') + return [] + # get absolute path of this file's location import pathlib current_path = pathlib.Path(__file__).parent.resolve() @@ -680,12 +748,16 @@ def tplx_package_data(): package_data = { # include libs in libexec - 'tuplex.libexec' : ['*.so', '*.dylib'], - 'tuplex.historyserver': ['thserver/templates/*.html', 'thserver/static/css/*.css', 'thserver/static/css/styles/*.css', - 'thserver/static/img/*.*', 'thserver/static/js/*.js', 'thserver/static/js/modules/*.js', - 'thserver/static/js/styles/*.css'] + 'tuplex.libexec' : ['*.so', '*.dylib'] } + # Check if history server should be included + include_historyserver = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') + if include_historyserver: + package_data['tuplex.historyserver'] = ['thserver/templates/*.html', 'thserver/static/css/*.css', 'thserver/static/css/styles/*.css', + 'thserver/static/img/*.*', 'thserver/static/js/*.js', 'thserver/static/js/modules/*.js', + 'thserver/static/js/styles/*.css'] + # package lambda as well? lambda_zip = os.environ.get('TUPLEX_LAMBDA_ZIP', None) if lambda_zip and os.path.isfile(lambda_zip): @@ -694,6 +766,10 @@ def tplx_package_data(): # The information here can also be placed in setup.cfg - better separation of # logic and declaration, and simpler if you include description/version in a file. +# +# Environment variables: +# - TUPLEX_INCLUDE_HISTORYSERVER: Set to 'False' to exclude the history server +# from the package (default: 'True'). This reduces package size and dependencies. setup(name="tuplex", python_requires='>=3.9.0', version="0.3.7", @@ -741,7 +817,7 @@ def tplx_package_data(): 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', ], - scripts=['tuplex/historyserver/bin/tuplex-webui'], + scripts=['tuplex/historyserver/bin/tuplex-webui'] if os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') else [], project_urls={ "Bug Tracker": "https://github.com/tuplex", "Documentation": "https://tuplex.cs.brown.edu/python-api.html", diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index 7adf4b7ef..8e97164f4 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -1,6 +1,11 @@ # (c) 2017-2023 Leonhard Spiegelberg cmake_minimum_required(VERSION 3.16 FATAL_ERROR) +# Set policies to suppress warnings +if(POLICY CMP0144) + cmake_policy(SET CMP0144 NEW) # FindPython3: Honor Python3_FIND_REGISTRY +endif() + # top-level language specification # enable c++17 set(CMAKE_CXX_STANDARD 17) @@ -223,9 +228,23 @@ if(PROTOBUF_REQUIRED) message(STATUS "Build requires Protobuf") set(Protobuf_USE_STATIC_LIBS ON) # https://github.com/protocolbuffers/protobuf/issues/12637 - find_package(Protobuf CONFIG) + + # Try CONFIG method first (preferred for modern protobuf installations) + find_package(Protobuf CONFIG QUIET) + if(NOT Protobuf_FOUND) + message(STATUS "Protobuf CONFIG not found, trying REQUIRED method") find_package(Protobuf REQUIRED) + else() + message(STATUS "Found Protobuf using CONFIG method") + endif() + + # Final verification + if(NOT Protobuf_FOUND) + message(FATAL_ERROR "Protobuf not found. Please install protobuf development libraries. " + "On macOS: brew install protobuf\n" + "On Ubuntu: sudo apt-get install libprotobuf-dev protobuf-compiler\n" + "On CentOS: sudo yum install protobuf-devel protobuf-compiler") endif() if(Protobuf_LIBRARY) @@ -388,7 +407,12 @@ endif() # use march=native if available and NOT building wheels # specific Release flags, i.e. targeting loop unrolling. -ucm_add_flags(-m64 -O3 -funroll-loops CONFIG Release) +# Only use -m64 for x86_64 architectures +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + ucm_add_flags(-m64 -O3 -funroll-loops CONFIG Release) +else() + ucm_add_flags(-O3 -funroll-loops CONFIG Release) +endif() # for GCC add stack protector # -fstack-protector-strong @@ -411,30 +435,38 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) # for release mode, don't use Ofast unless you're ok to break NAN/INF compatibility. if(BUILD_FOR_LAMBDA) - ucm_add_flags(-march=haswell) + # Only use x86_64 specific march flags for x86_64 architectures + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + ucm_add_flags(-march=haswell) + endif() endif() # add compiler flags for specific SIMD instructions -include(FindSSE) -include(FindSIMD) -CHECK_FOR_SSE42() -CHECK_FOR_AVX() -# add compiler flags -if(HAVE_SSE42) - message(STATUS "Found SSE4.2 support, adding -msse4.2") - # no msvc support, so use clang/gcc flag approach! - add_compile_options("-msse4.2") -endif() -if(HAVE_AVX) - # only add avx (because not any architecture supports it) if native build is enabled or build for lambda - # because Haswell supports both AVX and AVX2! - if(BUILD_NATIVE OR BUILD_FOR_LAMBDA) - message(STATUS "Found AVX support, adding -mavx") +# Only include SSE/AVX checks for x86_64 architectures +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + include(FindSSE) + include(FindSIMD) + CHECK_FOR_SSE42() + CHECK_FOR_AVX() + # add compiler flags + if(HAVE_SSE42) + message(STATUS "Found SSE4.2 support, adding -msse4.2") # no msvc support, so use clang/gcc flag approach! - add_compile_options("-mavx") + add_compile_options("-msse4.2") + endif() + if(HAVE_AVX) + # only add avx (because not any architecture supports it) if native build is enabled or build for lambda + # because Haswell supports both AVX and AVX2! + if(BUILD_NATIVE OR BUILD_FOR_LAMBDA) + message(STATUS "Found AVX support, adding -mavx") + # no msvc support, so use clang/gcc flag approach! + add_compile_options("-mavx") + endif() + else() + message(STATUS "no avx support, build with -DBUILD_NATIVE=ON to enable") endif() else() - message(STATUS "no avx support, build with -DBUILD_NATIVE=ON to enable") + message(STATUS "Skipping SSE/AVX checks for non-x86_64 architecture: ${CMAKE_SYSTEM_PROCESSOR}") endif() ########################################################################### @@ -587,6 +619,8 @@ if(PYTHON3_EXECUTABLE) set(Python3_EXECUTABLE ${Python3_EXECUTABLE}) message(STATUS "Using specific python executable ${PYTHON3_EXECUTABLE}") unset(PYTHON3_VERSION) + # Unset the legacy Python_EXECUTABLE variable to avoid CMake warnings + unset(Python_EXECUTABLE CACHE) # get version from executable execute_process (COMMAND "${PYTHON3_EXECUTABLE}" -c "import sys;print('{}.{}.{}'.format(sys.version_info.major,sys.version_info.minor,sys.version_info.micro))" @@ -785,7 +819,7 @@ execute_process (COMMAND "${Python3_EXECUTABLE}" -c "from distutils import sysco message(STATUS "Python3 extension module suffix is: ${PYTHON_MODULE_EXTENSION}") set(Boost_NO_WARN_NEW_VERSIONS ON) # cmake 3.20 finally has a switch to kill the boost warnings... -set(BOOST_COMPONENTS iostreams thread system filesystem) +set(BOOST_COMPONENTS iostreams thread filesystem) if(Python3_FOUND) if(Python3_VERSION_MAJOR LESS 3 OR Python3_VERSION_MINOR LESS 5) message(FATAL_ERROR "Tuplex requires python3.5 at least, found incompatible python ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}") @@ -797,7 +831,10 @@ if(Python3_FOUND) endif() message(STATUS "Found python${Python3_VERSION} - if you'd like to change a to different python version, use -DPython3_ROOT_DIR= or -DPYTHON3_VERSION= or set an environment variable PYTHON3_VERSION") - set(Boost_NO_BOOST_CMAKE ON) # findboost from cmake is buggy and does not work, explicitly disable here + + # For cmake < 4.0, finding boost may be buggy. + # set(Boost_NO_BOOST_CMAKE ON) # findboost from cmake is buggy and does not work, explicitly disable here + if(APPLE AND BREW_FOUND) # i.e. boost-python via brew? --> check maybe better in the future... set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) # gets rid off annoying boost warning. endif() diff --git a/tuplex/cmake/ExternalAntlr4Cpp.cmake b/tuplex/cmake/ExternalAntlr4Cpp.cmake index e4b8d1ae4..3ecf56827 100755 --- a/tuplex/cmake/ExternalAntlr4Cpp.cmake +++ b/tuplex/cmake/ExternalAntlr4Cpp.cmake @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR) if(${CMAKE_VERSION} VERSION_GREATER "3.19.0") - cmake_policy(SET CMP0114 OLD) # policy update in 3.19, explicitly use old behavior + cmake_policy(SET CMP0114 NEW) # policy update in 3.19, use new behavior endif() include(ExternalProject) diff --git a/tuplex/core/CMakeLists.txt b/tuplex/core/CMakeLists.txt index 031cb9fe7..9ea57b0eb 100755 --- a/tuplex/core/CMakeLists.txt +++ b/tuplex/core/CMakeLists.txt @@ -114,14 +114,19 @@ target_link_libraries(libcore ${YAMLCPP_LIBRARY} ${CURL_LIBRARIES} ${AWSSDK_LINK_LIBRARIES} - ${Protobuf_LIBRARIES} - proto-objects - protobuf::libprotobuf Boost::iostreams Boost::thread - Boost::system Boost::filesystem util ${ZLIB_LIBRARIES} ${ZSTD_LIBRARIES} ) + +# Add protobuf dependencies only if required. +if(BUILD_WITH_AWS OR BUILD_WITH_ORC) + target_link_libraries(libcore + ${Protobuf_LIBRARIES} + proto-objects + protobuf::libprotobuf + ) +endif() diff --git a/tuplex/core/src/physical/TextReader.cc b/tuplex/core/src/physical/TextReader.cc index 32d0da0ad..1c5f59f27 100644 --- a/tuplex/core/src/physical/TextReader.cc +++ b/tuplex/core/src/physical/TextReader.cc @@ -17,7 +17,7 @@ // use simd intrinsics or ARM Neon translation layer #if (defined __x86_64__) #include -#elif (defined __arm64__) +#elif (defined(__aarch64__) || defined(__arm64__)) #include #else #error "unsupported platform for intrinsics" diff --git a/tuplex/io/CMakeLists.txt b/tuplex/io/CMakeLists.txt index d7ffd541a..c22d02395 100644 --- a/tuplex/io/CMakeLists.txt +++ b/tuplex/io/CMakeLists.txt @@ -109,7 +109,7 @@ if(BUILD_WITH_ORC) "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep/contrib/cmake_unofficial") endif() - ExternalProject_Add (lz4_ep URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"s + ExternalProject_Add (lz4_ep URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" ${LZ4_CONFIGURE} BUILD_BYPRODUCTS "${LZ4_STATIC_LIB}") @@ -210,8 +210,11 @@ if(BUILD_WITH_ORC) ExternalProject_Add(orc GIT_REPOSITORY https://github.com/apache/orc.git - GIT_TAG rel/release-2.1.3 + GIT_TAG rel/release-2.2.0 TIMEOUT 5 + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/orc.diff ${CMAKE_BINARY_DIR}/orc-patch.diff && + cd ${CMAKE_BINARY_DIR}/third_party/src/orc && + patch -p1 < ${CMAKE_BINARY_DIR}/orc-patch.diff CMAKE_ARGS -DBUILD_LIBHDFSPP=OFF -DSNAPPY_HOME=${SNAPPY_HOME} -DLZ4_HOME=${LZ4_HOME} -DZSTD_HOME=${ZSTD_HOME} -DZLIB_HOME=${ZLIB_HOME} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} @@ -233,7 +236,6 @@ if(BUILD_WITH_ORC) set_target_properties(liborc PROPERTIES IMPORTED_LOCATION ${orc_LIBRARY}) add_dependencies(liborc orc) - include_directories(${orc_INCLUDE_DIR}) set(ORC_LIBRARIES ${SNAPPY_LIBRARIES} ${LZ4_LIBRARIES} @@ -242,6 +244,7 @@ if(BUILD_WITH_ORC) set(ORC_LIBRARIES ${SNAPPY_LIBRARIES} ${LZ4_LIBRARIES} PARENT_SCOPE) endif() + add_library(libio OBJECT ${CMAKE_CURRENT_BINARY_DIR} ${SOURCES} ${INCLUDES}) set_target_properties(libio PROPERTIES PREFIX "") diff --git a/tuplex/io/patches/orc.diff b/tuplex/io/patches/orc.diff new file mode 100644 index 000000000..778f63184 --- /dev/null +++ b/tuplex/io/patches/orc.diff @@ -0,0 +1,74 @@ +diff --git a/c++/src/io/InputStream.cc b/c++/src/io/InputStream.cc +index 06ef40b..5e1dc00 100644 +--- a/c++/src/io/InputStream.cc ++++ b/c++/src/io/InputStream.cc +@@ -112,8 +112,8 @@ namespace orc { + return false; + } + +- google::protobuf::int64 SeekableArrayInputStream::ByteCount() const { +- return static_cast(position_); ++ int64_t SeekableArrayInputStream::ByteCount() const { ++ return static_cast(position_); + } + + void SeekableArrayInputStream::seek(PositionProvider& seekPosition) { +diff --git a/c++/src/io/InputStream.hh b/c++/src/io/InputStream.hh +index 07aa623..5f526a2 100644 +--- a/c++/src/io/InputStream.hh ++++ b/c++/src/io/InputStream.hh +@@ -29,6 +29,8 @@ + #include + #include + ++#include ++ + namespace orc { + + void printBuffer(std::ostream& out, const char* buffer, uint64_t length); +@@ -72,7 +74,7 @@ namespace orc { + virtual bool Next(const void** data, int* size) override; + virtual void BackUp(int count) override; + virtual bool Skip(int count) override; +- virtual google::protobuf::int64 ByteCount() const override; ++ virtual int64_t ByteCount() const override; + virtual void seek(PositionProvider& position) override; + virtual std::string getName() const override; + }; +diff --git a/c++/src/io/OutputStream.cc b/c++/src/io/OutputStream.cc +index fbf1ca6..a55050d 100644 +--- a/c++/src/io/OutputStream.cc ++++ b/c++/src/io/OutputStream.cc +@@ -65,8 +65,8 @@ namespace orc { + // PASS + } + +- google::protobuf::int64 BufferedOutputStream::ByteCount() const { +- return static_cast(dataBuffer_->size()); ++ int64_t BufferedOutputStream::ByteCount() const { ++ return static_cast(dataBuffer_->size()); + } + + bool BufferedOutputStream::WriteAliasedRaw(const void*, int) { +diff --git a/c++/src/io/OutputStream.hh b/c++/src/io/OutputStream.hh +index 6319de9..51e5fc4 100644 +--- a/c++/src/io/OutputStream.hh ++++ b/c++/src/io/OutputStream.hh +@@ -24,6 +24,8 @@ + #include "orc/OrcFile.hh" + #include "wrap/zero-copy-stream-wrapper.h" + ++#include ++ + namespace orc { + + /** +@@ -61,7 +63,7 @@ namespace orc { + + virtual bool Next(void** data, int* size) override; + virtual void BackUp(int count) override; +- virtual google::protobuf::int64 ByteCount() const override; ++ virtual int64_t ByteCount() const override; + virtual bool WriteAliasedRaw(const void* data, int size) override; + virtual bool AllowsAliasing() const override; + diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index abb2bcaf3..a78e58945 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -12,22 +12,29 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) SET(MODULE_NAME tuplex) # deprecated -include(FetchContent) #set(PYBIND11_NOPYTHON ON CACHE INTERNAL "") message(STATUS "Pybind11 uses python version ${Python3_VERSION}") set(PYBIND11_FINDPYTHON OFF CACHE INTERNAL "") set(PYBIND11_PYTHON_VERSION "${Python3_VERSION}" CACHE INTERNAL "") + +# 3.0.0 requires python 3.10+, 2.13.6 supports python 3.9. +if(Python3_VERSION VERSION_LESS "3.10") + set(pybind11_version v2.13.6) +else() + set(pybind11_version v3.0.1) +endif() + FetchContent_Declare(pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v2.13.6) + GIT_TAG ${pybind11_version}) FetchContent_GetProperties(pybind11) if(NOT pybind11_POPULATED) + # Disable cmake warning, need to update to FetchContent_MakeAvailable eventually. + cmake_policy(SET CMP0169 OLD) FetchContent_Populate(pybind11) + cmake_policy(SET CMP0169 NEW) add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) endif() -# Source code & linking -file(GLOB_RECURSE SOURCES src/*.cc) - # this is the syntax for the old deprecated packages, switching out to newer one. #PYTHON_ADD_MODULE(${MODULE_NAME} ${SOURCES}) message(STATUS "libs: ${Python3_LIBRARIES}") @@ -53,11 +60,93 @@ message(STATUS "includes: ${Python3_INCLUDE_DIRS}") #include(${pybind11_SOURCE_DIR}/tools/pybind11Tools.cmake) #include(${pybind11_SOURCE_DIR}/tools/pybind11NewTools.cmake) - +# Method 1: # cf. https://pybind11.readthedocs.io/en/stable/cmake/index.html pybind11_add_module(${MODULE_NAME} ${SOURCES}) set(PYMODULE "PYBIND11_MODULE(${MODULE_NAME}, m)") target_compile_definitions(${MODULE_NAME} PRIVATE PYMODULE=${PYMODULE}) + +# Macro/function to detect musl libc or Alpine Linux +function(detect_musl OUT_VAR) + include(CheckCXXSourceCompiles) + set(_musl_detected OFF) + + # Method 1: Check for musl-specific functions that don't exist in glibc + check_cxx_source_compiles(" + #include + #include + int main() { + // musl has __progname, glibc doesn't + extern char *__progname; + return 0; + } + " _musl_progname_detected) + + if(_musl_progname_detected) + set(_musl_detected ON) + else() + # Method 2: Check for musl-specific header behavior + check_cxx_source_compiles(" + #include + #include + int main() { + // musl defines MSG_NOSIGNAL, glibc doesn't always + int flag = MSG_NOSIGNAL; + return 0; + } + " _musl_socket_detected) + + if(_musl_socket_detected) + set(_musl_detected ON) + else() + # Method 3: Check for Alpine Linux specific behavior + check_cxx_source_compiles(" + #include + int main() { + // Check if we're on Alpine/musl by looking for specific defines + #ifdef __GLIBC__ + #error glibc detected + #endif + return 0; + } + " _musl_features_detected) + + if(_musl_features_detected) + set(_musl_detected ON) + endif() + endif() + endif() + + if(APPLE) + set(_musl_detected OFF) + endif() + + set(${OUT_VAR} ${_musl_detected} PARENT_SCOPE) +endfunction() + +# Detect whether muslc is being used. +detect_musl(MUSL_DETECTED) + +if(MUSL_DETECTED) + message(STATUS "muslc detected, adjusting linker flags") + + # # Option 1: + # # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. + # message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") + # set_target_properties(${MODULE_NAME} PROPERTIES + # INTERPROCEDURAL_OPTIMIZATION OFF + # CXX_VISIBILITY_PRESET hidden + # VISIBILITY_INLINES_HIDDEN ON) + + # Option 2: + # Pass the right linker flags for Alpine. In addition, need to have a list of symbols to preserve as + # they need to be present when using the C-extension module. + target_link_options(${MODULE_NAME} PRIVATE + "-Wl,--export-dynamic" + "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt" + "-Wl,--allow-shlib-undefined") +endif() + add_dependencies(${MODULE_NAME} libcore libcodegen) target_include_directories(${MODULE_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include @@ -72,7 +161,6 @@ target_link_libraries(${MODULE_NAME} PUBLIC Backward::Interface libio Boost::iostreams Boost::thread - Boost::system Boost::filesystem) else() target_link_libraries(${MODULE_NAME} @@ -84,7 +172,6 @@ else() libio Boost::iostreams Boost::thread - Boost::system Boost::filesystem) endif() diff --git a/tuplex/python/include/PythonCommon.h b/tuplex/python/include/PythonCommon.h index 205f2b63d..675d78ad0 100644 --- a/tuplex/python/include/PythonCommon.h +++ b/tuplex/python/include/PythonCommon.h @@ -120,7 +120,7 @@ namespace tuplex { LogMessage msg; msg.message = std::string(spdlog_msg.payload.data()); msg.timestamp = spdlog_msg.time; - msg.logger = *spdlog_msg.logger_name; + msg.logger = std::string(spdlog_msg.logger_name.begin(), spdlog_msg.logger_name.end()); msg.level = spdlog_msg.level; _messageBuffer.push_back(msg); } diff --git a/tuplex/python/preserve_symbols.txt b/tuplex/python/preserve_symbols.txt new file mode 100644 index 000000000..2dcc2267b --- /dev/null +++ b/tuplex/python/preserve_symbols.txt @@ -0,0 +1,193 @@ +PyInit_tuplex +PyInit_tuplex +_Py_Dealloc +PyLong_FromLong +PyModuleDef_Init +PyUnicode_AsUTF8String +PyDict_SetItem +PyEval_GetBuiltins +PyUnicode_FromFormat +PySlice_New +PyDict_SetItemString +PyCapsule_GetName +PyImport_ImportModule +PyType_IsSubtype +PyException_SetTraceback +PyModule_GetDict +PyObject_SetAttrString +PyErr_Fetch +_PyThreadState_UncheckedGet +PyThread_acquire_lock +PyDict_Copy +PyUnicode_AsWideCharString +PyUnicode_FromString +PyObject_GetAttrString +PyUnicode_AsUTF8 +PyLong_AsLong +PyObject_CallFunctionObjArgs +PyObject_GetItem +PyObject_SetItem +PyObject_Size +PyObject_Str +PyObject_Repr +PyObject_Hash +PyObject_IsTrue +PyObject_Not +PyObject_RichCompare +PyObject_RichCompareBool +PyObject_GetIter +PyIter_Next +PyObject_GenericGetAttr +PyObject_GenericSetAttr +PyObject_GetAttr +PyObject_SetAttr +PyObject_DelAttr +PyObject_DelAttrString +PyObject_GetAttrString +PyObject_HasAttr +PyObject_HasAttrString +PyObject_GetItem +PyObject_SetItem +PyObject_DelItem +PyObject_Contains +PyObject_Length +PyObject_Size +PyObject_GetSlice +PyObject_SetSlice +PyObject_DelSlice +PySequence_Check +PySequence_Size +PySequence_Length +PySequence_Concat +PySequence_Repeat +PySequence_GetItem +PySequence_SetItem +PySequence_DelItem +PySequence_GetSlice +PySequence_SetSlice +PySequence_DelSlice +PySequence_Count +PySequence_Index +PySequence_List +PySequence_Tuple +PySequence_Fast +PySequence_Fast_GET_ITEM +PySequence_Fast_GET_SIZE +PySequence_InPlaceConcat +PySequence_InPlaceRepeat +PyMapping_Check +PyMapping_Size +PyMapping_Length +PyMapping_HasKey +PyMapping_HasKeyString +PyMapping_Keys +PyMapping_Values +PyMapping_Items +PyMapping_GetItemString +PyMapping_SetItemString +PyMapping_DelItemString +PyMapping_Clear +PyNumber_Check +PyNumber_Add +PyNumber_Subtract +PyNumber_Multiply +PyNumber_MatrixMultiply +PyNumber_FloorDivide +PyNumber_TrueDivide +PyNumber_Remainder +PyNumber_Divmod +PyNumber_Power +PyNumber_Negative +PyNumber_Positive +PyNumber_Absolute +PyNumber_Invert +PyNumber_Lshift +PyNumber_Rshift +PyNumber_And +PyNumber_Xor +PyNumber_Or +PyNumber_Index +PyNumber_ToBase +PyNumber_InPlaceAdd +PyNumber_InPlaceSubtract +PyNumber_InPlaceMultiply +PyNumber_InPlaceMatrixMultiply +PyNumber_InPlaceFloorDivide +PyNumber_InPlaceTrueDivide +PyNumber_InPlaceRemainder +PyNumber_InPlacePower +PyNumber_InPlaceLshift +PyNumber_InPlaceRshift +PyNumber_InPlaceAnd +PyNumber_InPlaceXor +PyNumber_InPlaceOr +PyNumber_Long +PyNumber_Float +PyNumber_Int +PyNumber_Complex +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode diff --git a/tuplex/python/requirements.txt b/tuplex/python/requirements.txt index 1c53763a5..587553299 100644 --- a/tuplex/python/requirements.txt +++ b/tuplex/python/requirements.txt @@ -17,6 +17,7 @@ jedi>=0.13.2 cloudpickle>=0.6.1,<2.0.0;python_version<'3.10' cloudpickle>=2.1.0;python_version=='3.10' cloudpickle>=2.2.1;python_version>='3.11' +numpy>=1.15.0 PyYAML>=3.13 psutil pymongo diff --git a/tuplex/runtime/src/Runtime.cc b/tuplex/runtime/src/Runtime.cc index 426af001a..f84d24b7d 100644 --- a/tuplex/runtime/src/Runtime.cc +++ b/tuplex/runtime/src/Runtime.cc @@ -18,11 +18,13 @@ #include // <-- implemented in StringUtils #include #include +#include #include #include #include #include #include // for aligned_malloc +#include // for va_arg #define RUNTIME_DEFAULT_BLOCK_SIZE (4 * 1024 * 1024) diff --git a/tuplex/test/CMakeLists.txt b/tuplex/test/CMakeLists.txt index 12d571a07..c20c484bb 100755 --- a/tuplex/test/CMakeLists.txt +++ b/tuplex/test/CMakeLists.txt @@ -16,18 +16,68 @@ find_package(Threads REQUIRED) #endif() if (Python3_FOUND) + message(STATUS "Python3 executable: ${Python3_EXECUTABLE}") + message(STATUS "Python3 version: ${Python3_VERSION}") + + # First, let's see what packages are actually installed + message(STATUS "Checking installed Python packages...") + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import pkg_resources; installed = [d.project_name for d in pkg_resources.working_set]; print('\\n'.join(sorted(installed)))" + OUTPUT_VARIABLE installed_packages + RESULT_VARIABLE ret_packages + ) + if("${ret_packages}" STREQUAL "0") + message(STATUS "Installed packages:") + message(STATUS "${installed_packages}") + else() + message(WARNING "Could not list installed packages") + endif() + + # Check Python path and site-packages + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import sys; print('\\n'.join(sys.path))" + OUTPUT_VARIABLE python_path + RESULT_VARIABLE ret_path + ) + if("${ret_path}" STREQUAL "0") + message(STATUS "Python sys.path:") + message(STATUS "${python_path}") + endif() + # check that cloudpickle is installed via import - set(cmd -c "import cloudpickle") - execute_process(COMMAND ${Python3_EXECUTABLE} ${cmd} RESULT_VARIABLE ret) + message(STATUS "Testing cloudpickle import...") + set(cmd -c "import cloudpickle; print('cloudpickle version:', cloudpickle.__version__)") + execute_process( + COMMAND ${Python3_EXECUTABLE} ${cmd} + OUTPUT_VARIABLE cloudpickle_output + ERROR_VARIABLE cloudpickle_error + RESULT_VARIABLE ret + ) if(NOT "${ret}" STREQUAL "0") - message(FATAL_ERROR "Could not find cloudpickle module, please install via pip3 install cloudpickle.") + message(STATUS "cloudpickle import failed with return code: ${ret}") + message(STATUS "cloudpickle stdout: ${cloudpickle_output}") + message(STATUS "cloudpickle stderr: ${cloudpickle_error}") + message(FATAL_ERROR "Could not find cloudpickle module. Please install via 'pip3 install cloudpickle'. The module is required for testing. Check the output above for details about what packages are currently installed.") + else() + message(STATUS "cloudpickle import successful: ${cloudpickle_output}") endif() # check that numpy is installed too for testing purposes... - set(cmd -c "import numpy") - execute_process(COMMAND ${Python3_EXECUTABLE} ${cmd} RESULT_VARIABLE ret) + message(STATUS "Testing numpy import...") + set(cmd -c "import numpy; print('numpy version:', numpy.__version__)") + execute_process( + COMMAND ${Python3_EXECUTABLE} ${cmd} + OUTPUT_VARIABLE numpy_output + ERROR_VARIABLE numpy_error + RESULT_VARIABLE ret + ) if(NOT "${ret}" STREQUAL "0") - message(FATAL_ERROR "Could not find numpy module, please install via pip3 install numpy.") + message(STATUS "numpy import failed with return code: ${ret}") + message(STATUS "numpy stdout: ${numpy_output}") + message(STATUS "numpy stderr: ${numpy_error}") + message(FATAL_ERROR "Could not find numpy module. Please install via 'pip3 install numpy'. The module is required for testing. Check the output above for details about what packages are currently installed.") + else() + message(STATUS "numpy import successful: ${numpy_output}") endif() else() diff --git a/tuplex/test/adapters/cpython/CMakeLists.txt b/tuplex/test/adapters/cpython/CMakeLists.txt index 0f518fe41..c6ff7aa34 100644 --- a/tuplex/test/adapters/cpython/CMakeLists.txt +++ b/tuplex/test/adapters/cpython/CMakeLists.txt @@ -30,4 +30,4 @@ TARGET_LINK_LIBRARIES(testcpythonadapter ${CURSES_LIBRARY} ) -gtest_add_tests(TARGET testcpythonadapter TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testcpythonadapter) \ No newline at end of file diff --git a/tuplex/test/codegen/CMakeLists.txt b/tuplex/test/codegen/CMakeLists.txt index 10fe5cd7d..a1c201f1b 100755 --- a/tuplex/test/codegen/CMakeLists.txt +++ b/tuplex/test/codegen/CMakeLists.txt @@ -23,4 +23,4 @@ TARGET_LINK_LIBRARIES(testcodegen runtime ) -gtest_add_tests(TARGET testcodegen TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testcodegen) \ No newline at end of file diff --git a/tuplex/test/core/CMakeLists.txt b/tuplex/test/core/CMakeLists.txt index 97d7b92d6..8451ffdd0 100644 --- a/tuplex/test/core/CMakeLists.txt +++ b/tuplex/test/core/CMakeLists.txt @@ -21,4 +21,4 @@ target_link_libraries(testcore ${CURSES_LIBRARY} ${LLVM_LIBRARIES}) -gtest_add_tests(TARGET testcore TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testcore) \ No newline at end of file diff --git a/tuplex/test/io/CMakeLists.txt b/tuplex/test/io/CMakeLists.txt index be92a0159..13c7fcc12 100644 --- a/tuplex/test/io/CMakeLists.txt +++ b/tuplex/test/io/CMakeLists.txt @@ -18,11 +18,16 @@ TARGET_LINK_LIBRARIES(testio ${GTest_LIBRARIES} ${AWSSDK_LINK_LIBRARIES} ${Python3_LIBRARIES} - ${Protobuf_LIBRARIES} - ${Protobuf_LITE_LIBRARIES} - ${Protobuf_LIBRARIES} ${CURSES_LIBRARY} - protobuf::libprotobuf ) -gtest_add_tests(TARGET testio TEST_PREFIX "") +# Add protobuf dependencies only if required +if(BUILD_WITH_AWS OR BUILD_WITH_ORC) + TARGET_LINK_LIBRARIES(testio + ${Protobuf_LIBRARIES} + ${Protobuf_LITE_LIBRARIES} + protobuf::libprotobuf + ) +endif() + +gtest_add_tests(TARGET testio) diff --git a/tuplex/test/runtime/CMakeLists.txt b/tuplex/test/runtime/CMakeLists.txt index cff9af37d..3ecce7c64 100755 --- a/tuplex/test/runtime/CMakeLists.txt +++ b/tuplex/test/runtime/CMakeLists.txt @@ -24,4 +24,4 @@ TARGET_LINK_LIBRARIES(testruntime ${CURSES_LIBRARY} ) -gtest_add_tests(TARGET testruntime TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testruntime) \ No newline at end of file diff --git a/tuplex/test/utils/CMakeLists.txt b/tuplex/test/utils/CMakeLists.txt index c2956ce5b..8be463543 100644 --- a/tuplex/test/utils/CMakeLists.txt +++ b/tuplex/test/utils/CMakeLists.txt @@ -15,4 +15,4 @@ TARGET_LINK_LIBRARIES(testutils ${AWSSDK_LINK_LIBRARIES} ) -gtest_add_tests(TARGET testutils TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testutils) \ No newline at end of file diff --git a/tuplex/test/wrappers/CMakeLists.txt b/tuplex/test/wrappers/CMakeLists.txt index 84b753641..1db010855 100644 --- a/tuplex/test/wrappers/CMakeLists.txt +++ b/tuplex/test/wrappers/CMakeLists.txt @@ -46,4 +46,4 @@ else() ) endif() -gtest_add_tests(TARGET testwrappers TEST_PREFIX "") +gtest_add_tests(TARGET testwrappers) diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index 472c02e86..4b38baef0 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -10,9 +10,12 @@ include(ExternalProject) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) ExternalProject_Add(fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git - GIT_TAG 6.2.0 + GIT_TAG 11.2.0 GIT_CONFIG advice.detachedHead=false TIMEOUT 5 + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/fmtlib.diff ${CMAKE_BINARY_DIR}/fmtlib-patch.diff && + cd ${CMAKE_BINARY_DIR}/third_party/src/fmt && + patch -p1 < ${CMAKE_BINARY_DIR}/fmtlib-patch.diff CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} PREFIX "${EXTERNAL_INSTALL_LOCATION}" CONFIGURE_COMMAND "" # Disable configure step @@ -27,7 +30,7 @@ set(fmt_INCLUDE_DIR ${source_dir}/include) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) ExternalProject_Add(spdlog GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.3.1 + GIT_TAG v1.15.3 GIT_CONFIG advice.detachedHead=false TIMEOUT 5 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} #-DSPDLOG_BUILD_BENCH=OFF @@ -52,13 +55,10 @@ if(NOT BUILD_WITH_AWS) GIT_TAG v1.7.14 GIT_CONFIG advice.detachedHead=false ) - FetchContent_GetProperties(cJSON) - if(NOT cjson_Populated) - FetchContent_Populate(cJSON) - message(STATUS "cJSON Source Dir: ${cjson_SOURCE_DIR}") - message(STATUS "cJSON Binary Dir: ${cjson_BINARY_DIR}") + FetchContent_MakeAvailable(cJSON) + message(STATUS "cJSON Source Dir: ${cjson_SOURCE_DIR}") + message(STATUS "cJSON Binary Dir: ${cjson_BINARY_DIR}") # add_subdirectory(${cjson_SOURCE_DIR} ${cjson_BINARY_DIR}) - endif() else() message(STATUS "Using cJSON shipped with AWSSDK") # Note: alternative is to use CJSON_HIDE_SYMBOLS IWTH AWS SDK. @@ -106,4 +106,4 @@ target_include_directories(libutils PUBLIC ${AWSSDK_INCLUDE_DIR}) # Specify here the libraries this program depends on -target_link_libraries(libutils Boost::filesystem Boost::thread Boost::system Boost::system Boost::iostreams ${AWSSDK_LINK_LIBRARIES}) +target_link_libraries(libutils Boost::filesystem Boost::thread Boost::iostreams ${AWSSDK_LINK_LIBRARIES}) diff --git a/tuplex/utils/include/Base.h b/tuplex/utils/include/Base.h index da3db47b7..1fc53737e 100644 --- a/tuplex/utils/include/Base.h +++ b/tuplex/utils/include/Base.h @@ -58,7 +58,7 @@ #endif #endif #if __GNUC__ -#if __x86_64__ || __ppc64__ || __arm64__ +#if __x86_64__ || __ppc64__ || __aarch64__ || __arm64__ #define ENV64BIT #else #define ENV32BIT diff --git a/tuplex/utils/include/Utils.h b/tuplex/utils/include/Utils.h index 87f05ca57..8e713e83d 100644 --- a/tuplex/utils/include/Utils.h +++ b/tuplex/utils/include/Utils.h @@ -43,7 +43,7 @@ namespace std { #if (defined __x86_64__) #include "third_party/levenshtein-sse.h" -#elif (defined __arm64__) +#elif (defined(__aarch64__) || defined(__arm64__)) namespace tuplex { // native C++ implementation for now (ARM) inline size_t levenshtein(const std::string& word1, const std::string& word2) { diff --git a/tuplex/utils/patches/fmtlib.diff b/tuplex/utils/patches/fmtlib.diff new file mode 100644 index 000000000..0f80612c2 --- /dev/null +++ b/tuplex/utils/patches/fmtlib.diff @@ -0,0 +1,17 @@ +diff --git a/include/fmt/format-inl.h b/include/fmt/format-inl.h +index a1e0166..8ab753a 100644 +--- a/include/fmt/format-inl.h ++++ b/include/fmt/format-inl.h +@@ -1476,6 +1476,12 @@ template auto getc_unlocked(F* f) -> decltype(_fgetc_nolock(f)) { + } + #endif + ++#ifndef EOF ++// Define via C++. ++#include ++#define EOF (std::char_traits::eof()) ++#endif ++ + template + struct has_flockfile : std::false_type {}; + diff --git a/tuplex/utils/src/Logger.cc b/tuplex/utils/src/Logger.cc index 4ce9bc62b..1959ed10b 100644 --- a/tuplex/utils/src/Logger.cc +++ b/tuplex/utils/src/Logger.cc @@ -72,7 +72,7 @@ MessageHandler& Logger::logger(const std::string &name) { #ifndef NDEBUG spdlogger->set_level(spdlog::level::debug); #endif - spdlog::register_logger(spdlogger); + spdlog::register_or_replace(spdlogger); return _handlers[name]; }