From e793b11929a27cfc434cf839419465c13d5ffffe Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 13:16:06 -0700 Subject: [PATCH 01/92] musllinux --- scripts/docker/musllinux/Dockerfile | 14 + .../docker/musllinux/install_requirements.sh | 265 ++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 scripts/docker/musllinux/Dockerfile create mode 100644 scripts/docker/musllinux/install_requirements.sh diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile new file mode 100644 index 000000000..66d1e70cc --- /dev/null +++ b/scripts/docker/musllinux/Dockerfile @@ -0,0 +1,14 @@ +# Build musllinux (Alpine) compatible version. +FROM quay.io/pypa/musllinux_1_2_x86_64:latest + +# docker build -t tuplex/musl . +# docker run -it tuplex/musl bash + +ARG BOOST_VERSION=1.79.0 + +RUN apk-update + +RUN mkdir -p /opt/sbin + +ADD install_requirements.sh /opt/sbin/install_requirements.sh + diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh new file mode 100644 index 000000000..3937a5056 --- /dev/null +++ b/scripts/docker/musllinux/install_requirements.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# (c) Tuplex team 2017-2023 +# Installs all tuplex dependencies required to build tuplex. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +CMAKE_VERSION="3.27.5" +BOOST_VERSION="1.88.0" +LLVM_VERSION="16.0.6" +AWSSDK_CPP_VERSION=1.11.524 +ANTLR4_VERSION=4.13.1 +YAML_CPP_VERSION=0.8.0 +AWS_LAMBDA_CPP_VERSION=0.2.10 +PCRE2_VERSION=10.45 +PROTOBUF_VERSION=24.3 +CELERO_VERSION=2.8.3 +CC=gcc +CXX=g++ + +CPU_COUNT=$(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) + +PYTHON_VERSION=$(echo $(python3 --version) | cut -d ' ' -f2) +PYTHON_MAJMIN_VERSION=${PYTHON_VERSION%.*} +echo ">> Installing dependencies for Python version ${PYTHON_VERSION}" + +function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [[ $(id -u) -ne 0 ]]; then + echo "Please run this script with root privileges" + exit 1 +fi + +# --- helper functions --- +# Parse the major, minor and patch versions. +# You use it like this: +# semver="3.4.5+xyz" +# a=($(parse_semver "$semver")) +# major=${a[0]} +# minor=${a[1]} +# patch=${a[2]} +# printf "%-32s %4d %4d %4d\n" "$semver" $major $minor $patch +function parse_semver() { + local token="$1" + local major=0 + local minor=0 + local patch=0 + + if egrep '^[0-9]+\.[0-9]+\.[0-9]+' <<<"$token" >/dev/null 2>&1 ; then + # It has the correct syntax. + local n=${token//[!0-9]/ } + local a=(${n//\./ }) + major=${a[0]} + minor=${a[1]} + patch=${a[2]} + fi + + echo "$major $minor $patch" +} + +function install_llvm { + LLVM_VERSION=$1 + LLVM_MAJOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f1` + LLVM_MINOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f2` + LLVM_MAJMIN_VERSION="${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}" + + # list of targets available to build: AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SystemZ;VE;WebAssembly;X86;XCore + # in order to cross-compile, should use targets: + + + echo ">> building LLVM ${LLVM_VERSION}" + LLVM_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz + CLANG_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/clang-${LLVM_VERSION}.src.tar.xz + # required when LLVM version > 15 + LLVM_CMAKE_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cmake-${LLVM_VERSION}.src.tar.xz + + PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} + PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" + PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) + echo ">> Building dependencies for ${PYTHON_VERSION}" + + echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}}" + LLVM_WORKDIR=${WORKDIR}/llvm${LLVM_VERSION} + mkdir -p ${LLVM_WORKDIR} + pushd "${LLVM_WORKDIR}" || exit 1 + + wget ${LLVM_URL} && tar xf llvm-${LLVM_VERSION}.src.tar.xz + wget ${CLANG_URL} && tar xf clang-${LLVM_VERSION}.src.tar.xz && mv clang-${LLVM_VERSION}.src llvm-${LLVM_VERSION}.src/../clang + + if (( LLVM_MAJOR_VERSION >= 15 )); then + wget ${LLVM_CMAKE_URL} && tar xf cmake-${LLVM_VERSION}.src.tar.xz && mv cmake-${LLVM_VERSION}.src cmake + fi + + mkdir -p llvm-${LLVM_VERSION}.src/build && cd llvm-${LLVM_VERSION}.src/build + + cmake -GNinja -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm-${LLVM_VERSION} .. + ninja install + popd +} + +export DEBIAN_FRONTEND=noninteractive + + + +echo ">> Installing packages into ${PREFIX}" +mkdir -p $PREFIX && chmod 0755 $PREFIX +mkdir -p $PREFIX/sbin +mkdir -p $PREFIX/bin +mkdir -p $PREFIX/share +mkdir -p $PREFIX/include +mkdir -p $PREFIX/lib + +echo ">> Files will be downloaded to ${WORKDIR}/tuplex-downloads" +WORKDIR=$WORKDIR/tuplex-downloads +mkdir -p $WORKDIR + +PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) +echo ">> Building dependencies for ${PYTHON_VERSION}" +echo ">> Installing all build dependencies for Tuplex under Ubuntu 24.04" + +echo ">> Installing apt dependencies" +apt update -y + +apt-get install -y apt-utils dh-autoreconf libmagic-dev curl libxml2-dev vim build-essential libssl-dev zlib1g-dev libncurses5-dev \ +libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev openssh-client unzip \ +libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev wget git libcurl4-openssl-dev python3-dev python3-pip openjdk-11-jdk ninja-build + +ldconfig +export CC=${CC} +export CXX=${CXX} + +echo ">> Installing recent cmake" +# fetch recent cmake & install +URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz +mkdir -p ${WORKDIR}/cmake && cd ${WORKDIR}/cmake && + curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && + tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && + rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && + cd cmake-${CMAKE_VERSION}-linux-x86_64 && + cp -rp bin/* ${PREFIX}/bin/ && + cp -rp share/* ${PREFIX}/share/ && + cd / && rm -rf ${WORKDIR}/cmake + +export PATH=$PREFIX/bin:$PATH +cmake --version + + +echo ">> Installing Boost" +mkdir -p ${WORKDIR}/boost + +# -- Install Boost. +# create underscored version +# e.g. 1.79.0 -> 1_79_0 +BOOST_UNDERSCORED_VERSION=$(echo ${BOOST_VERSION} | tr . _) + +# build incl. boost python +cd ${WORKDIR}/boost && curl -L -O https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz && tar xf boost-${BOOST_VERSION}-b2-nodocs.tar.gz && cd ${WORKDIR}/boost/boost-${BOOST_VERSION} \ + && ./bootstrap.sh --with-python=${PYTHON_EXECUTABLE} --prefix=${PREFIX} --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" \ + && ./b2 cxxflags="-fPIC" link=static -j "$(nproc)" \ + && ./b2 cxxflags="-fPIC" link=static install && sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' ${PREFIX}/include/boost/thread/pthread/thread_data.hpp + +cd $WORKDIR +rm -rf ${WORKDIR}/boost + +# -- install llvm +install_llvm $LLVM_VERSION + + +echo ">>> Installing tuplex dependencies." +# install recent zlib version (1.2.11) fork from cloudflare +# https://github.com/aws/aws-graviton-getting-started#zlib-on-linux +LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64:$LD_LIBRARY_PATH + +# Cloudflare fork is too old +#mkdir -p $WORKDIR/zlib && cd $WORKDIR && git clone https://github.com/cloudflare/zlib.git && cd zlib && ./configure --prefix=$PREFIX && make -j ${CPU_COUNT} && make install + +# note that zlib defines Z_NULL=0 whereas zlib-ng defines it as NULL, patch aws sdk accordingly +git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/2.1.3 && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON .. && make -j ${CPU_COUNT} && make install + +git clone https://github.com/google/googletest.git -b v1.14.0 && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release .. && make -j ${CPU_COUNT} && make install + +# build snappy as static lib +git clone https://github.com/google/snappy.git -b 1.1.10 && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install + +# add github to known hosts +mkdir -p /root/.ssh/ && + touch /root/.ssh/known_hosts && + ssh-keyscan github.com >> /root/.ssh/known_hosts + + +echo ">> Installing YAMLCPP" +mkdir -p ${WORKDIR}/yamlcpp && cd ${WORKDIR}/yamlcpp \ +&& git clone https://github.com/jbeder/yaml-cpp.git yaml-cpp \ +&& cd yaml-cpp \ +&& git checkout tags/${YAML_CPP_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Installing Celero" +mkdir -p ${WORKDIR}/celero && cd ${WORKDIR}/celero \ +&& git clone https://github.com/DigitalInBlue/Celero.git celero && cd celero \ +&& git checkout tags/v${CELERO_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC -std=c++11" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Installing ANTLR" +mkdir -p ${WORKDIR}/antlr && cd ${WORKDIR}/antlr \ +&& curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \ +&& cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ \ +&& curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime \ +&& rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& cd antlr4-cpp-runtime \ +&& mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT}&& make install + +echo ">> Installing AWS SDK" +# Note the z-lib patch here. +mkdir -p ${WORKDIR}/aws && cd ${WORKDIR}/aws \ +&& git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git \ +&& cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT} \ +&& make install + +# Installing AWS Lambda C++ runtime. +cd ${WORKDIR}/aws \ +&& git clone https://github.com/awslabs/aws-lambda-cpp.git \ +&& cd aws-lambda-cpp \ +&& git fetch && git fetch --tags \ +&& git checkout v${AWS_LAMBDA_CPP_VERSION} \ +&& mkdir build \ +&& cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j$(nproc) && make install + +echo ">> Installing PCRE2" +mkdir -p ${WORKDIR}/pcre2 && cd ${WORKDIR}/pcre2 \ +&& curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PCRE2_VERSION}/pcre2-${PCRE2_VERSION}.zip \ +&& unzip pcre2-${PCRE2_VERSION}.zip \ +&& rm pcre2-${PCRE2_VERSION}.zip \ +&& cd pcre2-${PCRE2_VERSION} \ +&& ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \ +&& make -j$(nproc) && make install + +echo ">> Installing protobuf" +mkdir -p ${WORKDIR}/protobuf && cd ${WORKDIR}/protobuf \ +&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j ${CPU_COUNT} && make install + + +# delete workdir (downloads dir) to clean up space +rm -rf ${WORKDIR} + +echo "-- Done, all Tuplex requirements installed to /opt --" From bbd72e40967c3ae6c2d9e542ebe21c25feddc982 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 13:20:45 -0700 Subject: [PATCH 02/92] check if it works --- .../docker/musllinux/install_requirements.sh | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh index 3937a5056..67c88a881 100644 --- a/scripts/docker/musllinux/install_requirements.sh +++ b/scripts/docker/musllinux/install_requirements.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env bash +#!/bin/sh # (c) Tuplex team 2017-2023 -# Installs all tuplex dependencies required to build tuplex. +# Installs all tuplex dependencies required to build tuplex on Alpine Linux. # Variables needed incl. defaults. PREFIX=${PREFIX:-/opt} @@ -19,7 +19,7 @@ CELERO_VERSION=2.8.3 CC=gcc CXX=g++ -CPU_COUNT=$(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) PYTHON_VERSION=$(echo $(python3 --version) | cut -d ' ' -f2) PYTHON_MAJMIN_VERSION=${PYTHON_VERSION%.*} @@ -31,7 +31,7 @@ function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4 set -euxo pipefail # need to run this with root privileges -if [[ $(id -u) -ne 0 ]]; then +if [ "$(id -u)" -ne 0 ]; then echo "Please run this script with root privileges" exit 1 fi @@ -51,7 +51,7 @@ function parse_semver() { local minor=0 local patch=0 - if egrep '^[0-9]+\.[0-9]+\.[0-9]+' <<<"$token" >/dev/null 2>&1 ; then + if echo "$token" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' >/dev/null 2>&1 ; then # It has the correct syntax. local n=${token//[!0-9]/ } local a=(${n//\./ }) @@ -87,12 +87,12 @@ function install_llvm { echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}}" LLVM_WORKDIR=${WORKDIR}/llvm${LLVM_VERSION} mkdir -p ${LLVM_WORKDIR} - pushd "${LLVM_WORKDIR}" || exit 1 + cd "${LLVM_WORKDIR}" || exit 1 wget ${LLVM_URL} && tar xf llvm-${LLVM_VERSION}.src.tar.xz wget ${CLANG_URL} && tar xf clang-${LLVM_VERSION}.src.tar.xz && mv clang-${LLVM_VERSION}.src llvm-${LLVM_VERSION}.src/../clang - if (( LLVM_MAJOR_VERSION >= 15 )); then + if [ $LLVM_MAJOR_VERSION -ge 15 ]; then wget ${LLVM_CMAKE_URL} && tar xf cmake-${LLVM_VERSION}.src.tar.xz && mv cmake-${LLVM_VERSION}.src cmake fi @@ -102,13 +102,9 @@ function install_llvm { -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ -DCMAKE_INSTALL_PREFIX=/opt/llvm-${LLVM_VERSION} .. ninja install - popd + cd /tmp } -export DEBIAN_FRONTEND=noninteractive - - - echo ">> Installing packages into ${PREFIX}" mkdir -p $PREFIX && chmod 0755 $PREFIX mkdir -p $PREFIX/sbin @@ -124,16 +120,22 @@ mkdir -p $WORKDIR PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) echo ">> Building dependencies for ${PYTHON_VERSION}" -echo ">> Installing all build dependencies for Tuplex under Ubuntu 24.04" - -echo ">> Installing apt dependencies" -apt update -y - -apt-get install -y apt-utils dh-autoreconf libmagic-dev curl libxml2-dev vim build-essential libssl-dev zlib1g-dev libncurses5-dev \ -libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev openssh-client unzip \ -libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev wget git libcurl4-openssl-dev python3-dev python3-pip openjdk-11-jdk ninja-build +echo ">> Installing all build dependencies for Tuplex under Alpine Linux" + +echo ">> Installing apk dependencies" +apk update + +apk add --no-cache \ + autoconf automake libtool \ + curl libxml2-dev vim build-base \ + openssl-dev zlib-dev ncurses-dev \ + readline-dev sqlite-dev \ + bzip2-dev expat-dev xz-dev \ + tk-dev libffi-dev wget git \ + curl-dev python3-dev py3-pip \ + openjdk11 ninja \ + linux-headers musl-dev -ldconfig export CC=${CC} export CXX=${CXX} From ff6286e38edab30ee88fbc5bdfe9da214b457f51 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 13:53:21 -0700 Subject: [PATCH 03/92] fixes --- scripts/docker/musllinux/Dockerfile | 79 +++++++++++++++++- .../docker/musllinux/install_requirements.sh | 81 ++++++++++--------- 2 files changed, 119 insertions(+), 41 deletions(-) diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index 66d1e70cc..975271d46 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -1,14 +1,87 @@ # Build musllinux (Alpine) compatible version. -FROM quay.io/pypa/musllinux_1_2_x86_64:latest +# You can create a multiplatform docker image via +# docker buildx build --platform linux/amd64,linux/arm64 -t tuplex/musl . + +# BUILD INSTRUCTIONS: +# =================== +# +# Single platform builds: +# ----------------------- +# For AMD64 (x86_64) systems: +# docker build --platform linux/amd64 -t tuplex/musl:amd64 . +# +# For ARM64 (aarch64) systems (like Apple Silicon): +# docker build --platform linux/arm64 -t tuplex/musl:arm64 . +# +# Multi-platform build (requires docker buildx): +# --------------------------------------------- +# docker buildx build --platform linux/amd64,linux/arm64 -t tuplex/musl:latest . +# +# To enable buildx (if not already enabled): +# docker buildx create --use +# +# To see available platforms: +# docker buildx inspect --bootstrap +# +# RUN INSTRUCTIONS: +# ================ +# docker run -it tuplex/musl:latest bash +# docker run -it tuplex/musl:amd64 bash # AMD64 specific +# docker run -it tuplex/musl:arm64 bash # ARM64 specific + +# Set platform-specific base image +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +# Use appropriate base image based on target platform +FROM quay.io/pypa/musllinux_1_2_x86_64:latest AS base-amd64 +FROM quay.io/pypa/musllinux_1_2_aarch64:latest AS base-arm64 + +# Select the appropriate base image +FROM base-${TARGETPLATFORM#linux/} # docker build -t tuplex/musl . # docker run -it tuplex/musl bash ARG BOOST_VERSION=1.79.0 -RUN apk-update +# Update package index and install basic dependencies +RUN apk update && \ + apk add --no-cache \ + python3 \ + py3-pip \ + bash \ + wget \ + curl \ + git \ + make \ + gcc \ + g++ \ + cmake \ + ninja \ + autoconf \ + automake \ + libtool \ + linux-headers \ + musl-dev -RUN mkdir -p /opt/sbin +# Create necessary directories +RUN mkdir -p /opt/sbin /opt/bin /opt/lib /opt/include +# Copy and make the installation script executable ADD install_requirements.sh /opt/sbin/install_requirements.sh +RUN chmod +x /opt/sbin/install_requirements.sh + +# Set working directory +WORKDIR /opt + +# Run the installation script +RUN /opt/sbin/install_requirements.sh + +# Set environment variables +ENV PATH="/opt/bin:$PATH" +ENV LD_LIBRARY_PATH="/opt/lib:$LD_LIBRARY_PATH" + +# Default command +CMD ["/bin/bash"] diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh index 67c88a881..cafa81303 100644 --- a/scripts/docker/musllinux/install_requirements.sh +++ b/scripts/docker/musllinux/install_requirements.sh @@ -25,7 +25,7 @@ PYTHON_VERSION=$(echo $(python3 --version) | cut -d ' ' -f2) PYTHON_MAJMIN_VERSION=${PYTHON_VERSION%.*} echo ">> Installing dependencies for Python version ${PYTHON_VERSION}" -function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } +version() { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } # Start script. set -euxo pipefail @@ -45,25 +45,24 @@ fi # minor=${a[1]} # patch=${a[2]} # printf "%-32s %4d %4d %4d\n" "$semver" $major $minor $patch -function parse_semver() { - local token="$1" - local major=0 - local minor=0 - local patch=0 +parse_semver() { + token="$1" + major=0 + minor=0 + patch=0 if echo "$token" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' >/dev/null 2>&1 ; then # It has the correct syntax. - local n=${token//[!0-9]/ } - local a=(${n//\./ }) - major=${a[0]} - minor=${a[1]} - patch=${a[2]} + n=$(echo "$token" | tr -cd '0-9.' | tr '.' ' ') + major=$(echo "$n" | cut -d' ' -f1) + minor=$(echo "$n" | cut -d' ' -f2) + patch=$(echo "$n" | cut -d' ' -f3) fi echo "$major $minor $patch" } -function install_llvm { +install_llvm() { LLVM_VERSION=$1 LLVM_MAJOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f1` LLVM_MINOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f2` @@ -85,7 +84,7 @@ function install_llvm { echo ">> Building dependencies for ${PYTHON_VERSION}" echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}}" - LLVM_WORKDIR=${WORKDIR}/llvm${LLVM_VERSION} + LLVM_WORKDIR=${DOWNLOAD_DIR}/llvm${LLVM_VERSION} mkdir -p ${LLVM_WORKDIR} cd "${LLVM_WORKDIR}" || exit 1 @@ -114,8 +113,8 @@ mkdir -p $PREFIX/include mkdir -p $PREFIX/lib echo ">> Files will be downloaded to ${WORKDIR}/tuplex-downloads" -WORKDIR=$WORKDIR/tuplex-downloads -mkdir -p $WORKDIR +DOWNLOAD_DIR=$WORKDIR/tuplex-downloads +mkdir -p $DOWNLOAD_DIR PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) @@ -140,23 +139,29 @@ export CC=${CC} export CXX=${CXX} echo ">> Installing recent cmake" -# fetch recent cmake & install -URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz -mkdir -p ${WORKDIR}/cmake && cd ${WORKDIR}/cmake && - curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && - tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && - rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && - cd cmake-${CMAKE_VERSION}-linux-x86_64 && +# fetch recent cmake & install - detect architecture +ARCH=$(uname -m) +if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + CMAKE_ARCH="aarch64" +else + CMAKE_ARCH="x86_64" +fi +URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz +mkdir -p ${DOWNLOAD_DIR}/cmake && cd ${DOWNLOAD_DIR}/cmake && + curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + tar -v -zxf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + rm -f cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + cd cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} && cp -rp bin/* ${PREFIX}/bin/ && cp -rp share/* ${PREFIX}/share/ && - cd / && rm -rf ${WORKDIR}/cmake + cd / && rm -rf ${DOWNLOAD_DIR}/cmake export PATH=$PREFIX/bin:$PATH cmake --version echo ">> Installing Boost" -mkdir -p ${WORKDIR}/boost +mkdir -p ${DOWNLOAD_DIR}/boost # -- Install Boost. # create underscored version @@ -164,13 +169,13 @@ mkdir -p ${WORKDIR}/boost BOOST_UNDERSCORED_VERSION=$(echo ${BOOST_VERSION} | tr . _) # build incl. boost python -cd ${WORKDIR}/boost && curl -L -O https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz && tar xf boost-${BOOST_VERSION}-b2-nodocs.tar.gz && cd ${WORKDIR}/boost/boost-${BOOST_VERSION} \ +cd ${DOWNLOAD_DIR}/boost && curl -L -O https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz && tar xf boost-${BOOST_VERSION}-b2-nodocs.tar.gz && cd ${DOWNLOAD_DIR}/boost/boost-${BOOST_VERSION} \ && ./bootstrap.sh --with-python=${PYTHON_EXECUTABLE} --prefix=${PREFIX} --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" \ - && ./b2 cxxflags="-fPIC" link=static -j "$(nproc)" \ + && ./b2 cxxflags="-fPIC" link=static -j "${CPU_COUNT}" \ && ./b2 cxxflags="-fPIC" link=static install && sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' ${PREFIX}/include/boost/thread/pthread/thread_data.hpp -cd $WORKDIR -rm -rf ${WORKDIR}/boost +cd $DOWNLOAD_DIR +rm -rf ${DOWNLOAD_DIR}/boost # -- install llvm install_llvm $LLVM_VERSION @@ -200,7 +205,7 @@ mkdir -p /root/.ssh/ && echo ">> Installing YAMLCPP" -mkdir -p ${WORKDIR}/yamlcpp && cd ${WORKDIR}/yamlcpp \ +mkdir -p ${DOWNLOAD_DIR}/yamlcpp && cd ${DOWNLOAD_DIR}/yamlcpp \ && git clone https://github.com/jbeder/yaml-cpp.git yaml-cpp \ && cd yaml-cpp \ && git checkout tags/${YAML_CPP_VERSION} \ @@ -209,7 +214,7 @@ mkdir -p ${WORKDIR}/yamlcpp && cd ${WORKDIR}/yamlcpp \ && make -j ${CPU_COUNT} && make install echo ">> Installing Celero" -mkdir -p ${WORKDIR}/celero && cd ${WORKDIR}/celero \ +mkdir -p ${DOWNLOAD_DIR}/celero && cd ${DOWNLOAD_DIR}/celero \ && git clone https://github.com/DigitalInBlue/Celero.git celero && cd celero \ && git checkout tags/v${CELERO_VERSION} \ && mkdir build && cd build \ @@ -217,7 +222,7 @@ mkdir -p ${WORKDIR}/celero && cd ${WORKDIR}/celero \ && make -j ${CPU_COUNT} && make install echo ">> Installing ANTLR" -mkdir -p ${WORKDIR}/antlr && cd ${WORKDIR}/antlr \ +mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr \ && curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \ && cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ \ && curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ @@ -229,7 +234,7 @@ mkdir -p ${WORKDIR}/antlr && cd ${WORKDIR}/antlr \ echo ">> Installing AWS SDK" # Note the z-lib patch here. -mkdir -p ${WORKDIR}/aws && cd ${WORKDIR}/aws \ +mkdir -p ${DOWNLOAD_DIR}/aws && cd ${DOWNLOAD_DIR}/aws \ && git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git \ && cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build \ && cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ @@ -237,7 +242,7 @@ mkdir -p ${WORKDIR}/aws && cd ${WORKDIR}/aws \ && make install # Installing AWS Lambda C++ runtime. -cd ${WORKDIR}/aws \ +cd ${DOWNLOAD_DIR}/aws \ && git clone https://github.com/awslabs/aws-lambda-cpp.git \ && cd aws-lambda-cpp \ && git fetch && git fetch --tags \ @@ -245,23 +250,23 @@ cd ${WORKDIR}/aws \ && mkdir build \ && cd build \ && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ -&& make -j$(nproc) && make install +&& make -j${CPU_COUNT} && make install echo ">> Installing PCRE2" -mkdir -p ${WORKDIR}/pcre2 && cd ${WORKDIR}/pcre2 \ +mkdir -p ${DOWNLOAD_DIR}/pcre2 && cd ${DOWNLOAD_DIR}/pcre2 \ && curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PCRE2_VERSION}/pcre2-${PCRE2_VERSION}.zip \ && unzip pcre2-${PCRE2_VERSION}.zip \ && rm pcre2-${PCRE2_VERSION}.zip \ && cd pcre2-${PCRE2_VERSION} \ && ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \ -&& make -j$(nproc) && make install +&& make -j${CPU_COUNT} && make install echo ">> Installing protobuf" -mkdir -p ${WORKDIR}/protobuf && cd ${WORKDIR}/protobuf \ +mkdir -p ${DOWNLOAD_DIR}/protobuf && cd ${DOWNLOAD_DIR}/protobuf \ && git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j ${CPU_COUNT} && make install # delete workdir (downloads dir) to clean up space -rm -rf ${WORKDIR} +rm -rf ${DOWNLOAD_DIR} echo "-- Done, all Tuplex requirements installed to /opt --" From ea67d12f80a0d6c92c721ab945faa93c062d5456 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 14:35:55 -0700 Subject: [PATCH 04/92] script fixes --- .../docker/musllinux/install_requirements.sh | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh index cafa81303..b7996564c 100644 --- a/scripts/docker/musllinux/install_requirements.sh +++ b/scripts/docker/musllinux/install_requirements.sh @@ -16,6 +16,7 @@ AWS_LAMBDA_CPP_VERSION=0.2.10 PCRE2_VERSION=10.45 PROTOBUF_VERSION=24.3 CELERO_VERSION=2.8.3 +SNAPPY_VERSION=1.2.2 CC=gcc CXX=g++ @@ -177,8 +178,8 @@ cd ${DOWNLOAD_DIR}/boost && curl -L -O https://github.com/boostorg/boost/release cd $DOWNLOAD_DIR rm -rf ${DOWNLOAD_DIR}/boost -# -- install llvm -install_llvm $LLVM_VERSION +## -- install llvm +#install_llvm $LLVM_VERSION echo ">>> Installing tuplex dependencies." @@ -196,12 +197,12 @@ git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout t git clone https://github.com/google/googletest.git -b v1.14.0 && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release .. && make -j ${CPU_COUNT} && make install # build snappy as static lib -git clone https://github.com/google/snappy.git -b 1.1.10 && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install +git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install -# add github to known hosts -mkdir -p /root/.ssh/ && - touch /root/.ssh/known_hosts && - ssh-keyscan github.com >> /root/.ssh/known_hosts +# SSH setup not needed for Docker build - ssh-keyscan not available +# mkdir -p /root/.ssh/ && +# touch /root/.ssh/known_hosts && +# ssh-keyscan github.com >> /root/.ssh/known_hosts echo ">> Installing YAMLCPP" @@ -270,3 +271,12 @@ mkdir -p ${DOWNLOAD_DIR}/protobuf && cd ${DOWNLOAD_DIR}/protobuf \ rm -rf ${DOWNLOAD_DIR} echo "-- Done, all Tuplex requirements installed to /opt --" + +# Verify that key tools are available +echo ">> Verifying installation..." +ls -la /opt/bin/ +echo ">> PATH: $PATH" +echo ">> CMake version:" +/opt/bin/cmake --version || echo "CMake not found in /opt/bin" +echo ">> Python version:" +python3 --version || echo "Python3 not found" From 4d441d63864977eab5acc21280083a0efb535e45 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 14:50:53 -0700 Subject: [PATCH 05/92] image --- .../docker/musllinux/install_requirements.sh | 55 ++++++++++++------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh index b7996564c..d30254dc6 100644 --- a/scripts/docker/musllinux/install_requirements.sh +++ b/scripts/docker/musllinux/install_requirements.sh @@ -134,7 +134,8 @@ apk add --no-cache \ tk-dev libffi-dev wget git \ curl-dev python3-dev py3-pip \ openjdk11 ninja \ - linux-headers musl-dev + linux-headers musl-dev \ + openssh export CC=${CC} export CXX=${CXX} @@ -162,24 +163,38 @@ cmake --version echo ">> Installing Boost" -mkdir -p ${DOWNLOAD_DIR}/boost - -# -- Install Boost. -# create underscored version -# e.g. 1.79.0 -> 1_79_0 -BOOST_UNDERSCORED_VERSION=$(echo ${BOOST_VERSION} | tr . _) - -# build incl. boost python -cd ${DOWNLOAD_DIR}/boost && curl -L -O https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz && tar xf boost-${BOOST_VERSION}-b2-nodocs.tar.gz && cd ${DOWNLOAD_DIR}/boost/boost-${BOOST_VERSION} \ - && ./bootstrap.sh --with-python=${PYTHON_EXECUTABLE} --prefix=${PREFIX} --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" \ - && ./b2 cxxflags="-fPIC" link=static -j "${CPU_COUNT}" \ - && ./b2 cxxflags="-fPIC" link=static install && sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' ${PREFIX}/include/boost/thread/pthread/thread_data.hpp +install_boost() { + local boost_version="${1:-$BOOST_VERSION}" + local download_dir="${2:-$DOWNLOAD_DIR}" + local prefix="${3:-$PREFIX}" + local python_executable="${4:-$PYTHON_EXECUTABLE}" + local cpu_count="${5:-$CPU_COUNT}" + + echo ">> Installing Boost ${boost_version}" + mkdir -p "${download_dir}/boost" + + # create underscored version, e.g. 1.79.0 -> 1_79_0 + local boost_underscored_version + boost_underscored_version=$(echo "${boost_version}" | tr . _) + + # Download and build Boost including boost python + cd "${download_dir}/boost" + curl -L -O "https://github.com/boostorg/boost/releases/download/boost-${boost_version}/boost-${boost_version}-b2-nodocs.tar.gz" + tar xf "boost-${boost_version}-b2-nodocs.tar.gz" + cd "${download_dir}/boost/boost-${boost_version}" + ./bootstrap.sh --with-python="${python_executable}" --prefix="${prefix}" --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" + ./b2 cxxflags="-fPIC" link=static -j "${cpu_count}" + ./b2 cxxflags="-fPIC" link=static install + sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' "${prefix}/include/boost/thread/pthread/thread_data.hpp" + + cd "${download_dir}" + rm -rf "${download_dir}/boost" +} -cd $DOWNLOAD_DIR -rm -rf ${DOWNLOAD_DIR}/boost +install_boost "$BOOST_VERSION" "$DOWNLOAD_DIR" "$PREFIX" "$PYTHON_EXECUTABLE" "$CPU_COUNT" ## -- install llvm -#install_llvm $LLVM_VERSION +install_llvm $LLVM_VERSION echo ">>> Installing tuplex dependencies." @@ -200,9 +215,9 @@ git clone https://github.com/google/googletest.git -b v1.14.0 && cd googletest & git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install # SSH setup not needed for Docker build - ssh-keyscan not available -# mkdir -p /root/.ssh/ && -# touch /root/.ssh/known_hosts && -# ssh-keyscan github.com >> /root/.ssh/known_hosts +mkdir -p /root/.ssh/ && + touch /root/.ssh/known_hosts && + ssh-keyscan github.com >> /root/.ssh/known_hosts echo ">> Installing YAMLCPP" @@ -236,7 +251,7 @@ mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr \ echo ">> Installing AWS SDK" # Note the z-lib patch here. mkdir -p ${DOWNLOAD_DIR}/aws && cd ${DOWNLOAD_DIR}/aws \ -&& git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git \ +&& git clone --progress --verbose --recurse-submodules https://github.com/aws/aws-sdk-cpp.git \ && cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build \ && cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ && make -j ${CPU_COUNT} \ From 2e589bde58bafcbe23d1de81bef9e359547dca6e Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 15:22:06 -0700 Subject: [PATCH 06/92] fixes --- scripts/docker/musllinux/Dockerfile | 49 +++++++++++++---------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index 975271d46..06f54e341 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -22,12 +22,6 @@ # # To see available platforms: # docker buildx inspect --bootstrap -# -# RUN INSTRUCTIONS: -# ================ -# docker run -it tuplex/musl:latest bash -# docker run -it tuplex/musl:amd64 bash # AMD64 specific -# docker run -it tuplex/musl:arm64 bash # ARM64 specific # Set platform-specific base image ARG TARGETPLATFORM @@ -45,25 +39,22 @@ FROM base-${TARGETPLATFORM#linux/} ARG BOOST_VERSION=1.79.0 -# Update package index and install basic dependencies -RUN apk update && \ - apk add --no-cache \ - python3 \ - py3-pip \ - bash \ - wget \ - curl \ - git \ - make \ - gcc \ - g++ \ - cmake \ - ninja \ - autoconf \ - automake \ - libtool \ - linux-headers \ - musl-dev +# Update package index +RUN apk update + +# Install basic dependencies needed for the script +RUN apk add --no-cache \ + bash \ + wget \ + curl \ + git \ + make \ + gcc \ + g++ \ + cmake \ + ninja \ + python3 \ + py3-pip # Create necessary directories RUN mkdir -p /opt/sbin /opt/bin /opt/lib /opt/include @@ -78,9 +69,13 @@ WORKDIR /opt # Run the installation script RUN /opt/sbin/install_requirements.sh +# Fix CMake issue by using the working pipx-installed version +RUN ln -sf /opt/_internal/pipx/venvs/cmake/bin/cmake /opt/bin/cmake + # Set environment variables -ENV PATH="/opt/bin:$PATH" -ENV LD_LIBRARY_PATH="/opt/lib:$LD_LIBRARY_PATH" +ENV PATH="/opt/bin:/opt/sbin:$PATH" +ENV LD_LIBRARY_PATH="/opt/lib:/opt/lib64:$LD_LIBRARY_PATH" +ENV PKG_CONFIG_PATH="/opt/lib/pkgconfig:$PKG_CONFIG_PATH" # Default command CMD ["/bin/bash"] From 0d30b4704c923e7ff848dd567e32162eda14899c Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 15:23:41 -0700 Subject: [PATCH 07/92] upgrade --- scripts/docker/musllinux/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh index d30254dc6..5361e1a83 100644 --- a/scripts/docker/musllinux/install_requirements.sh +++ b/scripts/docker/musllinux/install_requirements.sh @@ -14,7 +14,7 @@ ANTLR4_VERSION=4.13.1 YAML_CPP_VERSION=0.8.0 AWS_LAMBDA_CPP_VERSION=0.2.10 PCRE2_VERSION=10.45 -PROTOBUF_VERSION=24.3 +PROTOBUF_VERSION=32.0 CELERO_VERSION=2.8.3 SNAPPY_VERSION=1.2.2 CC=gcc From 3128968a95aa4145625d9e0368f129ce1aec42ce Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 15:47:16 -0700 Subject: [PATCH 08/92] patches --- .../protobuf-musl-compatibility.patch | 74 ++++++++++ .../docker/musllinux/protobuf-musl-fix.patch | 19 +++ .../docker/musllinux/test_protobuf_patched.sh | 128 ++++++++++++++++++ 3 files changed, 221 insertions(+) create mode 100644 scripts/docker/musllinux/protobuf-musl-compatibility.patch create mode 100644 scripts/docker/musllinux/protobuf-musl-fix.patch create mode 100644 scripts/docker/musllinux/test_protobuf_patched.sh diff --git a/scripts/docker/musllinux/protobuf-musl-compatibility.patch b/scripts/docker/musllinux/protobuf-musl-compatibility.patch new file mode 100644 index 000000000..657ba8fb3 --- /dev/null +++ b/scripts/docker/musllinux/protobuf-musl-compatibility.patch @@ -0,0 +1,74 @@ +diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h ++++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +@@ -23,6 +23,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + namespace absl { + ABSL_NAMESPACE_BEGIN + namespace base_internal { +diff --git a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc ++++ b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +@@ -20,6 +20,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + #include "absl/base/internal/low_level_alloc.h" + + #include +diff --git a/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc b/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc ++++ b/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +@@ -20,6 +20,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + #include "absl/strings/internal/ostringstream.h" + + #include +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1234567..abcdefg 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -7,6 +7,12 @@ + cmake_minimum_required(VERSION 3.5) + project(protobuf VERSION 24.3.0 LANGUAGES C CXX) + ++# Add musl libc compatibility flags ++if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ++ add_compile_definitions(_LARGEFILE64_SOURCE) ++ add_compile_definitions(_FILE_OFFSET_BITS=64) ++endif() ++ + # Set C++ standard + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/scripts/docker/musllinux/protobuf-musl-fix.patch b/scripts/docker/musllinux/protobuf-musl-fix.patch new file mode 100644 index 000000000..826bf9017 --- /dev/null +++ b/scripts/docker/musllinux/protobuf-musl-fix.patch @@ -0,0 +1,19 @@ +diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h ++++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +@@ -23,6 +23,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + namespace absl { + ABSL_NAMESPACE_BEGIN + namespace base_internal { diff --git a/scripts/docker/musllinux/test_protobuf_patched.sh b/scripts/docker/musllinux/test_protobuf_patched.sh new file mode 100644 index 000000000..2bb6e02e0 --- /dev/null +++ b/scripts/docker/musllinux/test_protobuf_patched.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +echo "=== Testing Protobuf 24.3 with Musl Compatibility Patch ===" + +# Step 1: Clone protobuf +echo "1. Cloning protobuf..." +git clone -b v24.3 https://github.com/protocolbuffers/protobuf.git /tmp/protobuf-test +cd /tmp/protobuf-test + +echo "2. Updating submodules..." +git submodule update --init --recursive + +echo "3. Applying musl compatibility patch..." +# Create the patch file inline +cat > /tmp/musl-fix.patch << 'EOF' +diff --git a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/direct_mmap.h ++++ b/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +@@ -23,6 +23,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + namespace absl { + ABSL_NAMESPACE_BEGIN + namespace base_internal { +diff --git a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +index 1234567..abcdefg 100644 +--- a/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc ++++ b/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +@@ -20,6 +20,14 @@ + #include + #include + ++// Fix for musl libc compatibility - off64_t is not defined ++#if defined(__MUSL__) || !defined(_LARGEFILE64_SOURCE) ++#ifndef off64_t ++#define off64_t off_t ++#endif ++#endif ++ ++ + #include "absl/base/internal/low_level_alloc.h" + + #include +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1234567..abcdefg 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -7,6 +7,12 @@ + cmake_minimum_required(VERSION 3.5) + project(protobuf VERSION 24.3.0 LANGUAGES C CXX) + ++# Add musl libc compatibility flags ++if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ++ add_compile_definitions(_LARGEFILE64_SOURCE) ++ add_compile_definitions(_FILE_OFFSET_BITS=64) ++endif() ++ + # Set C++ standard + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +EOF + +# Apply the patch +patch -p1 < /tmp/musl-fix.patch + +echo "4. Building protobuf with patch..." +mkdir build +cd build +/opt/bin/cmake -DCMAKE_CXX_FLAGS="-fPIC -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. +make -j4 + +echo "5. Installing protobuf..." +make install + +echo "6. Testing protoc..." +protoc --version + +echo "7. Creating test project..." +cd /tmp +mkdir proto-test +cd proto-test + +echo "8. Creating test proto file..." +cat > test.proto << 'EOF' +syntax = "proto3"; +message TestMessage { + string text = 1; + int32 number = 2; +} +EOF + +echo "9. Compiling proto file..." +protoc --cpp_out=. test.proto + +echo "10. Creating test program..." +cat > main.cpp << 'EOF' +#include +#include "test.pb.h" + +int main() { + TestMessage msg; + msg.set_text("Hello Protobuf!"); + msg.set_number(42); + + std::cout << "Text: " << msg.text() << std::endl; + std::cout << "Number: " << msg.number() << std::endl; + + return 0; +} +EOF + +echo "11. Building test program..." +g++ -std=c++17 main.cpp test.pb.cc -lprotobuf -o test_program + +echo "12. Running test..." +./test_program + +echo "=== Protobuf test with musl patch completed successfully! ===" From f53f71a9b1ada45649acef1f68d96209e2abfb82 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 16:11:06 -0700 Subject: [PATCH 09/92] refactor --- scripts/docker/musllinux/.dockerignore | 41 ++++ scripts/docker/musllinux/Dockerfile | 68 ++++++- scripts/docker/musllinux/README.md | 177 ++++++++++++++++++ scripts/docker/musllinux/build.sh | 169 +++++++++++++++++ scripts/docker/musllinux/install_all.sh | 102 ++++++++++ scripts/docker/musllinux/install_alpine.sh | 61 ++++++ scripts/docker/musllinux/install_antlr.sh | 38 ++++ scripts/docker/musllinux/install_aws_sdk.sh | 52 +++++ scripts/docker/musllinux/install_boost.sh | 47 +++++ scripts/docker/musllinux/install_celero.sh | 35 ++++ .../docker/musllinux/install_googletest.sh | 28 +++ scripts/docker/musllinux/install_llvm.sh | 63 +++++++ scripts/docker/musllinux/install_pcre2.sh | 36 ++++ scripts/docker/musllinux/install_protobuf.sh | 31 +++ .../docker/musllinux/install_requirements.sh | 2 +- scripts/docker/musllinux/install_snappy.sh | 29 +++ scripts/docker/musllinux/install_yamlcpp.sh | 36 ++++ scripts/docker/musllinux/install_zlib.sh | 33 ++++ scripts/docker/musllinux/test_docker_build.sh | 101 ++++++++++ 19 files changed, 1141 insertions(+), 8 deletions(-) create mode 100644 scripts/docker/musllinux/.dockerignore create mode 100644 scripts/docker/musllinux/README.md create mode 100755 scripts/docker/musllinux/build.sh create mode 100755 scripts/docker/musllinux/install_all.sh create mode 100755 scripts/docker/musllinux/install_alpine.sh create mode 100755 scripts/docker/musllinux/install_antlr.sh create mode 100755 scripts/docker/musllinux/install_aws_sdk.sh create mode 100755 scripts/docker/musllinux/install_boost.sh create mode 100755 scripts/docker/musllinux/install_celero.sh create mode 100755 scripts/docker/musllinux/install_googletest.sh create mode 100755 scripts/docker/musllinux/install_llvm.sh create mode 100755 scripts/docker/musllinux/install_pcre2.sh create mode 100755 scripts/docker/musllinux/install_protobuf.sh create mode 100755 scripts/docker/musllinux/install_snappy.sh create mode 100755 scripts/docker/musllinux/install_yamlcpp.sh create mode 100755 scripts/docker/musllinux/install_zlib.sh create mode 100755 scripts/docker/musllinux/test_docker_build.sh diff --git a/scripts/docker/musllinux/.dockerignore b/scripts/docker/musllinux/.dockerignore new file mode 100644 index 000000000..83cc17311 --- /dev/null +++ b/scripts/docker/musllinux/.dockerignore @@ -0,0 +1,41 @@ +# Git and version control +.git +.gitignore +.gitattributes + +# Documentation +README.md +*.md + +# Test files +test_*.sh + +# Original monolithic script (no longer needed) +install_requirements.sh + +# Docker files (avoid recursive copying) +Dockerfile* +.dockerignore + +# Build artifacts and temporary files +build/ +*.o +*.so +*.a +*.log + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index 06f54e341..922c5ec79 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -37,12 +37,24 @@ FROM base-${TARGETPLATFORM#linux/} # docker build -t tuplex/musl . # docker run -it tuplex/musl bash -ARG BOOST_VERSION=1.79.0 +# Version arguments for dependencies +ARG BOOST_VERSION=1.88.0 +ARG LLVM_VERSION=16.0.6 +ARG AWSSDK_CPP_VERSION=1.11.524 +ARG ANTLR4_VERSION=4.13.1 +ARG YAML_CPP_VERSION=0.8.0 +ARG AWS_LAMBDA_CPP_VERSION=0.2.10 +ARG PCRE2_VERSION=10.45 +ARG PROTOBUF_VERSION=32.0 +ARG CELERO_VERSION=2.8.3 +ARG SNAPPY_VERSION=1.2.2 +ARG ZLIB_VERSION=2.1.3 +ARG CMAKE_VERSION=3.27.5 # Update package index RUN apk update -# Install basic dependencies needed for the script +# Install basic dependencies needed for the scripts RUN apk add --no-cache \ bash \ wget \ @@ -59,15 +71,53 @@ RUN apk add --no-cache \ # Create necessary directories RUN mkdir -p /opt/sbin /opt/bin /opt/lib /opt/include -# Copy and make the installation script executable -ADD install_requirements.sh /opt/sbin/install_requirements.sh -RUN chmod +x /opt/sbin/install_requirements.sh +# Copy all installation scripts +COPY install_alpine.sh install_zlib.sh install_llvm.sh install_boost.sh \ + install_googletest.sh install_snappy.sh install_yamlcpp.sh install_celero.sh \ + install_antlr.sh install_pcre2.sh install_protobuf.sh install_aws_sdk.sh \ + /tmp/ + +# Make all scripts executable +RUN chmod +x /tmp/install_*.sh # Set working directory WORKDIR /opt -# Run the installation script -RUN /opt/sbin/install_requirements.sh +# Layer 1: Install Alpine packages and CMake (base system dependencies) +RUN /tmp/install_alpine.sh + +# Layer 2: Install zlib-ng (required by AWS SDK) +RUN /tmp/install_zlib.sh + +# Layer 5: Install testing and compression libraries +RUN /tmp/install_googletest.sh + +RUN /tmp/install_snappy.sh + +# Layer 6: Install parsing and data libraries +RUN /tmp/install_yamlcpp.sh + +RUN /tmp/install_celero.sh + +# Layer 7: Install parsing and regex libraries +RUN /tmp/install_antlr.sh + +RUN /tmp/install_pcre2.sh + +# Layer 8: Install Protocol Buffers +RUN /tmp/install_protobuf.sh + +# Layer 9: Install AWS SDK C++ and Lambda runtime +RUN /tmp/install_aws_sdk.sh + +# Layer 4: Install Boost (C++ libraries) +RUN /tmp/install_boost.sh + +# Layer 3: Install LLVM (compiler infrastructure) (this will take a while) +RUN /tmp/install_llvm.sh + +# Clean up installation scripts +RUN rm -f /tmp/install_*.sh # Fix CMake issue by using the working pipx-installed version RUN ln -sf /opt/_internal/pipx/venvs/cmake/bin/cmake /opt/bin/cmake @@ -77,6 +127,10 @@ ENV PATH="/opt/bin:/opt/sbin:$PATH" ENV LD_LIBRARY_PATH="/opt/lib:/opt/lib64:$LD_LIBRARY_PATH" ENV PKG_CONFIG_PATH="/opt/lib/pkgconfig:$PKG_CONFIG_PATH" +RUN mkdir -p /code + +WORKDIR /code + # Default command CMD ["/bin/bash"] diff --git a/scripts/docker/musllinux/README.md b/scripts/docker/musllinux/README.md new file mode 100644 index 000000000..252332deb --- /dev/null +++ b/scripts/docker/musllinux/README.md @@ -0,0 +1,177 @@ +# Tuplex Dependencies Installation Scripts + +This directory contains modular installation scripts for building Tuplex on Alpine Linux. Each script installs a specific dependency, allowing for flexible and optimized Docker builds. + +## Scripts Overview + +### Base System Dependencies +- **`install_alpine.sh`** - Installs Alpine Linux packages and CMake +- **`install_zlib.sh`** - Installs zlib-ng compression library + +### Core Compiler Infrastructure +- **`install_llvm.sh`** - Installs LLVM and Clang compiler +- **`install_boost.sh`** - Installs Boost C++ libraries + +### Testing and Benchmarking +- **`install_googletest.sh`** - Installs Google Test framework +- **`install_celero.sh`** - Installs Celero benchmarking library + +### Data Processing Libraries +- **`install_snappy.sh`** - Installs Snappy compression library +- **`install_yamlcpp.sh`** - Installs YAML-CPP parser +- **`install_protobuf.sh`** - Installs Protocol Buffers + +### Parsing and Regex +- **`install_antlr.sh`** - Installs ANTLR4 parser generator +- **`install_pcre2.sh`** - Installs PCRE2 regex library + +### Cloud Services +- **`install_aws_sdk.sh`** - Installs AWS SDK C++ and Lambda runtime + +### Master Script +- **`install_all.sh`** - Runs all scripts in the correct dependency order + +## Usage + +### Individual Installation +Each script can be run independently: + +```bash +# Install just LLVM +./install_llvm.sh + +# Install with custom version +LLVM_VERSION=17.0.0 ./install_llvm.sh + +# Install with custom prefix +PREFIX=/usr/local ./install_boost.sh +``` + +### Complete Installation +Use the master script to install everything: + +```bash +./install_all.sh +``` + +## Docker Build + +### Quick Start +Use the provided build script for easy Docker builds: + +```bash +# Build for current platform +./build.sh + +# Build for specific platform +./build.sh --platform amd64 +./build.sh --platform arm64 + +# Build multi-platform image +./build.sh --platform both --buildx + +# Build and push to registry +./build.sh --platform both --buildx --push +``` + +### Manual Docker Build +Build manually with Docker: + +```bash +# Single platform build +docker build --platform linux/amd64 -t tuplex/musl:amd64 . + +# Multi-platform build (requires docker buildx) +docker buildx build --platform linux/amd64,linux/arm64 -t tuplex/musl:latest . +``` + +### Docker Build Optimization +For optimal Docker layer caching, the Dockerfile is structured with separate layers: + +```dockerfile +# Base system +RUN /tmp/install_alpine.sh + +# Core dependencies +RUN /tmp/install_llvm.sh && /tmp/install_boost.sh + +# Libraries +RUN /tmp/install_zlib.sh && /tmp/install_googletest.sh && /tmp/install_snappy.sh + +# Parsing libraries +RUN /tmp/install_yamlcpp.sh && /tmp/install_celero.sh && /tmp/install_antlr.sh + +# System libraries +RUN /tmp/install_pcre2.sh && /tmp/install_protobuf.sh + +# Cloud services +RUN /tmp/install_aws_sdk.sh +``` + +### Customizing Versions +Override dependency versions using Docker build arguments: + +```bash +docker build \ + --build-arg BOOST_VERSION=1.89.0 \ + --build-arg LLVM_VERSION=17.0.0 \ + --build-arg AWSSDK_CPP_VERSION=1.12.0 \ + -t tuplex/musl:custom . +``` + +## Environment Variables + +All scripts support the following environment variables: + +- **`PREFIX`** - Installation prefix (default: `/opt`) +- **`WORKDIR`** - Working directory for downloads (default: `/tmp`) +- **`CC`** - C compiler (default: `gcc`) +- **`CXX`** - C++ compiler (default: `g++`) +- **`CPU_COUNT`** - Number of CPU cores for parallel builds (auto-detected) + +### Version-Specific Variables +- **`LLVM_VERSION`** - LLVM version (default: `16.0.6`) +- **`BOOST_VERSION`** - Boost version (default: `1.88.0`) +- **`AWSSDK_CPP_VERSION`** - AWS SDK version (default: `1.11.524`) +- **`ANTLR4_VERSION`** - ANTLR4 version (default: `4.13.1`) +- **`YAML_CPP_VERSION`** - YAML-CPP version (default: `0.8.0`) +- **`CELERO_VERSION`** - Celero version (default: `2.8.3`) +- **`PCRE2_VERSION`** - PCRE2 version (default: `10.45`) +- **`PROTOBUF_VERSION`** - Protocol Buffers version (default: `32.0`) +- **`SNAPPY_VERSION`** - Snappy version (default: `1.2.2`) +- **`ZLIB_VERSION`** - zlib-ng version (default: `2.1.3`) +- **`CMAKE_VERSION`** - CMake version (default: `3.27.5`) + +## Dependencies Order + +The installation order is important due to dependencies: + +1. **Alpine packages + CMake** - Base system and build tools +2. **zlib-ng** - Required by AWS SDK +3. **LLVM** - Compiler infrastructure +4. **Boost** - C++ libraries +5. **Google Test** - Testing framework +6. **Snappy** - Compression library +7. **YAML-CPP** - YAML parsing +8. **Celero** - Benchmarking +9. **ANTLR4** - Parser generator +10. **PCRE2** - Regex library +11. **Protocol Buffers** - Serialization +12. **AWS SDK** - Cloud services (depends on zlib-ng) + +## Requirements + +- Alpine Linux +- Root privileges (run with `sudo` or as root) +- Internet connection for downloading source code +- Sufficient disk space for builds + +## Notes + +- All scripts use `set -euxo pipefail` for strict error handling +- Scripts automatically detect CPU count for parallel builds +- Downloads are cleaned up after installation to save space +- All libraries are built with `-fPIC` for position-independent code +- Most libraries are built as static libraries for better portability +- Docker builds are optimized with separate layers for better caching +- Multi-platform builds are supported via docker buildx diff --git a/scripts/docker/musllinux/build.sh b/scripts/docker/musllinux/build.sh new file mode 100755 index 000000000..e8161b6e7 --- /dev/null +++ b/scripts/docker/musllinux/build.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# Build script for Tuplex musllinux Docker image +# This script demonstrates how to build with different configurations + +set -e + +# Default values +PLATFORM="" +TAG="tuplex/musl" +PUSH=false +BUILDX=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --platform) + PLATFORM="$2" + shift 2 + ;; + --tag) + TAG="$2" + shift 2 + ;; + --push) + PUSH=true + shift + ;; + --buildx) + BUILDX=true + shift + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --platform PLATFORM Target platform (amd64, arm64, or both)" + echo " --tag TAG Docker image tag (default: tuplex/musl)" + echo " --push Push image to registry after build" + echo " --buildx Use docker buildx for multi-platform builds" + echo " --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo "Building Tuplex musllinux Docker image..." +echo "Tag: $TAG" +echo "Platform: ${PLATFORM:-auto-detect}" +echo "Buildx: $BUILDX" +echo "Push: $PUSH" +echo "" + +# Function to build single platform +build_single() { + local platform=$1 + local tag_suffix=$2 + + echo "Building for platform: $platform" + docker build --platform linux/$platform \ + --build-arg BOOST_VERSION=1.88.0 \ + --build-arg LLVM_VERSION=16.0.6 \ + --build-arg AWSSDK_CPP_VERSION=1.11.524 \ + --build-arg ANTLR4_VERSION=4.13.1 \ + --build-arg YAML_CPP_VERSION=0.8.0 \ + --build-arg AWS_LAMBDA_CPP_VERSION=0.2.10 \ + --build-arg PCRE2_VERSION=10.45 \ + --build-arg PROTOBUF_VERSION=32.0 \ + --build-arg CELERO_VERSION=2.8.3 \ + --build-arg SNAPPY_VERSION=1.2.2 \ + --build-arg ZLIB_VERSION=2.1.3 \ + --build-arg CMAKE_VERSION=3.27.5 \ + -t $TAG:$tag_suffix . + + if [ "$PUSH" = true ]; then + echo "Pushing $TAG:$tag_suffix..." + docker push $TAG:$tag_suffix + fi +} + +# Function to build multi-platform +build_multi() { + echo "Building multi-platform image..." + + # Enable buildx if requested + if [ "$BUILDX" = true ]; then + docker buildx create --use --name tuplex-builder || true + fi + + docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg BOOST_VERSION=1.88.0 \ + --build-arg LLVM_VERSION=16.0.6 \ + --build-arg AWSSDK_CPP_VERSION=1.11.524 \ + --build-arg ANTLR4_VERSION=4.13.1 \ + --build-arg YAML_CPP_VERSION=0.8.0 \ + --build-arg AWS_LAMBDA_CPP_VERSION=0.2.10 \ + --build-arg PCRE2_VERSION=10.45 \ + --build-arg PROTOBUF_VERSION=32.0 \ + --build-arg CELERO_VERSION=2.8.3 \ + --build-arg SNAPPY_VERSION=1.2.2 \ + --build-arg ZLIB_VERSION=2.1.3 \ + --build-arg CMAKE_VERSION=3.27.5 \ + -t $TAG:latest . + + if [ "$PUSH" = true ]; then + echo "Pushing multi-platform image..." + docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg BOOST_VERSION=1.88.0 \ + --build-arg LLVM_VERSION=16.0.6 \ + --build-arg AWSSDK_CPP_VERSION=1.11.524 \ + --build-arg ANTLR4_VERSION=4.13.1 \ + --build-arg YAML_CPP_VERSION=0.8.0 \ + --build-arg AWS_LAMBDA_CPP_VERSION=0.2.10 \ + --build-arg PCRE2_VERSION=10.45 \ + --build-arg PROTOBUF_VERSION=32.0 \ + --build-arg CELERO_VERSION=2.8.3 \ + --build-arg SNAPPY_VERSION=1.2.2 \ + --build-arg ZLIB_VERSION=2.1.3 \ + --build-arg CMAKE_VERSION=3.27.5 \ + -t $TAG:latest --push . + fi +} + +# Main build logic +if [ -z "$PLATFORM" ]; then + # Auto-detect platform + if [ "$BUILDX" = true ]; then + build_multi + else + # Build for current platform + CURRENT_ARCH=$(uname -m) + if [ "$CURRENT_ARCH" = "x86_64" ]; then + build_single "amd64" "latest" + elif [ "$CURRENT_ARCH" = "aarch64" ] || [ "$CURRENT_ARCH" = "arm64" ]; then + build_single "arm64" "latest" + else + echo "Unsupported architecture: $CURRENT_ARCH" + exit 1 + fi + fi +else + case $PLATFORM in + "amd64") + build_single "amd64" "amd64" + ;; + "arm64") + build_single "arm64" "arm64" + ;; + "both"|"multi") + build_multi + ;; + *) + echo "Unsupported platform: $PLATFORM" + echo "Supported platforms: amd64, arm64, both" + exit 1 + ;; + esac +fi + +echo "" +echo "Build completed successfully!" +echo "Image: $TAG" + +# Show available images +echo "" +echo "Available images:" +docker images | grep $TAG || echo "No images found with tag $TAG" diff --git a/scripts/docker/musllinux/install_all.sh b/scripts/docker/musllinux/install_all.sh new file mode 100755 index 000000000..6dbf099a1 --- /dev/null +++ b/scripts/docker/musllinux/install_all.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Master script that installs all tuplex dependencies in the correct order. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Starting installation of all Tuplex dependencies" + +# Create base directories +echo ">> Creating base directories" +mkdir -p $PREFIX && chmod 0755 $PREFIX +mkdir -p $PREFIX/sbin +mkdir -p $PREFIX/bin +mkdir -p $PREFIX/share +mkdir -p $PREFIX/include +mkdir -p $PREFIX/lib + +echo ">> Files will be downloaded to ${WORKDIR}/tuplex-downloads" +DOWNLOAD_DIR=$WORKDIR/tuplex-downloads +mkdir -p $DOWNLOAD_DIR + +# Set environment variables +export CC=${CC:-gcc} +export CXX=${CXX:-g++} +export PATH=$PREFIX/bin:$PATH + +# 1. Install Alpine packages and CMake (base system dependencies) +echo ">> Step 1: Installing Alpine packages and CMake" +./install_alpine.sh + +# 2. Install zlib-ng (required by AWS SDK) +echo ">> Step 2: Installing zlib-ng" +./install_zlib.sh + +# 3. Install LLVM (compiler infrastructure) +echo ">> Step 3: Installing LLVM" +./install_llvm.sh + +# 4. Install Boost (C++ libraries) +echo ">> Step 4: Installing Boost" +./install_boost.sh + +# 5. Install Google Test (testing framework) +echo ">> Step 5: Installing Google Test" +./install_googletest.sh + +# 6. Install Snappy (compression library) +echo ">> Step 6: Installing Snappy" +./install_snappy.sh + +# 7. Install YAML-CPP (YAML parsing) +echo ">> Step 7: Installing YAML-CPP" +./install_yamlcpp.sh + +# 8. Install Celero (benchmarking) +echo ">> Step 8: Installing Celero" +./install_celero.sh + +# 9. Install ANTLR4 (parser generator) +echo ">> Step 9: Installing ANTLR4" +./install_antlr.sh + +# 10. Install PCRE2 (regex library) +echo ">> Step 10: Installing PCRE2" +./install_pcre2.sh + +# 11. Install Protocol Buffers (serialization) +echo ">> Step 11: Installing Protocol Buffers" +./install_protobuf.sh + +# 12. Install AWS SDK C++ and Lambda runtime +echo ">> Step 12: Installing AWS SDK C++ and Lambda runtime" +./install_aws_sdk.sh + +# Clean up downloads +echo ">> Cleaning up download directory" +rm -rf ${DOWNLOAD_DIR} + +echo ">> All Tuplex dependencies installed successfully to ${PREFIX}" + +# Verify that key tools are available +echo ">> Verifying installation..." +ls -la /opt/bin/ +echo ">> PATH: $PATH" +echo ">> CMake version:" +/opt/bin/cmake --version || echo "CMake not found in /opt/bin" +echo ">> Python version:" +python3 --version || echo "Python3 not found" + +echo "-- Done, all Tuplex requirements installed to /opt --" diff --git a/scripts/docker/musllinux/install_alpine.sh b/scripts/docker/musllinux/install_alpine.sh new file mode 100755 index 000000000..91468f453 --- /dev/null +++ b/scripts/docker/musllinux/install_alpine.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Alpine Linux packages and CMake required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +CMAKE_VERSION=${CMAKE_VERSION:-3.27.5} + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Alpine Linux packages and CMake" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p $DOWNLOAD_DIR + +echo ">> Installing apk dependencies" +apk update + +apk add --no-cache \ + autoconf automake libtool \ + curl libxml2-dev vim build-base \ + openssl-dev zlib-dev ncurses-dev \ + readline-dev sqlite-dev \ + bzip2-dev expat-dev xz-dev \ + tk-dev libffi-dev wget git \ + curl-dev python3-dev py3-pip \ + openjdk11 ninja \ + linux-headers musl-dev \ + openssh file-dev + +echo ">> Installing recent cmake" +# fetch recent cmake & install - detect architecture +ARCH=$(uname -m) +if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + CMAKE_ARCH="aarch64" +else + CMAKE_ARCH="x86_64" +fi +URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz +mkdir -p ${DOWNLOAD_DIR}/cmake && cd ${DOWNLOAD_DIR}/cmake && + curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + tar -v -zxf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + rm -f cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && + cd cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} && + cp -rp bin/* ${PREFIX}/bin/ && + cp -rp share/* ${PREFIX}/share/ && + cd / && rm -rf ${DOWNLOAD_DIR}/cmake + +export PATH=$PREFIX/bin:$PATH +cmake --version + +echo ">> Alpine Linux packages and CMake installation completed successfully" diff --git a/scripts/docker/musllinux/install_antlr.sh b/scripts/docker/musllinux/install_antlr.sh new file mode 100755 index 000000000..2476ba815 --- /dev/null +++ b/scripts/docker/musllinux/install_antlr.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs ANTLR4 dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +ANTLR4_VERSION=${ANTLR4_VERSION:-4.13.1} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing ANTLR4 ${ANTLR4_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr \ +&& curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \ +&& cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ \ +&& curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime \ +&& rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& cd antlr4-cpp-runtime \ +&& mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT}&& make install + +echo ">> ANTLR4 ${ANTLR4_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_aws_sdk.sh b/scripts/docker/musllinux/install_aws_sdk.sh new file mode 100755 index 000000000..61f1004d0 --- /dev/null +++ b/scripts/docker/musllinux/install_aws_sdk.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs AWS SDK C++ and AWS Lambda C++ runtime dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +AWSSDK_CPP_VERSION=${AWSSDK_CPP_VERSION:-1.11.524} +AWS_LAMBDA_CPP_VERSION=${AWS_LAMBDA_CPP_VERSION:-0.2.10} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing AWS SDK C++ ${AWSSDK_CPP_VERSION} and AWS Lambda C++ runtime ${AWS_LAMBDA_CPP_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/aws + +# Install AWS SDK C++ +echo ">> Installing AWS SDK C++" +cd ${DOWNLOAD_DIR}/aws +git clone --progress --verbose --recurse-submodules https://github.com/aws/aws-sdk-cpp.git +cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. +make -j ${CPU_COUNT} +make install + +# Installing AWS Lambda C++ runtime +echo ">> Installing AWS Lambda C++ runtime" +cd ${DOWNLOAD_DIR}/aws +git clone https://github.com/awslabs/aws-lambda-cpp.git +cd aws-lambda-cpp +git fetch && git fetch --tags +git checkout v${AWS_LAMBDA_CPP_VERSION} +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. +make -j${CPU_COUNT} && make install + +echo ">> AWS SDK C++ ${AWSSDK_CPP_VERSION} and AWS Lambda C++ runtime ${AWS_LAMBDA_CPP_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_boost.sh b/scripts/docker/musllinux/install_boost.sh new file mode 100755 index 000000000..88f389718 --- /dev/null +++ b/scripts/docker/musllinux/install_boost.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Boost dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +BOOST_VERSION=${BOOST_VERSION:-1.88.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Boost ${BOOST_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p "${DOWNLOAD_DIR}/boost" + +# create underscored version, e.g. 1.79.0 -> 1_79_0 +boost_underscored_version=$(echo "${BOOST_VERSION}" | tr . _) + +# Download and build Boost including boost python +cd "${DOWNLOAD_DIR}/boost" +curl -L -O "https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz" +tar xf "boost-${BOOST_VERSION}-b2-nodocs.tar.gz" +cd "${DOWNLOAD_DIR}/boost/boost-${BOOST_VERSION}" +./bootstrap.sh --with-python="${PYTHON_EXECUTABLE}" --prefix="${PREFIX}" --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" +./b2 cxxflags="-fPIC" link=static -j "${CPU_COUNT}" +./b2 cxxflags="-fPIC" link=static install +sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' "${PREFIX}/include/boost/thread/pthread/thread_data.hpp" + +cd "${DOWNLOAD_DIR}" +rm -rf "${DOWNLOAD_DIR}/boost" + +echo ">> Boost ${BOOST_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_celero.sh b/scripts/docker/musllinux/install_celero.sh new file mode 100755 index 000000000..dc8551c74 --- /dev/null +++ b/scripts/docker/musllinux/install_celero.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Celero dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +CELERO_VERSION=${CELERO_VERSION:-2.8.3} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Celero ${CELERO_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/celero && cd ${DOWNLOAD_DIR}/celero \ +&& git clone https://github.com/DigitalInBlue/Celero.git celero && cd celero \ +&& git checkout tags/v${CELERO_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC -std=c++11" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Celero ${CELERO_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_googletest.sh b/scripts/docker/musllinux/install_googletest.sh new file mode 100755 index 000000000..1b0c10b70 --- /dev/null +++ b/scripts/docker/musllinux/install_googletest.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Google Test dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +GOOGLETEST_VERSION=${GOOGLETEST_VERSION:-1.14.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Google Test ${GOOGLETEST_VERSION}" + +git clone https://github.com/google/googletest.git -b v${GOOGLETEST_VERSION} && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release .. && make -j ${CPU_COUNT} && make install + +echo ">> Google Test ${GOOGLETEST_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_llvm.sh b/scripts/docker/musllinux/install_llvm.sh new file mode 100755 index 000000000..c038831b3 --- /dev/null +++ b/scripts/docker/musllinux/install_llvm.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs LLVM dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +LLVM_VERSION=${LLVM_VERSION:-16.0.6} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing LLVM ${LLVM_VERSION}" + +# Parse LLVM version components +LLVM_MAJOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f1` +LLVM_MINOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f2` +LLVM_MAJMIN_VERSION="${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}" + +# list of targets available to build: AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SystemZ;VE;WebAssembly;X86;XCore +# in order to cross-compile, should use targets: + +echo ">> building LLVM ${LLVM_VERSION}" +LLVM_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz +CLANG_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/clang-${LLVM_VERSION}.src.tar.xz +# required when LLVM version > 15 +LLVM_CMAKE_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cmake-${LLVM_VERSION}.src.tar.xz + +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) +echo ">> Building dependencies for ${PYTHON_VERSION}" + +echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}" +LLVM_WORKDIR=${WORKDIR}/tuplex-downloads/llvm${LLVM_VERSION} +mkdir -p ${LLVM_WORKDIR} +cd "${LLVM_WORKDIR}" || exit 1 + +wget ${LLVM_URL} && tar xf llvm-${LLVM_VERSION}.src.tar.xz +wget ${CLANG_URL} && tar xf clang-${LLVM_VERSION}.src.tar.xz && mv clang-${LLVM_VERSION}.src llvm-${LLVM_VERSION}.src/../clang + +if [ $LLVM_MAJOR_VERSION -ge 15 ]; then + wget ${LLVM_CMAKE_URL} && tar xf cmake-${LLVM_VERSION}.src.tar.xz && mv cmake-${LLVM_VERSION}.src cmake +fi + +mkdir -p llvm-${LLVM_VERSION}.src/build && cd llvm-${LLVM_VERSION}.src/build + +cmake -GNinja -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm-${LLVM_VERSION} .. +ninja install +cd /tmp + +echo ">> LLVM ${LLVM_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_pcre2.sh b/scripts/docker/musllinux/install_pcre2.sh new file mode 100755 index 000000000..b77419292 --- /dev/null +++ b/scripts/docker/musllinux/install_pcre2.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs PCRE2 dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PCRE2_VERSION=${PCRE2_VERSION:-10.45} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing PCRE2 ${PCRE2_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/pcre2 && cd ${DOWNLOAD_DIR}/pcre2 \ +&& curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PCRE2_VERSION}/pcre2-${PCRE2_VERSION}.zip \ +&& unzip pcre2-${PCRE2_VERSION}.zip \ +&& rm pcre2-${PCRE2_VERSION}.zip \ +&& cd pcre2-${PCRE2_VERSION} \ +&& ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \ +&& make -j${CPU_COUNT} && make install + +echo ">> PCRE2 ${PCRE2_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_protobuf.sh b/scripts/docker/musllinux/install_protobuf.sh new file mode 100755 index 000000000..16c499304 --- /dev/null +++ b/scripts/docker/musllinux/install_protobuf.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Protocol Buffers dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PROTOBUF_VERSION=${PROTOBUF_VERSION:-32.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Protocol Buffers ${PROTOBUF_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/protobuf && cd ${DOWNLOAD_DIR}/protobuf \ +&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j ${CPU_COUNT} && make install + +echo ">> Protocol Buffers ${PROTOBUF_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_requirements.sh b/scripts/docker/musllinux/install_requirements.sh index 5361e1a83..320b3069c 100644 --- a/scripts/docker/musllinux/install_requirements.sh +++ b/scripts/docker/musllinux/install_requirements.sh @@ -135,7 +135,7 @@ apk add --no-cache \ curl-dev python3-dev py3-pip \ openjdk11 ninja \ linux-headers musl-dev \ - openssh + openssh file-dev export CC=${CC} export CXX=${CXX} diff --git a/scripts/docker/musllinux/install_snappy.sh b/scripts/docker/musllinux/install_snappy.sh new file mode 100755 index 000000000..17334a857 --- /dev/null +++ b/scripts/docker/musllinux/install_snappy.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs Snappy dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +SNAPPY_VERSION=${SNAPPY_VERSION:-1.2.2} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing Snappy ${SNAPPY_VERSION}" + +# build snappy as static lib +git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install + +echo ">> Snappy ${SNAPPY_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_yamlcpp.sh b/scripts/docker/musllinux/install_yamlcpp.sh new file mode 100755 index 000000000..117cc4b6d --- /dev/null +++ b/scripts/docker/musllinux/install_yamlcpp.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs YAML-CPP dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +YAML_CPP_VERSION=${YAML_CPP_VERSION:-0.8.0} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing YAML-CPP ${YAML_CPP_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/yamlcpp && cd ${DOWNLOAD_DIR}/yamlcpp \ +&& git clone https://github.com/jbeder/yaml-cpp.git yaml-cpp \ +&& cd yaml-cpp \ +&& git checkout tags/${YAML_CPP_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> YAML-CPP ${YAML_CPP_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_zlib.sh b/scripts/docker/musllinux/install_zlib.sh new file mode 100755 index 000000000..72ad2b343 --- /dev/null +++ b/scripts/docker/musllinux/install_zlib.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs zlib-ng dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +ZLIB_VERSION=${ZLIB_VERSION:-2.1.3} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing zlib-ng ${ZLIB_VERSION}" + +# Set LD_LIBRARY_PATH +LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64:$LD_LIBRARY_PATH + +# note that zlib defines Z_NULL=0 whereas zlib-ng defines it as NULL, patch aws sdk accordingly +git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/${ZLIB_VERSION} && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON .. && make -j ${CPU_COUNT} && make install + +echo ">> zlib-ng ${ZLIB_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/test_docker_build.sh b/scripts/docker/musllinux/test_docker_build.sh new file mode 100755 index 000000000..25ac6b0fc --- /dev/null +++ b/scripts/docker/musllinux/test_docker_build.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Test script to verify Docker build works with new modular scripts + +set -e + +echo "Testing Docker build with new modular scripts..." +echo "================================================" + +# Check if Docker is available +if ! command -v docker &> /dev/null; then + echo "❌ Docker is not installed or not in PATH" + exit 1 +fi + +echo "✅ Docker is available" + +# Check if all required scripts exist +REQUIRED_SCRIPTS=( + "install_alpine.sh" + "install_zlib.sh" + "install_llvm.sh" + "install_boost.sh" + "install_googletest.sh" + "install_snappy.sh" + "install_yamlcpp.sh" + "install_celero.sh" + "install_antlr.sh" + "install_pcre2.sh" + "install_protobuf.sh" + "install_aws_sdk.sh" +) + +echo "Checking required scripts..." +for script in "${REQUIRED_SCRIPTS[@]}"; do + if [ -f "$script" ]; then + echo "✅ $script exists" + else + echo "❌ $script missing" + exit 1 + fi +done + +# Check if scripts are executable +echo "Checking script permissions..." +for script in "${REQUIRED_SCRIPTS[@]}"; do + if [ -x "$script" ]; then + echo "✅ $script is executable" + else + echo "❌ $script is not executable" + exit 1 + fi +done + +# Check if Dockerfile exists +if [ -f "Dockerfile" ]; then + echo "✅ Dockerfile exists" +else + echo "❌ Dockerfile missing" + exit 1 +fi + +# Check if .dockerignore exists +if [ -f ".dockerignore" ]; then + echo "✅ .dockerignore exists" +else + echo "❌ .dockerignore missing" + exit 1 +fi + +# Check if build script exists +if [ -f "build.sh" ]; then + echo "✅ build.sh exists" + if [ -x "build.sh" ]; then + echo "✅ build.sh is executable" + else + echo "❌ build.sh is not executable" + exit 1 + fi +else + echo "❌ build.sh missing" + exit 1 +fi + +echo "" +echo "All checks passed! 🎉" +echo "" +echo "You can now build the Docker image using:" +echo " ./build.sh # Build for current platform" +echo " ./build.sh --platform amd64 # Build for AMD64" +echo " ./build.sh --platform arm64 # Build for ARM64" +echo " ./build.sh --platform both # Build multi-platform" +echo "" +echo "Or build manually with:" +echo " docker build -t tuplex/musl ." +echo "" +echo "The new modular approach provides:" +echo " ✅ Better Docker layer caching" +echo " ✅ Easier debugging and maintenance" +echo " ✅ Flexible version customization" +echo " ✅ Individual dependency installation" +echo " ✅ Multi-platform build support" From 026fb705aaeaf384d0ecc6acc2e87b180432c66a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 16:39:00 -0700 Subject: [PATCH 10/92] prefix --- scripts/docker/musllinux/Dockerfile | 3 + scripts/docker/musllinux/README.md | 20 +++- scripts/docker/musllinux/build.sh | 9 ++ .../docker/musllinux/install_googletest.sh | 2 +- scripts/docker/musllinux/install_llvm.sh | 2 +- scripts/docker/musllinux/install_protobuf.sh | 2 +- scripts/docker/musllinux/install_snappy.sh | 2 +- scripts/docker/musllinux/install_zlib.sh | 2 +- .../docker/musllinux/test_prefix_config.sh | 95 +++++++++++++++++++ 9 files changed, 130 insertions(+), 7 deletions(-) create mode 100755 scripts/docker/musllinux/test_prefix_config.sh diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index 922c5ec79..aa2d96313 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -37,6 +37,9 @@ FROM base-${TARGETPLATFORM#linux/} # docker build -t tuplex/musl . # docker run -it tuplex/musl bash +# Installation prefix - all dependencies will be installed to this directory +ARG PREFIX=/opt + # Version arguments for dependencies ARG BOOST_VERSION=1.88.0 ARG LLVM_VERSION=16.0.6 diff --git a/scripts/docker/musllinux/README.md b/scripts/docker/musllinux/README.md index 252332deb..875b73686 100644 --- a/scripts/docker/musllinux/README.md +++ b/scripts/docker/musllinux/README.md @@ -72,6 +72,9 @@ Use the provided build script for easy Docker builds: # Build and push to registry ./build.sh --platform both --buildx --push + +# Build with custom installation prefix +./build.sh --prefix /usr/local ``` ### Manual Docker Build @@ -108,15 +111,28 @@ RUN /tmp/install_pcre2.sh && /tmp/install_protobuf.sh RUN /tmp/install_aws_sdk.sh ``` -### Customizing Versions -Override dependency versions using Docker build arguments: +### Customizing Versions and Installation Path +Override dependency versions and installation path using Docker build arguments: ```bash +# Custom versions docker build \ --build-arg BOOST_VERSION=1.89.0 \ --build-arg LLVM_VERSION=17.0.0 \ --build-arg AWSSDK_CPP_VERSION=1.12.0 \ -t tuplex/musl:custom . + +# Custom installation prefix +docker build \ + --build-arg PREFIX=/usr/local \ + -t tuplex/musl:custom-prefix . + +# Both custom versions and prefix +docker build \ + --build-arg PREFIX=/usr/local \ + --build-arg BOOST_VERSION=1.89.0 \ + --build-arg LLVM_VERSION=17.0.0 \ + -t tuplex/musl:custom-all . ``` ## Environment Variables diff --git a/scripts/docker/musllinux/build.sh b/scripts/docker/musllinux/build.sh index e8161b6e7..69f647788 100755 --- a/scripts/docker/musllinux/build.sh +++ b/scripts/docker/musllinux/build.sh @@ -7,6 +7,7 @@ set -e # Default values PLATFORM="" TAG="tuplex/musl" +PREFIX="/opt" PUSH=false BUILDX=false @@ -21,6 +22,10 @@ while [[ $# -gt 0 ]]; do TAG="$2" shift 2 ;; + --prefix) + PREFIX="$2" + shift 2 + ;; --push) PUSH=true shift @@ -34,6 +39,7 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --platform PLATFORM Target platform (amd64, arm64, or both)" echo " --tag TAG Docker image tag (default: tuplex/musl)" + echo " --prefix PREFIX Installation prefix (default: /opt)" echo " --push Push image to registry after build" echo " --buildx Use docker buildx for multi-platform builds" echo " --help Show this help message" @@ -48,6 +54,7 @@ done echo "Building Tuplex musllinux Docker image..." echo "Tag: $TAG" +echo "Prefix: $PREFIX" echo "Platform: ${PLATFORM:-auto-detect}" echo "Buildx: $BUILDX" echo "Push: $PUSH" @@ -60,6 +67,7 @@ build_single() { echo "Building for platform: $platform" docker build --platform linux/$platform \ + --build-arg PREFIX=$PREFIX \ --build-arg BOOST_VERSION=1.88.0 \ --build-arg LLVM_VERSION=16.0.6 \ --build-arg AWSSDK_CPP_VERSION=1.11.524 \ @@ -90,6 +98,7 @@ build_multi() { fi docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg PREFIX=$PREFIX \ --build-arg BOOST_VERSION=1.88.0 \ --build-arg LLVM_VERSION=16.0.6 \ --build-arg AWSSDK_CPP_VERSION=1.11.524 \ diff --git a/scripts/docker/musllinux/install_googletest.sh b/scripts/docker/musllinux/install_googletest.sh index 1b0c10b70..d7917e2db 100755 --- a/scripts/docker/musllinux/install_googletest.sh +++ b/scripts/docker/musllinux/install_googletest.sh @@ -23,6 +23,6 @@ fi echo ">> Installing Google Test ${GOOGLETEST_VERSION}" -git clone https://github.com/google/googletest.git -b v${GOOGLETEST_VERSION} && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release .. && make -j ${CPU_COUNT} && make install +git clone https://github.com/google/googletest.git -b v${GOOGLETEST_VERSION} && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install echo ">> Google Test ${GOOGLETEST_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_llvm.sh b/scripts/docker/musllinux/install_llvm.sh index c038831b3..cd9c73f61 100755 --- a/scripts/docker/musllinux/install_llvm.sh +++ b/scripts/docker/musllinux/install_llvm.sh @@ -56,7 +56,7 @@ mkdir -p llvm-${LLVM_VERSION}.src/build && cd llvm-${LLVM_VERSION}.src/build cmake -GNinja -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ - -DCMAKE_INSTALL_PREFIX=/opt/llvm-${LLVM_VERSION} .. + -DCMAKE_INSTALL_PREFIX=${PREFIX}/llvm-${LLVM_VERSION} .. ninja install cd /tmp diff --git a/scripts/docker/musllinux/install_protobuf.sh b/scripts/docker/musllinux/install_protobuf.sh index 16c499304..ceb2ca443 100755 --- a/scripts/docker/musllinux/install_protobuf.sh +++ b/scripts/docker/musllinux/install_protobuf.sh @@ -26,6 +26,6 @@ echo ">> Installing Protocol Buffers ${PROTOBUF_VERSION}" # Create download directory DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads mkdir -p ${DOWNLOAD_DIR}/protobuf && cd ${DOWNLOAD_DIR}/protobuf \ -&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j ${CPU_COUNT} && make install +&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install echo ">> Protocol Buffers ${PROTOBUF_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_snappy.sh b/scripts/docker/musllinux/install_snappy.sh index 17334a857..c6c8437d1 100755 --- a/scripts/docker/musllinux/install_snappy.sh +++ b/scripts/docker/musllinux/install_snappy.sh @@ -24,6 +24,6 @@ fi echo ">> Installing Snappy ${SNAPPY_VERSION}" # build snappy as static lib -git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install +git clone https://github.com/google/snappy.git -b ${SNAPPY_VERSION} && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install echo ">> Snappy ${SNAPPY_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/install_zlib.sh b/scripts/docker/musllinux/install_zlib.sh index 72ad2b343..adfb67bdc 100755 --- a/scripts/docker/musllinux/install_zlib.sh +++ b/scripts/docker/musllinux/install_zlib.sh @@ -28,6 +28,6 @@ LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64:$LD_LIBRARY_PATH # note that zlib defines Z_NULL=0 whereas zlib-ng defines it as NULL, patch aws sdk accordingly -git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/${ZLIB_VERSION} && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON .. && make -j ${CPU_COUNT} && make install +git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/${ZLIB_VERSION} && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON -DCMAKE_INSTALL_PREFIX=${PREFIX} .. && make -j ${CPU_COUNT} && make install echo ">> zlib-ng ${ZLIB_VERSION} installation completed successfully" diff --git a/scripts/docker/musllinux/test_prefix_config.sh b/scripts/docker/musllinux/test_prefix_config.sh new file mode 100755 index 000000000..8dd74b1af --- /dev/null +++ b/scripts/docker/musllinux/test_prefix_config.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Test script to verify PREFIX variable configuration in all installation scripts + +set -e + +echo "Testing PREFIX variable configuration in installation scripts..." +echo "=============================================================" + +# Test PREFIX variable definition +echo "1. Checking PREFIX variable definition..." +for script in install_*.sh; do + if [ "$script" = "install_all.sh" ]; then + continue # Skip the master script + fi + + if grep -q "PREFIX=\${PREFIX:-/opt}" "$script"; then + echo "✅ $script: PREFIX variable properly defined" + else + echo "❌ $script: PREFIX variable missing or incorrectly defined" + exit 1 + fi +done + +echo "" +echo "2. Checking cmake installations use -DCMAKE_INSTALL_PREFIX..." +CMAKE_SCRIPTS=( + "install_zlib.sh" + "install_googletest.sh" + "install_snappy.sh" + "install_yamlcpp.sh" + "install_celero.sh" + "install_antlr.sh" + "install_protobuf.sh" + "install_aws_sdk.sh" +) + +for script in "${CMAKE_SCRIPTS[@]}"; do + if grep -q "CMAKE_INSTALL_PREFIX.*PREFIX" "$script"; then + echo "✅ $script: cmake uses PREFIX variable" + else + echo "❌ $script: cmake missing PREFIX variable" + exit 1 + fi +done + +echo "" +echo "3. Checking configure installations use --prefix..." +CONFIGURE_SCRIPTS=( + "install_pcre2.sh" +) + +for script in "${CONFIGURE_SCRIPTS[@]}"; do + if grep -q "prefix.*PREFIX" "$script"; then + echo "✅ $script: configure uses PREFIX variable" + else + echo "❌ $script: configure missing PREFIX variable" + exit 1 + fi +done + +echo "" +echo "4. Checking bootstrap installations use --prefix..." +BOOTSTRAP_SCRIPTS=( + "install_boost.sh" +) + +for script in "${BOOTSTRAP_SCRIPTS[@]}"; do + if grep -q "prefix.*PREFIX" "$script"; then + echo "✅ $script: bootstrap uses PREFIX variable" + else + echo "❌ $script: bootstrap missing PREFIX variable" + exit 1 + fi +done + +echo "" +echo "5. Checking LLVM installation path..." +if grep -q "CMAKE_INSTALL_PREFIX.*PREFIX.*llvm" install_llvm.sh; then + echo "✅ install_llvm.sh: Uses PREFIX variable for LLVM installation" +else + echo "❌ install_llvm.sh: LLVM installation path not using PREFIX variable" + exit 1 +fi + +echo "" +echo "All PREFIX variable configurations are correct! 🎉" +echo "" +echo "Summary of installation methods:" +echo " ✅ cmake: -DCMAKE_INSTALL_PREFIX=\${PREFIX}" +echo " ✅ configure: --prefix=\${PREFIX}" +echo " ✅ bootstrap: --prefix=\"\${PREFIX}\"" +echo " ✅ LLVM: \${PREFIX}/llvm-\${LLVM_VERSION}" +echo "" +echo "All scripts will now install to the directory specified by the PREFIX" +echo "environment variable (default: /opt) instead of hardcoded paths." From 3e1cb5a23b1cd5963b2aaa43d901284e8cfea2a7 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 16:52:53 -0700 Subject: [PATCH 11/92] update antlr install because of cmake 4.x issues --- scripts/docker/musllinux/Dockerfile | 22 ++--- scripts/docker/musllinux/install_antlr.sh | 106 ++++++++++++++++++++-- 2 files changed, 108 insertions(+), 20 deletions(-) diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index aa2d96313..0ea1d07e3 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -105,22 +105,22 @@ RUN /tmp/install_celero.sh # Layer 7: Install parsing and regex libraries RUN /tmp/install_antlr.sh -RUN /tmp/install_pcre2.sh +# RUN /tmp/install_pcre2.sh -# Layer 8: Install Protocol Buffers -RUN /tmp/install_protobuf.sh +# # Layer 8: Install Protocol Buffers +# RUN /tmp/install_protobuf.sh -# Layer 9: Install AWS SDK C++ and Lambda runtime -RUN /tmp/install_aws_sdk.sh +# # Layer 9: Install AWS SDK C++ and Lambda runtime +# RUN /tmp/install_aws_sdk.sh -# Layer 4: Install Boost (C++ libraries) -RUN /tmp/install_boost.sh +# # Layer 4: Install Boost (C++ libraries) +# RUN /tmp/install_boost.sh -# Layer 3: Install LLVM (compiler infrastructure) (this will take a while) -RUN /tmp/install_llvm.sh +# # Layer 3: Install LLVM (compiler infrastructure) (this will take a while) +# RUN /tmp/install_llvm.sh -# Clean up installation scripts -RUN rm -f /tmp/install_*.sh +# # Clean up installation scripts +# RUN rm -f /tmp/install_*.sh # Fix CMake issue by using the working pipx-installed version RUN ln -sf /opt/_internal/pipx/venvs/cmake/bin/cmake /opt/bin/cmake diff --git a/scripts/docker/musllinux/install_antlr.sh b/scripts/docker/musllinux/install_antlr.sh index 2476ba815..1197a1442 100755 --- a/scripts/docker/musllinux/install_antlr.sh +++ b/scripts/docker/musllinux/install_antlr.sh @@ -1,6 +1,10 @@ #!/bin/sh # (c) Tuplex team 2017-2023 # Installs ANTLR4 dependencies required to build tuplex on Alpine Linux. +# +# KNOWN ISSUE: The ANTLR C++ runtime build may fail due to CMake version +# compatibility issues with its embedded Google Test dependency. This script +# will fail explicitly if the build doesn't succeed. # Variables needed incl. defaults. PREFIX=${PREFIX:-/opt} @@ -15,6 +19,17 @@ CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) # Start script. set -euxo pipefail +# Cleanup function +cleanup() { + if [ -d "${DOWNLOAD_DIR}/antlr" ]; then + echo ">> Cleaning up on exit..." + rm -rf "${DOWNLOAD_DIR}/antlr" + fi +} + +# Set trap to cleanup on exit +trap cleanup EXIT + # need to run this with root privileges if [ "$(id -u)" -ne 0 ]; then echo "Please run this script with root privileges" @@ -25,14 +40,87 @@ echo ">> Installing ANTLR4 ${ANTLR4_VERSION}" # Create download directory DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads -mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr \ -&& curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \ -&& cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ \ -&& curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ -&& unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime \ -&& rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ -&& cd antlr4-cpp-runtime \ -&& mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ -&& make -j ${CPU_COUNT}&& make install +mkdir -p ${DOWNLOAD_DIR}/antlr && cd ${DOWNLOAD_DIR}/antlr + +echo ">> Downloading ANTLR JAR file..." +curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar +cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ +echo ">> ANTLR JAR file installed successfully" + +echo ">> Downloading ANTLR C++ runtime source..." +curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip +unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime +rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip + +echo ">> Building ANTLR C++ runtime..." +cd antlr4-cpp-runtime +echo ">> Removing existing build directory..." +rm -rf build +echo ">> Creating fresh build directory..." +mkdir -p build +echo ">> Entering build directory..." +cd build +echo ">> Current working directory: $(pwd)" + +echo ">> Running cmake configuration..." +echo ">> Note: ANTLR C++ runtime has embedded Google Test with CMake compatibility issues" +echo ">> Attempting to configure with policy overrides..." +echo ">> Current directory: $(pwd)" +echo ">> Running cmake command..." +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. +echo ">> Cmake exit code: $?" +if [ $? -ne 0 ]; then + echo ">> ERROR: cmake configuration failed" + echo ">> This is likely due to CMake version compatibility issues with embedded Google Test" + echo ">> ANTLR C++ runtime requires CMake 3.15+ but has embedded dependencies requiring older versions" + echo ">> Consider using a pre-built ANTLR C++ runtime package instead of building from source" + exit 1 +fi +echo ">> Cmake configuration completed successfully" + +echo ">> Compiling ANTLR C++ runtime..." +echo ">> Current directory: $(pwd)" +echo ">> Checking for Makefile..." +ls -la Makefile* || echo ">> No Makefile found!" +echo ">> Running make command..." +make -j ${CPU_COUNT} +echo ">> Make exit code: $?" +if [ $? -ne 0 ]; then + echo ">> ERROR: make compilation failed" + echo ">> This indicates that the cmake configuration did not generate proper build files" + echo ">> The ANTLR C++ runtime build is not working properly" + exit 1 +fi +echo ">> Make compilation completed successfully" + +echo ">> Installing ANTLR C++ runtime..." +make install +if [ $? -ne 0 ]; then + echo ">> ERROR: make install failed" + exit 1 +fi + +echo ">> Verifying ANTLR C++ runtime installation..." +if ! ls -la ${PREFIX}/lib/libantlr4-runtime* >/dev/null 2>&1; then + echo ">> ERROR: ANTLR C++ runtime library not found after installation" + echo ">> Expected files in ${PREFIX}/lib/:" + ls -la ${PREFIX}/lib/ | grep -i antlr || echo ">> No antlr files found in lib directory" + exit 1 +fi + +if ! ls -la ${PREFIX}/include/antlr4-runtime/ >/dev/null 2>&1; then + echo ">> ERROR: ANTLR C++ runtime headers not found after installation" + echo ">> Expected directory: ${PREFIX}/include/antlr4-runtime/" + echo ">> Contents of ${PREFIX}/include/:" + ls -la ${PREFIX}/include/ || echo ">> Include directory not found" + exit 1 +fi + +echo ">> ANTLR C++ runtime verification successful" + +# Clean up build artifacts +echo ">> Cleaning up build artifacts..." +cd ${DOWNLOAD_DIR} +rm -rf antlr echo ">> ANTLR4 ${ANTLR4_VERSION} installation completed successfully" From 9fc68d4eaba0605378fd2e5e26eff9ecb9f93771 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 16:55:27 -0700 Subject: [PATCH 12/92] cleanup --- scripts/docker/musllinux/Dockerfile | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index 0ea1d07e3..245b15766 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -92,35 +92,38 @@ RUN /tmp/install_alpine.sh # Layer 2: Install zlib-ng (required by AWS SDK) RUN /tmp/install_zlib.sh -# Layer 5: Install testing and compression libraries +# Layer 3: Install testing and compression libraries RUN /tmp/install_googletest.sh +# Layer 4: Install snappy compression library RUN /tmp/install_snappy.sh -# Layer 6: Install parsing and data libraries +# Layer 5: Install parsing and data libraries RUN /tmp/install_yamlcpp.sh +# Layer 6: Install celero benchmarking library RUN /tmp/install_celero.sh # Layer 7: Install parsing and regex libraries RUN /tmp/install_antlr.sh -# RUN /tmp/install_pcre2.sh +# Layer 8: Install parsing and regex library +RUN /tmp/install_pcre2.sh -# # Layer 8: Install Protocol Buffers -# RUN /tmp/install_protobuf.sh +# Layer 9: Install Protocol Buffers +RUN /tmp/install_protobuf.sh -# # Layer 9: Install AWS SDK C++ and Lambda runtime -# RUN /tmp/install_aws_sdk.sh +# Layer 10: Install AWS SDK C++ and Lambda runtime +RUN /tmp/install_aws_sdk.sh -# # Layer 4: Install Boost (C++ libraries) -# RUN /tmp/install_boost.sh +# Layer 11: Install Boost (C++ libraries) +RUN /tmp/install_boost.sh -# # Layer 3: Install LLVM (compiler infrastructure) (this will take a while) -# RUN /tmp/install_llvm.sh +# Layer 12: Install LLVM (compiler infrastructure) (this will take a while) +RUN /tmp/install_llvm.sh -# # Clean up installation scripts -# RUN rm -f /tmp/install_*.sh +# Clean up installation scripts +RUN rm -f /tmp/install_*.sh # Fix CMake issue by using the working pipx-installed version RUN ln -sf /opt/_internal/pipx/venvs/cmake/bin/cmake /opt/bin/cmake From 9dd07b0a4ce9a14b06e3b3c89b0811e13c299ae9 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 16:56:08 -0700 Subject: [PATCH 13/92] cleanup --- scripts/docker/musllinux/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index 245b15766..aa382c9c7 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -125,6 +125,9 @@ RUN /tmp/install_llvm.sh # Clean up installation scripts RUN rm -f /tmp/install_*.sh +# Clean up downloaded temporary files and build artifacts +RUN rm -rf /tmp/tuplex-downloads + # Fix CMake issue by using the working pipx-installed version RUN ln -sf /opt/_internal/pipx/venvs/cmake/bin/cmake /opt/bin/cmake From 485af9d82748f0955e237ced845ebc5dfa8598d6 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 20:18:30 -0700 Subject: [PATCH 14/92] fixes --- scripts/build_linux_musl_wheels.sh | 183 +++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100755 scripts/build_linux_musl_wheels.sh diff --git a/scripts/build_linux_musl_wheels.sh b/scripts/build_linux_musl_wheels.sh new file mode 100755 index 000000000..4c925569d --- /dev/null +++ b/scripts/build_linux_musl_wheels.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# (c) 2017 - 2023 Tuplex team +# this script invokes the cibuildwheel process with necessary env variables to build musl wheels for linux/docker +# builds wheels for python 3.11+ with musl support +# supports building for amd64, arm64, or both architectures + +# check from where script is invoked +CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +# Parse command line arguments +TARGET_ARCH="" +while [[ $# -gt 0 ]]; do + case $1 in + --arch) + TARGET_ARCH="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --arch ARCH Target architecture: amd64, arm64, or both (default: native)" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 # Build for native architecture" + echo " $0 --arch amd64 # Build for AMD64 only" + echo " $0 --arch arm64 # Build for ARM64 only" + echo " $0 --arch both # Build for both architectures" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Determine target architecture +if [[ -z "$TARGET_ARCH" ]]; then + # Auto-detect native architecture + NATIVE_ARCH=$(uname -m) + case "$NATIVE_ARCH" in + x86_64) + TARGET_ARCH="amd64" + ;; + aarch64|arm64) + TARGET_ARCH="arm64" + ;; + *) + echo "Unsupported native architecture: $NATIVE_ARCH" + echo "Please specify --arch explicitly (amd64, arm64, or both)" + exit 1 + ;; + esac + echo "No architecture specified, using native architecture: $TARGET_ARCH" +fi + +# Validate architecture choice +case "$TARGET_ARCH" in + amd64|arm64|both) + echo "Target architecture: $TARGET_ARCH" + ;; + *) + echo "Invalid architecture: $TARGET_ARCH" + echo "Supported architectures: amd64, arm64, both" + exit 1 + ;; +esac + +echo "Executing musl buildwheel script located in $CWD" +pushd $CWD > /dev/null +cd .. + +# delete dir if exists +rm -rf wheelhouse +# delete in tree build files +rm -rf tuplex/python/tuplex/libexec/tuplex*.so + +# Check if musl Docker image exists, build if not +echo "Checking for tuplex/musl Docker image..." +if ! docker image inspect tuplex/musl:latest >/dev/null 2>&1; then + echo "tuplex/musl:latest image not found." + echo "" + case "$TARGET_ARCH" in + both) + echo "To build the tuplex/musl Docker image for both architectures, run:" + echo " cd scripts/docker/musllinux" + echo " ./build.sh --platform both --buildx" + ;; + amd64) + echo "To build the tuplex/musl Docker image for AMD64, run:" + echo " cd scripts/docker/musllinux" + echo " ./build.sh --platform amd64" + ;; + arm64) + echo "To build the tuplex/musl Docker image for ARM64, run:" + echo " cd scripts/docker/musllinux" + echo " ./build.sh --platform arm64" + ;; + esac + echo "" + echo "After building the image, re-run this script." + exit 1 +else + echo "tuplex/musl:latest image found" + if [[ "$TARGET_ARCH" == "both" ]]; then + echo "Note: For multi-architecture builds, ensure the image supports both x86_64 and aarch64" + fi +fi + +# CIBUILDWHEEL CONFIGURATION +export CIBUILDWHEEL=1 +export TUPLEX_BUILD_ALL=0 + +# Set architecture-specific configuration +case "$TARGET_ARCH" in + amd64) + export CIBW_ARCHS_LINUX="x86_64" + export CIBW_MUSLLINUX_X86_64_IMAGE='tuplex/musl:latest' + ;; + arm64) + export CIBW_ARCHS_LINUX="aarch64" + export CIBW_MUSLLINUX_AARCH64_IMAGE='tuplex/musl:latest' + ;; + both) + export CIBW_ARCHS_LINUX="x86_64 aarch64" + export CIBW_MUSLLINUX_X86_64_IMAGE='tuplex/musl:latest' + export CIBW_MUSLLINUX_AARCH64_IMAGE='tuplex/musl:latest' + ;; +esac + +# check whether lambda zip was build and stored in build-lambda +TUPLEX_LAMBDA_ZIP=${TUPLEX_LAMBDA_ZIP:-build-lambda/tplxlam.zip} + +echo "work dir is: $(pwd)" +if [[ -f "${TUPLEX_LAMBDA_ZIP}" ]]; then + echo "Found lambda runner ${TUPLEX_LAMBDA_ZIP}, adding to package" + mkdir -p tuplex/other + cp ${TUPLEX_LAMBDA_ZIP} tuplex/other/tplxlam.zip +fi + +# add to environment, e.g. TUPLEX_BUILD_TYPE=tsan to force a tsan build. Release is the default mode +# Note: The tuplex/musl image uses /opt as the installation prefix for both architectures +export CIBW_ENVIRONMENT="TUPLEX_LAMBDA_ZIP='./tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/opt/lib:/opt/lib64" + +# Build only musllinux wheels for Python 3.11+ +export CIBW_BUILD="cp3{11,12,13}-*" +# Note: CIBW_ARCHS_LINUX is already set above based on TARGET_ARCH + +# Only build musllinux wheels (skip manylinux) +export CIBW_SKIP="*-manylinux_*" + +export CIBW_BUILD_VERBOSITY=3 +export CIBW_PROJECT_REQUIRES_PYTHON=">=3.11" + +# Set platform tag for musl wheels +export CIBW_PLATFORM_LINUX="musllinux_1_2" + +echo "Building musl wheels with the following configuration:" +echo " - Image: tuplex/musl:latest" +echo " - Python versions: 3.11, 3.12, 3.13" +case "$TARGET_ARCH" in + amd64) + echo " - Architecture: x86_64 (AMD64)" + ;; + arm64) + echo " - Architecture: aarch64 (ARM64)" + ;; + both) + echo " - Architectures: x86_64 (AMD64), aarch64 (ARM64)" + ;; +esac +echo " - Platform: musllinux_1_2" +echo " - Skip: manylinux wheels" +echo "" + +cibuildwheel --platform linux . + +popd > /dev/null + +echo "Done building musl wheels!" +echo "Check the wheelhouse directory for the generated .whl files" From 4e55833a53df10b7cce4a255c948a14fcce4684a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 20:51:24 -0700 Subject: [PATCH 15/92] fix yamlcpp install --- scripts/docker/musllinux/install_yamlcpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/musllinux/install_yamlcpp.sh b/scripts/docker/musllinux/install_yamlcpp.sh index 117cc4b6d..efe7367e3 100755 --- a/scripts/docker/musllinux/install_yamlcpp.sh +++ b/scripts/docker/musllinux/install_yamlcpp.sh @@ -30,7 +30,7 @@ mkdir -p ${DOWNLOAD_DIR}/yamlcpp && cd ${DOWNLOAD_DIR}/yamlcpp \ && cd yaml-cpp \ && git checkout tags/${YAML_CPP_VERSION} \ && mkdir build && cd build \ -&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ +&& cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ && make -j ${CPU_COUNT} && make install echo ">> YAML-CPP ${YAML_CPP_VERSION} installation completed successfully" From 56fafc4a6908fbe24d3e566c05fd53dc6e43de7d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 20:56:32 -0700 Subject: [PATCH 16/92] adding zstd to the mix --- scripts/docker/musllinux/Dockerfile | 21 ++++++++------ scripts/docker/musllinux/install_zstd.sh | 37 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) create mode 100755 scripts/docker/musllinux/install_zstd.sh diff --git a/scripts/docker/musllinux/Dockerfile b/scripts/docker/musllinux/Dockerfile index aa382c9c7..653ce3588 100644 --- a/scripts/docker/musllinux/Dockerfile +++ b/scripts/docker/musllinux/Dockerfile @@ -76,7 +76,7 @@ RUN mkdir -p /opt/sbin /opt/bin /opt/lib /opt/include # Copy all installation scripts COPY install_alpine.sh install_zlib.sh install_llvm.sh install_boost.sh \ - install_googletest.sh install_snappy.sh install_yamlcpp.sh install_celero.sh \ + install_googletest.sh install_snappy.sh install_zstd.sh install_yamlcpp.sh install_celero.sh \ install_antlr.sh install_pcre2.sh install_protobuf.sh install_aws_sdk.sh \ /tmp/ @@ -98,28 +98,31 @@ RUN /tmp/install_googletest.sh # Layer 4: Install snappy compression library RUN /tmp/install_snappy.sh -# Layer 5: Install parsing and data libraries +# Layer 5: Install zstd compression library +RUN /tmp/install_zstd.sh + +# Layer 6: Install parsing and data libraries RUN /tmp/install_yamlcpp.sh -# Layer 6: Install celero benchmarking library +# Layer 7: Install celero benchmarking library RUN /tmp/install_celero.sh -# Layer 7: Install parsing and regex libraries +# Layer 8: Install parsing and regex libraries RUN /tmp/install_antlr.sh -# Layer 8: Install parsing and regex library +# Layer 9: Install parsing and regex library RUN /tmp/install_pcre2.sh -# Layer 9: Install Protocol Buffers +# Layer 10: Install Protocol Buffers RUN /tmp/install_protobuf.sh -# Layer 10: Install AWS SDK C++ and Lambda runtime +# Layer 11: Install AWS SDK C++ and Lambda runtime RUN /tmp/install_aws_sdk.sh -# Layer 11: Install Boost (C++ libraries) +# Layer 12: Install Boost (C++ libraries) RUN /tmp/install_boost.sh -# Layer 12: Install LLVM (compiler infrastructure) (this will take a while) +# Layer 13: Install LLVM (compiler infrastructure) (this will take a while) RUN /tmp/install_llvm.sh # Clean up installation scripts diff --git a/scripts/docker/musllinux/install_zstd.sh b/scripts/docker/musllinux/install_zstd.sh new file mode 100755 index 000000000..cdca31446 --- /dev/null +++ b/scripts/docker/musllinux/install_zstd.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# (c) Tuplex team 2017-2023 +# Installs zstd dependencies required to build tuplex on Alpine Linux. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +ZSTD_VERSION=${ZSTD_VERSION:-1.5.6} +CC=${CC:-gcc} +CXX=${CXX:-g++} + +# Get CPU count for parallel builds +CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script with root privileges" + exit 1 +fi + +echo ">> Installing zstd ${ZSTD_VERSION}" + +# Create download directory +DOWNLOAD_DIR=${WORKDIR}/tuplex-downloads +mkdir -p ${DOWNLOAD_DIR}/zstd && cd ${DOWNLOAD_DIR}/zstd \ +&& curl -LO https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz \ +&& tar -xzf zstd-${ZSTD_VERSION}.tar.gz \ +&& rm zstd-${ZSTD_VERSION}.tar.gz \ +&& cd zstd-${ZSTD_VERSION} \ +&& make -j${CPU_COUNT} CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC" \ +&& make install PREFIX=${PREFIX} \ +&& make install PREFIX=${PREFIX} MOREFLAGS="-fPIC" + +echo ">> zstd ${ZSTD_VERSION} installation completed successfully" From 53a16006cf23ddf95f10b3ae78e02ded0eceefae Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 2 Sep 2025 23:22:45 -0700 Subject: [PATCH 17/92] boost update --- scripts/docker/musllinux/install_boost.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/docker/musllinux/install_boost.sh b/scripts/docker/musllinux/install_boost.sh index 88f389718..6750c7458 100755 --- a/scripts/docker/musllinux/install_boost.sh +++ b/scripts/docker/musllinux/install_boost.sh @@ -10,6 +10,10 @@ BOOST_VERSION=${BOOST_VERSION:-1.88.0} CC=${CC:-gcc} CXX=${CXX:-g++} +# Get Python version for Boost installation path +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") +BOOST_INSTALL_DIR="${PREFIX}/boost/python${PYTHON_VERSION}" + # Get CPU count for parallel builds CPU_COUNT=$(( 1 * $( grep '^processor[[:space:]]*:' /proc/cpuinfo | wc -l ) )) @@ -36,10 +40,10 @@ cd "${DOWNLOAD_DIR}/boost" curl -L -O "https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz" tar xf "boost-${BOOST_VERSION}-b2-nodocs.tar.gz" cd "${DOWNLOAD_DIR}/boost/boost-${BOOST_VERSION}" -./bootstrap.sh --with-python="${PYTHON_EXECUTABLE}" --prefix="${PREFIX}" --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" +./bootstrap.sh --with-python="${PYTHON_EXECUTABLE}" --prefix="${BOOST_INSTALL_DIR}" --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" ./b2 cxxflags="-fPIC" link=static -j "${CPU_COUNT}" ./b2 cxxflags="-fPIC" link=static install -sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' "${PREFIX}/include/boost/thread/pthread/thread_data.hpp" +sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' "${BOOST_INSTALL_DIR}/include/boost/thread/pthread/thread_data.hpp" cd "${DOWNLOAD_DIR}" rm -rf "${DOWNLOAD_DIR}/boost" From e87ee2f208daf31029361aeaa2c4ae5691b0f2e9 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 00:30:10 -0700 Subject: [PATCH 18/92] boost fix --- scripts/build_linux_musl_wheels.sh | 2 +- tuplex/CMakeLists.txt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/build_linux_musl_wheels.sh b/scripts/build_linux_musl_wheels.sh index 4c925569d..7575ad280 100755 --- a/scripts/build_linux_musl_wheels.sh +++ b/scripts/build_linux_musl_wheels.sh @@ -142,7 +142,7 @@ fi # add to environment, e.g. TUPLEX_BUILD_TYPE=tsan to force a tsan build. Release is the default mode # Note: The tuplex/musl image uses /opt as the installation prefix for both architectures -export CIBW_ENVIRONMENT="TUPLEX_LAMBDA_ZIP='./tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/opt/lib:/opt/lib64" +export CIBW_ENVIRONMENT="CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/opt/lib:/opt/lib64" # Build only musllinux wheels for Python 3.11+ export CIBW_BUILD="cp3{11,12,13}-*" diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index 7adf4b7ef..87c78e569 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -797,7 +797,10 @@ if(Python3_FOUND) endif() message(STATUS "Found python${Python3_VERSION} - if you'd like to change a to different python version, use -DPython3_ROOT_DIR= or -DPYTHON3_VERSION= or set an environment variable PYTHON3_VERSION") - set(Boost_NO_BOOST_CMAKE ON) # findboost from cmake is buggy and does not work, explicitly disable here + + # For cmake < 4.0, finding boost may be buggy. + # set(Boost_NO_BOOST_CMAKE ON) # findboost from cmake is buggy and does not work, explicitly disable here + if(APPLE AND BREW_FOUND) # i.e. boost-python via brew? --> check maybe better in the future... set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) # gets rid off annoying boost warning. endif() From f7b09b80194e248eb7c2eb74275b31242d4e8b34 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 10:35:38 -0700 Subject: [PATCH 19/92] arch fix --- tuplex/CMakeLists.txt | 51 +++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index 87c78e569..36bd6d2bd 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -388,7 +388,12 @@ endif() # use march=native if available and NOT building wheels # specific Release flags, i.e. targeting loop unrolling. -ucm_add_flags(-m64 -O3 -funroll-loops CONFIG Release) +# Only use -m64 for x86_64 architectures +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + ucm_add_flags(-m64 -O3 -funroll-loops CONFIG Release) +else() + ucm_add_flags(-O3 -funroll-loops CONFIG Release) +endif() # for GCC add stack protector # -fstack-protector-strong @@ -411,30 +416,38 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) # for release mode, don't use Ofast unless you're ok to break NAN/INF compatibility. if(BUILD_FOR_LAMBDA) - ucm_add_flags(-march=haswell) + # Only use x86_64 specific march flags for x86_64 architectures + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + ucm_add_flags(-march=haswell) + endif() endif() # add compiler flags for specific SIMD instructions -include(FindSSE) -include(FindSIMD) -CHECK_FOR_SSE42() -CHECK_FOR_AVX() -# add compiler flags -if(HAVE_SSE42) - message(STATUS "Found SSE4.2 support, adding -msse4.2") - # no msvc support, so use clang/gcc flag approach! - add_compile_options("-msse4.2") -endif() -if(HAVE_AVX) - # only add avx (because not any architecture supports it) if native build is enabled or build for lambda - # because Haswell supports both AVX and AVX2! - if(BUILD_NATIVE OR BUILD_FOR_LAMBDA) - message(STATUS "Found AVX support, adding -mavx") +# Only include SSE/AVX checks for x86_64 architectures +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + include(FindSSE) + include(FindSIMD) + CHECK_FOR_SSE42() + CHECK_FOR_AVX() + # add compiler flags + if(HAVE_SSE42) + message(STATUS "Found SSE4.2 support, adding -msse4.2") # no msvc support, so use clang/gcc flag approach! - add_compile_options("-mavx") + add_compile_options("-msse4.2") + endif() + if(HAVE_AVX) + # only add avx (because not any architecture supports it) if native build is enabled or build for lambda + # because Haswell supports both AVX and AVX2! + if(BUILD_NATIVE OR BUILD_FOR_LAMBDA) + message(STATUS "Found AVX support, adding -mavx") + # no msvc support, so use clang/gcc flag approach! + add_compile_options("-mavx") + endif() + else() + message(STATUS "no avx support, build with -DBUILD_NATIVE=ON to enable") endif() else() - message(STATUS "no avx support, build with -DBUILD_NATIVE=ON to enable") + message(STATUS "Skipping SSE/AVX checks for non-x86_64 architecture: ${CMAKE_SYSTEM_PROCESSOR}") endif() ########################################################################### From 644746cf995d296f4b6d148f397a94654a04f774 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 10:48:55 -0700 Subject: [PATCH 20/92] fix --- tuplex/io/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tuplex/io/CMakeLists.txt b/tuplex/io/CMakeLists.txt index d7ffd541a..6c8255aa6 100644 --- a/tuplex/io/CMakeLists.txt +++ b/tuplex/io/CMakeLists.txt @@ -210,7 +210,7 @@ if(BUILD_WITH_ORC) ExternalProject_Add(orc GIT_REPOSITORY https://github.com/apache/orc.git - GIT_TAG rel/release-2.1.3 + GIT_TAG rel/release-2.2.0 TIMEOUT 5 CMAKE_ARGS -DBUILD_LIBHDFSPP=OFF -DSNAPPY_HOME=${SNAPPY_HOME} -DLZ4_HOME=${LZ4_HOME} -DZSTD_HOME=${ZSTD_HOME} -DZLIB_HOME=${ZLIB_HOME} @@ -218,6 +218,7 @@ if(BUILD_WITH_ORC) -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DSTOP_BUILD_ON_WARNING=OFF -DBUILD_JAVA=OFF -DBUILD_TOOLS=OFF -DBUILD_CPP_TESTS=OFF -DBUILD_POSITION_INDEPENDENT_LIB=ON -DPROTOBUF_HOME=${Protobuf_HOME} + -DPROTOBUF_BUILD_TESTS=OFF -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON PREFIX "${EXTERNAL_INSTALL_LOCATION}" UPDATE_COMMAND "" # Disable update step: clones the project only once BUILD_BYPRODUCTS ${orc_LIBRARY} ${ORC_THIRD_PARTY_LIBS} From 75b166858513af015c48e3e2aa69672c3934dad2 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 10:58:57 -0700 Subject: [PATCH 21/92] config --- tuplex/io/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tuplex/io/CMakeLists.txt b/tuplex/io/CMakeLists.txt index 6c8255aa6..22fc49610 100644 --- a/tuplex/io/CMakeLists.txt +++ b/tuplex/io/CMakeLists.txt @@ -109,7 +109,7 @@ if(BUILD_WITH_ORC) "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep/contrib/cmake_unofficial") endif() - ExternalProject_Add (lz4_ep URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz"s + ExternalProject_Add (lz4_ep URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" ${LZ4_CONFIGURE} BUILD_BYPRODUCTS "${LZ4_STATIC_LIB}") @@ -218,7 +218,6 @@ if(BUILD_WITH_ORC) -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DSTOP_BUILD_ON_WARNING=OFF -DBUILD_JAVA=OFF -DBUILD_TOOLS=OFF -DBUILD_CPP_TESTS=OFF -DBUILD_POSITION_INDEPENDENT_LIB=ON -DPROTOBUF_HOME=${Protobuf_HOME} - -DPROTOBUF_BUILD_TESTS=OFF -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON PREFIX "${EXTERNAL_INSTALL_LOCATION}" UPDATE_COMMAND "" # Disable update step: clones the project only once BUILD_BYPRODUCTS ${orc_LIBRARY} ${ORC_THIRD_PARTY_LIBS} @@ -234,7 +233,6 @@ if(BUILD_WITH_ORC) set_target_properties(liborc PROPERTIES IMPORTED_LOCATION ${orc_LIBRARY}) add_dependencies(liborc orc) - include_directories(${orc_INCLUDE_DIR}) set(ORC_LIBRARIES ${SNAPPY_LIBRARIES} ${LZ4_LIBRARIES} @@ -243,6 +241,7 @@ if(BUILD_WITH_ORC) set(ORC_LIBRARIES ${SNAPPY_LIBRARIES} ${LZ4_LIBRARIES} PARENT_SCOPE) endif() + add_library(libio OBJECT ${CMAKE_CURRENT_BINARY_DIR} ${SOURCES} ${INCLUDES}) set_target_properties(libio PROPERTIES PREFIX "") From 2d9f67270103042f491383dae9fb83df0c4ee225 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 11:08:14 -0700 Subject: [PATCH 22/92] patches --- tuplex/io/CMakeLists.txt | 3 + ...001-fix-protobuf-int64-compatibility.patch | 86 +++++++++++++++++++ tuplex/io/patches/orc/README.md | 21 +++++ 3 files changed, 110 insertions(+) create mode 100644 tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch create mode 100644 tuplex/io/patches/orc/README.md diff --git a/tuplex/io/CMakeLists.txt b/tuplex/io/CMakeLists.txt index 22fc49610..40e65d13a 100644 --- a/tuplex/io/CMakeLists.txt +++ b/tuplex/io/CMakeLists.txt @@ -212,6 +212,9 @@ if(BUILD_WITH_ORC) GIT_REPOSITORY https://github.com/apache/orc.git GIT_TAG rel/release-2.2.0 TIMEOUT 5 + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/orc/0001-fix-protobuf-int64-compatibility.patch ${CMAKE_CURRENT_BINARY_DIR}/orc-patch.patch && + cd ${CMAKE_CURRENT_BINARY_DIR}/orc-prefix/src/orc && + patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/orc-patch.patch CMAKE_ARGS -DBUILD_LIBHDFSPP=OFF -DSNAPPY_HOME=${SNAPPY_HOME} -DLZ4_HOME=${LZ4_HOME} -DZSTD_HOME=${ZSTD_HOME} -DZLIB_HOME=${ZLIB_HOME} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch b/tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch new file mode 100644 index 000000000..26dc61b68 --- /dev/null +++ b/tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch @@ -0,0 +1,86 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Tuplex Team +Date: Mon, 17 Sep 2024 00:00:00 +0000 +Subject: [PATCH] Fix protobuf int64 compatibility for newer protobuf versions for orc 2.2.0 + +This patch replaces the deprecated google::protobuf::int64 with int64_t +and adds the necessary #include to maintain compatibility +with newer protobuf versions that removed the google::protobuf::int64 type. + +--- + c++/src/io/InputStream.hh | 2 ++ + c++/src/io/InputStream.cc | 2 +- + c++/src/io/OutputStream.hh | 2 ++ + c++/src/io/OutputStream.cc | 2 +- + 4 files changed, 5 insertions(+), 3 deletions(-) + +diff --git a/c++/src/io/InputStream.hh b/c++/src/io/InputStream.hh +index 0000000..0000000 100644 +--- a/c++/src/io/InputStream.hh ++++ b/c++/src/io/InputStream.hh +@@ -22,6 +22,7 @@ + #ifndef ORC_INPUTSTREAM_HH + #define ORC_INPUTSTREAM_HH + ++#include + #include "Adaptor.hh" + #include "orc/OrcFile.hh" + #include "wrap/zero-copy-stream-wrapper.h" +@@ -75,7 +76,7 @@ namespace orc { + virtual void BackUp(int count) override; + virtual bool Skip(int count) override; +- virtual google::protobuf::int64 ByteCount() const override; ++ virtual int64_t ByteCount() const override; + virtual void seek(PositionProvider& position) override; + virtual std::string getName() const override; + +diff --git a/c++/src/io/InputStream.cc b/c++/src/io/InputStream.cc +index 0000000..0000000 100644 +--- a/c++/src/io/InputStream.cc ++++ b/c++/src/io/InputStream.cc +@@ -115,8 +115,8 @@ namespace orc { + } + +- google::protobuf::int64 SeekableArrayInputStream::ByteCount() const { +- return static_cast(position_); ++ int64_t SeekableArrayInputStream::ByteCount() const { ++ return static_cast(position_); + } + + void SeekableArrayInputStream::seek(PositionProvider& position) { +diff --git a/c++/src/io/OutputStream.hh b/c++/src/io/OutputStream.hh +index 0000000..0000000 100644 +--- a/c++/src/io/OutputStream.hh ++++ b/c++/src/io/OutputStream.hh +@@ -22,6 +22,7 @@ + #ifndef ORC_OUTPUTSTREAM_HH + #define ORC_OUTPUTSTREAM_HH + ++#include + #include "Adaptor.hh" + #include "BlockBuffer.hh" + #include "orc/OrcFile.hh" +@@ -64,7 +65,7 @@ namespace orc { + virtual bool Next(void** data, int* size) override; + virtual void BackUp(int count) override; +- virtual google::protobuf::int64 ByteCount() const override; ++ virtual int64_t ByteCount() const override; + virtual bool WriteAliasedRaw(const void* data, int size) override; + virtual bool AllowsAliasing() const override; + +diff --git a/c++/src/io/OutputStream.cc b/c++/src/io/OutputStream.cc +index 0000000..0000000 100644 +--- a/c++/src/io/OutputStream.cc ++++ b/c++/src/io/OutputStream.cc +@@ -68,8 +68,8 @@ namespace orc { + } + +- google::protobuf::int64 BufferedOutputStream::ByteCount() const { +- return static_cast(dataBuffer_->size()); ++ int64_t BufferedOutputStream::ByteCount() const { ++ return static_cast(dataBuffer_->size()); + } + + void BufferedOutputStream::BackUp(int count) { +-- +2.39.2 diff --git a/tuplex/io/patches/orc/README.md b/tuplex/io/patches/orc/README.md new file mode 100644 index 000000000..83060d57f --- /dev/null +++ b/tuplex/io/patches/orc/README.md @@ -0,0 +1,21 @@ +# ORC Patches + +This directory contains patches for the Apache ORC library to fix compatibility issues. + +## 0001-fix-protobuf-int64-compatibility.patch + +**Purpose**: Fixes protobuf compatibility issues when building ORC with newer protobuf versions (3.0+) + +**Problem**: Newer protobuf versions removed the deprecated `google::protobuf::int64` type, causing compilation errors in ORC. + +**Solution**: Replaces all occurrences of `google::protobuf::int64` with the standard `int64_t` type and adds `#include ` to the necessary header files. + +**Files Modified**: +- `c++/src/io/InputStream.hh` - Added `#include ` and updated method signature +- `c++/src/io/InputStream.cc` - Updated implementation to use `int64_t` +- `c++/src/io/OutputStream.hh` - Added `#include ` and updated method signature +- `c++/src/io/OutputStream.cc` - Updated implementation to use `int64_t` + +**Applied**: Automatically applied during CMake build via `PATCH_COMMAND` in the ORC ExternalProject_Add configuration. + +**Compatibility**: This patch allows ORC to compile successfully with protobuf 3.0+ while maintaining backward compatibility. From f9fd07d1952adf4fea0933bcbebfd59faccb5435 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 11:09:49 -0700 Subject: [PATCH 23/92] path fix --- tuplex/io/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuplex/io/CMakeLists.txt b/tuplex/io/CMakeLists.txt index 40e65d13a..432b3c632 100644 --- a/tuplex/io/CMakeLists.txt +++ b/tuplex/io/CMakeLists.txt @@ -213,7 +213,7 @@ if(BUILD_WITH_ORC) GIT_TAG rel/release-2.2.0 TIMEOUT 5 PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/orc/0001-fix-protobuf-int64-compatibility.patch ${CMAKE_CURRENT_BINARY_DIR}/orc-patch.patch && - cd ${CMAKE_CURRENT_BINARY_DIR}/orc-prefix/src/orc && + cd ${EXTERNAL_INSTALL_LOCATION}/src/orc && patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/orc-patch.patch CMAKE_ARGS -DBUILD_LIBHDFSPP=OFF -DSNAPPY_HOME=${SNAPPY_HOME} -DLZ4_HOME=${LZ4_HOME} -DZSTD_HOME=${ZSTD_HOME} -DZLIB_HOME=${ZLIB_HOME} From 028e92ceaa4b372399f956705ad5b735a2bc14f8 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 11:24:47 -0700 Subject: [PATCH 24/92] patch --- tuplex/io/CMakeLists.txt | 6 +- ...buf-int64-compatibility.patch => orc.diff} | 106 ++++++++---------- tuplex/io/patches/orc/README.md | 21 ---- 3 files changed, 50 insertions(+), 83 deletions(-) rename tuplex/io/patches/{orc/0001-fix-protobuf-int64-compatibility.patch => orc.diff} (58%) delete mode 100644 tuplex/io/patches/orc/README.md diff --git a/tuplex/io/CMakeLists.txt b/tuplex/io/CMakeLists.txt index 432b3c632..c22d02395 100644 --- a/tuplex/io/CMakeLists.txt +++ b/tuplex/io/CMakeLists.txt @@ -212,9 +212,9 @@ if(BUILD_WITH_ORC) GIT_REPOSITORY https://github.com/apache/orc.git GIT_TAG rel/release-2.2.0 TIMEOUT 5 - PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/orc/0001-fix-protobuf-int64-compatibility.patch ${CMAKE_CURRENT_BINARY_DIR}/orc-patch.patch && - cd ${EXTERNAL_INSTALL_LOCATION}/src/orc && - patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/orc-patch.patch + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/orc.diff ${CMAKE_BINARY_DIR}/orc-patch.diff && + cd ${CMAKE_BINARY_DIR}/third_party/src/orc && + patch -p1 < ${CMAKE_BINARY_DIR}/orc-patch.diff CMAKE_ARGS -DBUILD_LIBHDFSPP=OFF -DSNAPPY_HOME=${SNAPPY_HOME} -DLZ4_HOME=${LZ4_HOME} -DZSTD_HOME=${ZSTD_HOME} -DZLIB_HOME=${ZLIB_HOME} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch b/tuplex/io/patches/orc.diff similarity index 58% rename from tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch rename to tuplex/io/patches/orc.diff index 26dc61b68..778f63184 100644 --- a/tuplex/io/patches/orc/0001-fix-protobuf-int64-compatibility.patch +++ b/tuplex/io/patches/orc.diff @@ -1,44 +1,9 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Tuplex Team -Date: Mon, 17 Sep 2024 00:00:00 +0000 -Subject: [PATCH] Fix protobuf int64 compatibility for newer protobuf versions for orc 2.2.0 - -This patch replaces the deprecated google::protobuf::int64 with int64_t -and adds the necessary #include to maintain compatibility -with newer protobuf versions that removed the google::protobuf::int64 type. - ---- - c++/src/io/InputStream.hh | 2 ++ - c++/src/io/InputStream.cc | 2 +- - c++/src/io/OutputStream.hh | 2 ++ - c++/src/io/OutputStream.cc | 2 +- - 4 files changed, 5 insertions(+), 3 deletions(-) - -diff --git a/c++/src/io/InputStream.hh b/c++/src/io/InputStream.hh -index 0000000..0000000 100644 ---- a/c++/src/io/InputStream.hh -+++ b/c++/src/io/InputStream.hh -@@ -22,6 +22,7 @@ - #ifndef ORC_INPUTSTREAM_HH - #define ORC_INPUTSTREAM_HH - -+#include - #include "Adaptor.hh" - #include "orc/OrcFile.hh" - #include "wrap/zero-copy-stream-wrapper.h" -@@ -75,7 +76,7 @@ namespace orc { - virtual void BackUp(int count) override; - virtual bool Skip(int count) override; -- virtual google::protobuf::int64 ByteCount() const override; -+ virtual int64_t ByteCount() const override; - virtual void seek(PositionProvider& position) override; - virtual std::string getName() const override; - diff --git a/c++/src/io/InputStream.cc b/c++/src/io/InputStream.cc -index 0000000..0000000 100644 +index 06ef40b..5e1dc00 100644 --- a/c++/src/io/InputStream.cc +++ b/c++/src/io/InputStream.cc -@@ -115,8 +115,8 @@ namespace orc { +@@ -112,8 +112,8 @@ namespace orc { + return false; } - google::protobuf::int64 SeekableArrayInputStream::ByteCount() const { @@ -47,32 +12,35 @@ index 0000000..0000000 100644 + return static_cast(position_); } - void SeekableArrayInputStream::seek(PositionProvider& position) { -diff --git a/c++/src/io/OutputStream.hh b/c++/src/io/OutputStream.hh -index 0000000..0000000 100644 ---- a/c++/src/io/OutputStream.hh -+++ b/c++/src/io/OutputStream.hh -@@ -22,6 +22,7 @@ - #ifndef ORC_OUTPUTSTREAM_HH - #define ORC_OUTPUTSTREAM_HH + void SeekableArrayInputStream::seek(PositionProvider& seekPosition) { +diff --git a/c++/src/io/InputStream.hh b/c++/src/io/InputStream.hh +index 07aa623..5f526a2 100644 +--- a/c++/src/io/InputStream.hh ++++ b/c++/src/io/InputStream.hh +@@ -29,6 +29,8 @@ + #include + #include +#include - #include "Adaptor.hh" - #include "BlockBuffer.hh" - #include "orc/OrcFile.hh" -@@ -64,7 +65,7 @@ namespace orc { - virtual bool Next(void** data, int* size) override; ++ + namespace orc { + + void printBuffer(std::ostream& out, const char* buffer, uint64_t length); +@@ -72,7 +74,7 @@ namespace orc { + virtual bool Next(const void** data, int* size) override; virtual void BackUp(int count) override; + virtual bool Skip(int count) override; - virtual google::protobuf::int64 ByteCount() const override; + virtual int64_t ByteCount() const override; - virtual bool WriteAliasedRaw(const void* data, int size) override; - virtual bool AllowsAliasing() const override; - + virtual void seek(PositionProvider& position) override; + virtual std::string getName() const override; + }; diff --git a/c++/src/io/OutputStream.cc b/c++/src/io/OutputStream.cc -index 0000000..0000000 100644 +index fbf1ca6..a55050d 100644 --- a/c++/src/io/OutputStream.cc +++ b/c++/src/io/OutputStream.cc -@@ -68,8 +68,8 @@ namespace orc { +@@ -65,8 +65,8 @@ namespace orc { + // PASS } - google::protobuf::int64 BufferedOutputStream::ByteCount() const { @@ -81,6 +49,26 @@ index 0000000..0000000 100644 + return static_cast(dataBuffer_->size()); } - void BufferedOutputStream::BackUp(int count) { --- -2.39.2 + bool BufferedOutputStream::WriteAliasedRaw(const void*, int) { +diff --git a/c++/src/io/OutputStream.hh b/c++/src/io/OutputStream.hh +index 6319de9..51e5fc4 100644 +--- a/c++/src/io/OutputStream.hh ++++ b/c++/src/io/OutputStream.hh +@@ -24,6 +24,8 @@ + #include "orc/OrcFile.hh" + #include "wrap/zero-copy-stream-wrapper.h" + ++#include ++ + namespace orc { + + /** +@@ -61,7 +63,7 @@ namespace orc { + + virtual bool Next(void** data, int* size) override; + virtual void BackUp(int count) override; +- virtual google::protobuf::int64 ByteCount() const override; ++ virtual int64_t ByteCount() const override; + virtual bool WriteAliasedRaw(const void* data, int size) override; + virtual bool AllowsAliasing() const override; + diff --git a/tuplex/io/patches/orc/README.md b/tuplex/io/patches/orc/README.md deleted file mode 100644 index 83060d57f..000000000 --- a/tuplex/io/patches/orc/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# ORC Patches - -This directory contains patches for the Apache ORC library to fix compatibility issues. - -## 0001-fix-protobuf-int64-compatibility.patch - -**Purpose**: Fixes protobuf compatibility issues when building ORC with newer protobuf versions (3.0+) - -**Problem**: Newer protobuf versions removed the deprecated `google::protobuf::int64` type, causing compilation errors in ORC. - -**Solution**: Replaces all occurrences of `google::protobuf::int64` with the standard `int64_t` type and adds `#include ` to the necessary header files. - -**Files Modified**: -- `c++/src/io/InputStream.hh` - Added `#include ` and updated method signature -- `c++/src/io/InputStream.cc` - Updated implementation to use `int64_t` -- `c++/src/io/OutputStream.hh` - Added `#include ` and updated method signature -- `c++/src/io/OutputStream.cc` - Updated implementation to use `int64_t` - -**Applied**: Automatically applied during CMake build via `PATCH_COMMAND` in the ORC ExternalProject_Add configuration. - -**Compatibility**: This patch allows ORC to compile successfully with protobuf 3.0+ while maintaining backward compatibility. From 412edf49eb2c156afca03c50305cf7854064aa3a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 11:27:57 -0700 Subject: [PATCH 25/92] arch fix --- tuplex/utils/include/Base.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuplex/utils/include/Base.h b/tuplex/utils/include/Base.h index da3db47b7..0aeaf82ba 100644 --- a/tuplex/utils/include/Base.h +++ b/tuplex/utils/include/Base.h @@ -58,7 +58,7 @@ #endif #endif #if __GNUC__ -#if __x86_64__ || __ppc64__ || __arm64__ +#if __x86_64__ || __ppc64__ || __aarch64__ #define ENV64BIT #else #define ENV32BIT From cb04e96f706d18508c3ba82875ccd10a6d00c935 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 11:29:31 -0700 Subject: [PATCH 26/92] aarch64 fix --- tuplex/utils/include/Utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuplex/utils/include/Utils.h b/tuplex/utils/include/Utils.h index 87f05ca57..47f73ce07 100644 --- a/tuplex/utils/include/Utils.h +++ b/tuplex/utils/include/Utils.h @@ -43,7 +43,7 @@ namespace std { #if (defined __x86_64__) #include "third_party/levenshtein-sse.h" -#elif (defined __arm64__) +#elif (defined __aarch64__) namespace tuplex { // native C++ implementation for now (ARM) inline size_t levenshtein(const std::string& word1, const std::string& word2) { From fa770b4bf72fdcaad2a79b39dbd59e69342dca40 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 11:32:19 -0700 Subject: [PATCH 27/92] better 64bit aarch64 detect --- tuplex/core/src/physical/TextReader.cc | 2 +- tuplex/utils/include/Base.h | 2 +- tuplex/utils/include/Utils.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tuplex/core/src/physical/TextReader.cc b/tuplex/core/src/physical/TextReader.cc index 32d0da0ad..1c5f59f27 100644 --- a/tuplex/core/src/physical/TextReader.cc +++ b/tuplex/core/src/physical/TextReader.cc @@ -17,7 +17,7 @@ // use simd intrinsics or ARM Neon translation layer #if (defined __x86_64__) #include -#elif (defined __arm64__) +#elif (defined(__aarch64__) || defined(__arm64__)) #include #else #error "unsupported platform for intrinsics" diff --git a/tuplex/utils/include/Base.h b/tuplex/utils/include/Base.h index 0aeaf82ba..1fc53737e 100644 --- a/tuplex/utils/include/Base.h +++ b/tuplex/utils/include/Base.h @@ -58,7 +58,7 @@ #endif #endif #if __GNUC__ -#if __x86_64__ || __ppc64__ || __aarch64__ +#if __x86_64__ || __ppc64__ || __aarch64__ || __arm64__ #define ENV64BIT #else #define ENV32BIT diff --git a/tuplex/utils/include/Utils.h b/tuplex/utils/include/Utils.h index 47f73ce07..8e713e83d 100644 --- a/tuplex/utils/include/Utils.h +++ b/tuplex/utils/include/Utils.h @@ -43,7 +43,7 @@ namespace std { #if (defined __x86_64__) #include "third_party/levenshtein-sse.h" -#elif (defined __aarch64__) +#elif (defined(__aarch64__) || defined(__arm64__)) namespace tuplex { // native C++ implementation for now (ARM) inline size_t levenshtein(const std::string& word1, const std::string& word2) { From 4a604637bf43733993c375ad274129ed1011c03f Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 12:14:53 -0700 Subject: [PATCH 28/92] upgrade --- tuplex/utils/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index 472c02e86..bb45a49c2 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -10,7 +10,7 @@ include(ExternalProject) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) ExternalProject_Add(fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git - GIT_TAG 6.2.0 + GIT_TAG 11.2.0 GIT_CONFIG advice.detachedHead=false TIMEOUT 5 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} @@ -27,7 +27,7 @@ set(fmt_INCLUDE_DIR ${source_dir}/include) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) ExternalProject_Add(spdlog GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.3.1 + GIT_TAG v1.15.3 GIT_CONFIG advice.detachedHead=false TIMEOUT 5 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} #-DSPDLOG_BUILD_BENCH=OFF From af0896c3ff18c6adfdde5b98c64beb295b3970b8 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 12:15:23 -0700 Subject: [PATCH 29/92] experiment update --- tuplex/utils/CMakeLists.txt | 2 +- tuplex/utils/src/Logger.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index 472c02e86..ddd009c49 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -27,7 +27,7 @@ set(fmt_INCLUDE_DIR ${source_dir}/include) set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) ExternalProject_Add(spdlog GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.3.1 + GIT_TAG v1.15.3 GIT_CONFIG advice.detachedHead=false TIMEOUT 5 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} #-DSPDLOG_BUILD_BENCH=OFF diff --git a/tuplex/utils/src/Logger.cc b/tuplex/utils/src/Logger.cc index 4ce9bc62b..1959ed10b 100644 --- a/tuplex/utils/src/Logger.cc +++ b/tuplex/utils/src/Logger.cc @@ -72,7 +72,7 @@ MessageHandler& Logger::logger(const std::string &name) { #ifndef NDEBUG spdlogger->set_level(spdlog::level::debug); #endif - spdlog::register_logger(spdlogger); + spdlog::register_or_replace(spdlogger); return _handlers[name]; } From 422e8093d2ed984d0e88a93d45339267edbaa68e Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 12:34:41 -0700 Subject: [PATCH 30/92] fmt update --- tuplex/runtime/src/Runtime.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tuplex/runtime/src/Runtime.cc b/tuplex/runtime/src/Runtime.cc index 426af001a..3b32f407b 100644 --- a/tuplex/runtime/src/Runtime.cc +++ b/tuplex/runtime/src/Runtime.cc @@ -18,6 +18,7 @@ #include // <-- implemented in StringUtils #include #include +#include #include #include #include From 2bc60bc040f5088cda65b5d9bdacfecc6a0da8dd Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 12:43:31 -0700 Subject: [PATCH 31/92] fmtlib EOF patch --- tuplex/utils/CMakeLists.txt | 3 +++ tuplex/utils/patches/fmtlib.diff | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tuplex/utils/patches/fmtlib.diff diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index bb45a49c2..8181180e7 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -13,6 +13,9 @@ ExternalProject_Add(fmt GIT_TAG 11.2.0 GIT_CONFIG advice.detachedHead=false TIMEOUT 5 + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/patches/fmtlib.diff ${CMAKE_BINARY_DIR}/fmtlib-patch.diff && + cd ${CMAKE_BINARY_DIR}/third_party/src/fmt && + patch -p1 < ${CMAKE_BINARY_DIR}/fmtlib-patch.diff CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} PREFIX "${EXTERNAL_INSTALL_LOCATION}" CONFIGURE_COMMAND "" # Disable configure step diff --git a/tuplex/utils/patches/fmtlib.diff b/tuplex/utils/patches/fmtlib.diff new file mode 100644 index 000000000..0f80612c2 --- /dev/null +++ b/tuplex/utils/patches/fmtlib.diff @@ -0,0 +1,17 @@ +diff --git a/include/fmt/format-inl.h b/include/fmt/format-inl.h +index a1e0166..8ab753a 100644 +--- a/include/fmt/format-inl.h ++++ b/include/fmt/format-inl.h +@@ -1476,6 +1476,12 @@ template auto getc_unlocked(F* f) -> decltype(_fgetc_nolock(f)) { + } + #endif + ++#ifndef EOF ++// Define via C++. ++#include ++#define EOF (std::char_traits::eof()) ++#endif ++ + template + struct has_flockfile : std::false_type {}; + From 2864ea18efebebe01a623546bead2704757ba7cf Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 12:48:23 -0700 Subject: [PATCH 32/92] fmt update fix for new string view --- tuplex/python/include/PythonCommon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuplex/python/include/PythonCommon.h b/tuplex/python/include/PythonCommon.h index 205f2b63d..675d78ad0 100644 --- a/tuplex/python/include/PythonCommon.h +++ b/tuplex/python/include/PythonCommon.h @@ -120,7 +120,7 @@ namespace tuplex { LogMessage msg; msg.message = std::string(spdlog_msg.payload.data()); msg.timestamp = spdlog_msg.time; - msg.logger = *spdlog_msg.logger_name; + msg.logger = std::string(spdlog_msg.logger_name.begin(), spdlog_msg.logger_name.end()); msg.level = spdlog_msg.level; _messageBuffer.push_back(msg); } From ccebfffaeeac6a056f870879bd69bd242eaa82ab Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 3 Sep 2025 12:50:51 -0700 Subject: [PATCH 33/92] va_arg update --- tuplex/runtime/src/Runtime.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tuplex/runtime/src/Runtime.cc b/tuplex/runtime/src/Runtime.cc index 3b32f407b..f84d24b7d 100644 --- a/tuplex/runtime/src/Runtime.cc +++ b/tuplex/runtime/src/Runtime.cc @@ -24,6 +24,7 @@ #include #include #include // for aligned_malloc +#include // for va_arg #define RUNTIME_DEFAULT_BLOCK_SIZE (4 * 1024 * 1024) From 9ae4fc8fc1a6bce732d1544ef6d97ebec4812a60 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 12:37:30 -0700 Subject: [PATCH 34/92] remove boost::system lib --- tuplex/core/CMakeLists.txt | 1 - tuplex/python/CMakeLists.txt | 2 -- tuplex/utils/CMakeLists.txt | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tuplex/core/CMakeLists.txt b/tuplex/core/CMakeLists.txt index 031cb9fe7..9fb589a4c 100755 --- a/tuplex/core/CMakeLists.txt +++ b/tuplex/core/CMakeLists.txt @@ -119,7 +119,6 @@ target_link_libraries(libcore protobuf::libprotobuf Boost::iostreams Boost::thread - Boost::system Boost::filesystem util ${ZLIB_LIBRARIES} diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index abb2bcaf3..7bcb806be 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -72,7 +72,6 @@ target_link_libraries(${MODULE_NAME} PUBLIC Backward::Interface libio Boost::iostreams Boost::thread - Boost::system Boost::filesystem) else() target_link_libraries(${MODULE_NAME} @@ -84,7 +83,6 @@ else() libio Boost::iostreams Boost::thread - Boost::system Boost::filesystem) endif() diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index 8181180e7..cbecee976 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -109,4 +109,4 @@ target_include_directories(libutils PUBLIC ${AWSSDK_INCLUDE_DIR}) # Specify here the libraries this program depends on -target_link_libraries(libutils Boost::filesystem Boost::thread Boost::system Boost::system Boost::iostreams ${AWSSDK_LINK_LIBRARIES}) +target_link_libraries(libutils Boost::filesystem Boost::thread Boost::iostreams ${AWSSDK_LINK_LIBRARIES}) From aba4a35d618ca39acdac022794471bfc71aa1ee9 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 12:38:56 -0700 Subject: [PATCH 35/92] cleanup --- scripts/macos/user-config.jam | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 scripts/macos/user-config.jam diff --git a/scripts/macos/user-config.jam b/scripts/macos/user-config.jam deleted file mode 100644 index c6f2a5431..000000000 --- a/scripts/macos/user-config.jam +++ /dev/null @@ -1,4 +0,0 @@ -using python : 3.6 : /Library/Frameworks/Python.framework/Versions/3.6/bin/python3.6 : /Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/ : /Library/Frameworks/Python.framework/Versions/3.6/lib ; -using python : 3.7 : /Library/Frameworks/Python.framework/Versions/3.7/bin/python3.7 : /Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/ : /Library/Frameworks/Python.framework/Versions/3.7/lib ; -using python : 3.8 : /Library/Frameworks/Python.framework/Versions/3.8/bin/python3.8 : /Library/Frameworks/Python.framework/Versions/3.8/include/python3.8/ : /Library/Frameworks/Python.framework/Versions/3.8/lib ; -using python : 3.9 : /Library/Frameworks/Python.framework/Versions/3.9/bin/python3.9 : /Library/Frameworks/Python.framework/Versions/3.9/include/python3.9/ : /Library/Frameworks/Python.framework/Versions/3.9/lib ; From 6b24a6205356d096c2ed3151690abf8ab798524d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 12:58:56 -0700 Subject: [PATCH 36/92] fixes --- tuplex/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index 36bd6d2bd..fa7d4e5f2 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -798,7 +798,7 @@ execute_process (COMMAND "${Python3_EXECUTABLE}" -c "from distutils import sysco message(STATUS "Python3 extension module suffix is: ${PYTHON_MODULE_EXTENSION}") set(Boost_NO_WARN_NEW_VERSIONS ON) # cmake 3.20 finally has a switch to kill the boost warnings... -set(BOOST_COMPONENTS iostreams thread system filesystem) +set(BOOST_COMPONENTS iostreams thread filesystem) if(Python3_FOUND) if(Python3_VERSION_MAJOR LESS 3 OR Python3_VERSION_MINOR LESS 5) message(FATAL_ERROR "Tuplex requires python3.5 at least, found incompatible python ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}") From d9a0fd401548706218cbb520567b735edddb2768 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 13:03:33 -0700 Subject: [PATCH 37/92] cmake setup fixes --- tuplex/core/CMakeLists.txt | 12 +++++++++--- tuplex/test/io/CMakeLists.txt | 13 +++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tuplex/core/CMakeLists.txt b/tuplex/core/CMakeLists.txt index 9fb589a4c..9ea57b0eb 100755 --- a/tuplex/core/CMakeLists.txt +++ b/tuplex/core/CMakeLists.txt @@ -114,9 +114,6 @@ target_link_libraries(libcore ${YAMLCPP_LIBRARY} ${CURL_LIBRARIES} ${AWSSDK_LINK_LIBRARIES} - ${Protobuf_LIBRARIES} - proto-objects - protobuf::libprotobuf Boost::iostreams Boost::thread Boost::filesystem @@ -124,3 +121,12 @@ target_link_libraries(libcore ${ZLIB_LIBRARIES} ${ZSTD_LIBRARIES} ) + +# Add protobuf dependencies only if required. +if(BUILD_WITH_AWS OR BUILD_WITH_ORC) + target_link_libraries(libcore + ${Protobuf_LIBRARIES} + proto-objects + protobuf::libprotobuf + ) +endif() diff --git a/tuplex/test/io/CMakeLists.txt b/tuplex/test/io/CMakeLists.txt index be92a0159..d09b0a93e 100644 --- a/tuplex/test/io/CMakeLists.txt +++ b/tuplex/test/io/CMakeLists.txt @@ -18,11 +18,16 @@ TARGET_LINK_LIBRARIES(testio ${GTest_LIBRARIES} ${AWSSDK_LINK_LIBRARIES} ${Python3_LIBRARIES} - ${Protobuf_LIBRARIES} - ${Protobuf_LITE_LIBRARIES} - ${Protobuf_LIBRARIES} ${CURSES_LIBRARY} - protobuf::libprotobuf ) +# Add protobuf dependencies only if required +if(BUILD_WITH_AWS OR BUILD_WITH_ORC) + TARGET_LINK_LIBRARIES(testio + ${Protobuf_LIBRARIES} + ${Protobuf_LITE_LIBRARIES} + protobuf::libprotobuf + ) +endif() + gtest_add_tests(TARGET testio TEST_PREFIX "") From 2e744d0e91ca5ea6ea8e3a08e503007f1c7cc75d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 13:14:11 -0700 Subject: [PATCH 38/92] cmake fixes --- tuplex/CMakeLists.txt | 7 +++++++ tuplex/cmake/ExternalAntlr4Cpp.cmake | 2 +- tuplex/python/CMakeLists.txt | 8 ++------ tuplex/test/adapters/cpython/CMakeLists.txt | 2 +- tuplex/test/codegen/CMakeLists.txt | 2 +- tuplex/test/core/CMakeLists.txt | 2 +- tuplex/test/io/CMakeLists.txt | 2 +- tuplex/test/runtime/CMakeLists.txt | 2 +- tuplex/test/utils/CMakeLists.txt | 2 +- tuplex/test/wrappers/CMakeLists.txt | 2 +- tuplex/utils/CMakeLists.txt | 9 +++------ 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index fa7d4e5f2..f275af89d 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -1,6 +1,11 @@ # (c) 2017-2023 Leonhard Spiegelberg cmake_minimum_required(VERSION 3.16 FATAL_ERROR) +# Set policies to suppress warnings +if(POLICY CMP0144) + cmake_policy(SET CMP0144 NEW) # FindPython3: Honor Python3_FIND_REGISTRY +endif() + # top-level language specification # enable c++17 set(CMAKE_CXX_STANDARD 17) @@ -600,6 +605,8 @@ if(PYTHON3_EXECUTABLE) set(Python3_EXECUTABLE ${Python3_EXECUTABLE}) message(STATUS "Using specific python executable ${PYTHON3_EXECUTABLE}") unset(PYTHON3_VERSION) + # Unset the legacy Python_EXECUTABLE variable to avoid CMake warnings + unset(Python_EXECUTABLE CACHE) # get version from executable execute_process (COMMAND "${PYTHON3_EXECUTABLE}" -c "import sys;print('{}.{}.{}'.format(sys.version_info.major,sys.version_info.minor,sys.version_info.micro))" diff --git a/tuplex/cmake/ExternalAntlr4Cpp.cmake b/tuplex/cmake/ExternalAntlr4Cpp.cmake index e4b8d1ae4..3ecf56827 100755 --- a/tuplex/cmake/ExternalAntlr4Cpp.cmake +++ b/tuplex/cmake/ExternalAntlr4Cpp.cmake @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR) if(${CMAKE_VERSION} VERSION_GREATER "3.19.0") - cmake_policy(SET CMP0114 OLD) # policy update in 3.19, explicitly use old behavior + cmake_policy(SET CMP0114 NEW) # policy update in 3.19, use new behavior endif() include(ExternalProject) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 7bcb806be..5b82be7f5 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -18,12 +18,8 @@ message(STATUS "Pybind11 uses python version ${Python3_VERSION}") set(PYBIND11_FINDPYTHON OFF CACHE INTERNAL "") set(PYBIND11_PYTHON_VERSION "${Python3_VERSION}" CACHE INTERNAL "") FetchContent_Declare(pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v2.13.6) -FetchContent_GetProperties(pybind11) -if(NOT pybind11_POPULATED) - FetchContent_Populate(pybind11) - add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) -endif() + GIT_TAG v3.0.1) +FetchContent_MakeAvailable(pybind11) # Source code & linking file(GLOB_RECURSE SOURCES src/*.cc) diff --git a/tuplex/test/adapters/cpython/CMakeLists.txt b/tuplex/test/adapters/cpython/CMakeLists.txt index 0f518fe41..c6ff7aa34 100644 --- a/tuplex/test/adapters/cpython/CMakeLists.txt +++ b/tuplex/test/adapters/cpython/CMakeLists.txt @@ -30,4 +30,4 @@ TARGET_LINK_LIBRARIES(testcpythonadapter ${CURSES_LIBRARY} ) -gtest_add_tests(TARGET testcpythonadapter TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testcpythonadapter) \ No newline at end of file diff --git a/tuplex/test/codegen/CMakeLists.txt b/tuplex/test/codegen/CMakeLists.txt index 10fe5cd7d..a1c201f1b 100755 --- a/tuplex/test/codegen/CMakeLists.txt +++ b/tuplex/test/codegen/CMakeLists.txt @@ -23,4 +23,4 @@ TARGET_LINK_LIBRARIES(testcodegen runtime ) -gtest_add_tests(TARGET testcodegen TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testcodegen) \ No newline at end of file diff --git a/tuplex/test/core/CMakeLists.txt b/tuplex/test/core/CMakeLists.txt index 97d7b92d6..8451ffdd0 100644 --- a/tuplex/test/core/CMakeLists.txt +++ b/tuplex/test/core/CMakeLists.txt @@ -21,4 +21,4 @@ target_link_libraries(testcore ${CURSES_LIBRARY} ${LLVM_LIBRARIES}) -gtest_add_tests(TARGET testcore TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testcore) \ No newline at end of file diff --git a/tuplex/test/io/CMakeLists.txt b/tuplex/test/io/CMakeLists.txt index d09b0a93e..13c7fcc12 100644 --- a/tuplex/test/io/CMakeLists.txt +++ b/tuplex/test/io/CMakeLists.txt @@ -30,4 +30,4 @@ if(BUILD_WITH_AWS OR BUILD_WITH_ORC) ) endif() -gtest_add_tests(TARGET testio TEST_PREFIX "") +gtest_add_tests(TARGET testio) diff --git a/tuplex/test/runtime/CMakeLists.txt b/tuplex/test/runtime/CMakeLists.txt index cff9af37d..3ecce7c64 100755 --- a/tuplex/test/runtime/CMakeLists.txt +++ b/tuplex/test/runtime/CMakeLists.txt @@ -24,4 +24,4 @@ TARGET_LINK_LIBRARIES(testruntime ${CURSES_LIBRARY} ) -gtest_add_tests(TARGET testruntime TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testruntime) \ No newline at end of file diff --git a/tuplex/test/utils/CMakeLists.txt b/tuplex/test/utils/CMakeLists.txt index c2956ce5b..8be463543 100644 --- a/tuplex/test/utils/CMakeLists.txt +++ b/tuplex/test/utils/CMakeLists.txt @@ -15,4 +15,4 @@ TARGET_LINK_LIBRARIES(testutils ${AWSSDK_LINK_LIBRARIES} ) -gtest_add_tests(TARGET testutils TEST_PREFIX "") \ No newline at end of file +gtest_add_tests(TARGET testutils) \ No newline at end of file diff --git a/tuplex/test/wrappers/CMakeLists.txt b/tuplex/test/wrappers/CMakeLists.txt index 84b753641..1db010855 100644 --- a/tuplex/test/wrappers/CMakeLists.txt +++ b/tuplex/test/wrappers/CMakeLists.txt @@ -46,4 +46,4 @@ else() ) endif() -gtest_add_tests(TARGET testwrappers TEST_PREFIX "") +gtest_add_tests(TARGET testwrappers) diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index cbecee976..4b38baef0 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -55,13 +55,10 @@ if(NOT BUILD_WITH_AWS) GIT_TAG v1.7.14 GIT_CONFIG advice.detachedHead=false ) - FetchContent_GetProperties(cJSON) - if(NOT cjson_Populated) - FetchContent_Populate(cJSON) - message(STATUS "cJSON Source Dir: ${cjson_SOURCE_DIR}") - message(STATUS "cJSON Binary Dir: ${cjson_BINARY_DIR}") + FetchContent_MakeAvailable(cJSON) + message(STATUS "cJSON Source Dir: ${cjson_SOURCE_DIR}") + message(STATUS "cJSON Binary Dir: ${cjson_BINARY_DIR}") # add_subdirectory(${cjson_SOURCE_DIR} ${cjson_BINARY_DIR}) - endif() else() message(STATUS "Using cJSON shipped with AWSSDK") # Note: alternative is to use CJSON_HIDE_SYMBOLS IWTH AWS SDK. From fd9721d651e7a1c9e95028069b2f2c6e3c9f031b Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 14:36:48 -0700 Subject: [PATCH 39/92] try gh actions --- .github/workflows/build_wheels.yml | 4 ++-- tuplex/CMakeLists.txt | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f4dc95937..840dcfd89 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -126,7 +126,7 @@ jobs: CIBW_BUILD: ${{ matrix.cibw-build }} # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile + CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile && echo "Protobuf installation check:" && find /opt/homebrew /usr/local -name "protobuf*" -type d 2>/dev/null | head -5 # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory # the OpenSSL3 lib is stored under /usr/local/lib64. @@ -136,7 +136,7 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DProtobuf_DIR=/opt/homebrew/lib/cmake/protobuf' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" # run all python tests to make sure wheels are not defunct CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter" diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index f275af89d..b6e6b0ada 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -228,9 +228,33 @@ if(PROTOBUF_REQUIRED) message(STATUS "Build requires Protobuf") set(Protobuf_USE_STATIC_LIBS ON) # https://github.com/protocolbuffers/protobuf/issues/12637 - find_package(Protobuf CONFIG) + + # Try to find Protobuf using different methods + # First try with explicit paths for common installation locations + set(PROTOBUF_SEARCH_PATHS "/opt/homebrew" "/usr/local" "/usr") + foreach(SEARCH_PATH ${PROTOBUF_SEARCH_PATHS}) + if(EXISTS "${SEARCH_PATH}/lib/cmake/protobuf") + set(Protobuf_DIR "${SEARCH_PATH}/lib/cmake/protobuf") + set(Protobuf_ROOT "${SEARCH_PATH}") + message(STATUS "Trying Protobuf at ${SEARCH_PATH}") + break() + endif() + endforeach() + + find_package(Protobuf CONFIG QUIET) if(NOT Protobuf_FOUND) + message(STATUS "Protobuf CONFIG not found, trying REQUIRED method") find_package(Protobuf REQUIRED) + else() + message(STATUS "Found Protobuf using CONFIG method") + endif() + + # Additional check for CI environments + if(NOT Protobuf_FOUND) + message(FATAL_ERROR "Protobuf not found. Please install protobuf development libraries. " + "On macOS: brew install protobuf\n" + "On Ubuntu: sudo apt-get install libprotobuf-dev protobuf-compiler\n" + "On CentOS: sudo yum install protobuf-devel protobuf-compiler") endif() if(Protobuf_LIBRARY) From 55e0050a690eccae2a051546541f79b1fbc3d463 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 14:48:57 -0700 Subject: [PATCH 40/92] Fix macOS protobuf build issues - Fix Homebrew cmake conflicts by uninstalling conflicting versions first - Simplify protobuf detection logic in CMakeLists.txt to use standard find_package - Remove explicit protobuf paths from workflow environment variables - Add better verification and error handling in dependency scripts - Let CMake's find_package handle protobuf detection automatically --- .github/workflows/build_wheels.yml | 4 ++-- scripts/macos/brew_dependencies.sh | 12 ++++++++++++ tuplex/CMakeLists.txt | 16 +++------------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 840dcfd89..3eafe9edd 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -126,7 +126,7 @@ jobs: CIBW_BUILD: ${{ matrix.cibw-build }} # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile && echo "Protobuf installation check:" && find /opt/homebrew /usr/local -name "protobuf*" -type d 2>/dev/null | head -5 + CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile && echo "Verifying protobuf installation:" && protoc --version && find /opt/homebrew /usr/local -name "protobuf*" -type d 2>/dev/null | head -5 # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory # the OpenSSL3 lib is stored under /usr/local/lib64. @@ -136,7 +136,7 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DProtobuf_DIR=/opt/homebrew/lib/cmake/protobuf' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" # run all python tests to make sure wheels are not defunct CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter" diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index ac2a5512c..b0f6f341c 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -2,11 +2,23 @@ # This script installs all required dependencies via brew # for instructions on how to install brew, visit https://brew.sh/ +set -euxo pipefail + +# Uninstall conflicting cmake first if it exists from a different tap +echo "Checking for conflicting cmake installations..." +brew uninstall cmake --ignore-dependencies || true # brew doesn't provide llvm@16 bottle anymore for big sur, but python3.8 only works with big sur tags. use llvm@15 instead +echo "Installing dependencies..." brew install openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf # link (when e.g. used from restoring cache) +echo "Linking packages..." brew link --overwrite cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil +# Verify protobuf installation +echo "Verifying protobuf installation..." +protoc --version || echo "Warning: protoc not found in PATH" +find /opt/homebrew /usr/local -name "protobuf*" -type d 2>/dev/null | head -5 + echo "Done!" \ No newline at end of file diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index b6e6b0ada..8e97164f4 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -229,19 +229,9 @@ if(PROTOBUF_REQUIRED) set(Protobuf_USE_STATIC_LIBS ON) # https://github.com/protocolbuffers/protobuf/issues/12637 - # Try to find Protobuf using different methods - # First try with explicit paths for common installation locations - set(PROTOBUF_SEARCH_PATHS "/opt/homebrew" "/usr/local" "/usr") - foreach(SEARCH_PATH ${PROTOBUF_SEARCH_PATHS}) - if(EXISTS "${SEARCH_PATH}/lib/cmake/protobuf") - set(Protobuf_DIR "${SEARCH_PATH}/lib/cmake/protobuf") - set(Protobuf_ROOT "${SEARCH_PATH}") - message(STATUS "Trying Protobuf at ${SEARCH_PATH}") - break() - endif() - endforeach() - + # Try CONFIG method first (preferred for modern protobuf installations) find_package(Protobuf CONFIG QUIET) + if(NOT Protobuf_FOUND) message(STATUS "Protobuf CONFIG not found, trying REQUIRED method") find_package(Protobuf REQUIRED) @@ -249,7 +239,7 @@ if(PROTOBUF_REQUIRED) message(STATUS "Found Protobuf using CONFIG method") endif() - # Additional check for CI environments + # Final verification if(NOT Protobuf_FOUND) message(FATAL_ERROR "Protobuf not found. Please install protobuf development libraries. " "On macOS: brew install protobuf\n" From d1b7c3592695c156cda9b87de16da057257ade2a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 15:06:19 -0700 Subject: [PATCH 41/92] Fix slow find command in macOS workflow verification - Replace slow find command that searches entire /opt/homebrew and /usr/local directories - Use targeted search in /opt/homebrew/lib/cmake and /usr/local/lib/cmake for protobuf cmake files - Reduce search scope to prevent timeout and improve build reliability --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 3eafe9edd..9b5ad82fb 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -126,7 +126,7 @@ jobs: CIBW_BUILD: ${{ matrix.cibw-build }} # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile && echo "Verifying protobuf installation:" && protoc --version && find /opt/homebrew /usr/local -name "protobuf*" -type d 2>/dev/null | head -5 + CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile && echo "Verifying protobuf installation:" && protoc --version && echo "Protobuf cmake files:" && find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -3 # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory # the OpenSSL3 lib is stored under /usr/local/lib64. From 2278215a73ba057d8bc5a5433df84830a0320cc4 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 15:33:06 -0700 Subject: [PATCH 42/92] pybind11 fix --- tuplex/python/CMakeLists.txt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 5b82be7f5..e0b7bfb2c 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -12,17 +12,21 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) SET(MODULE_NAME tuplex) # deprecated -include(FetchContent) #set(PYBIND11_NOPYTHON ON CACHE INTERNAL "") message(STATUS "Pybind11 uses python version ${Python3_VERSION}") set(PYBIND11_FINDPYTHON OFF CACHE INTERNAL "") set(PYBIND11_PYTHON_VERSION "${Python3_VERSION}" CACHE INTERNAL "") +# 3.0.0 requires python 3.10+ FetchContent_Declare(pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v3.0.1) -FetchContent_MakeAvailable(pybind11) - -# Source code & linking -file(GLOB_RECURSE SOURCES src/*.cc) + GIT_TAG v3.0.0) +FetchContent_GetProperties(pybind11) +if(NOT pybind11_POPULATED) + # Disable cmake warning, need to update to FetchContent_MakeAvailable eventually. + cmake_policy(SET CMP0169 OLD) + FetchContent_Populate(pybind11) + cmake_policy(SET CMP0169 NEW) + add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR}) +endif() # this is the syntax for the old deprecated packages, switching out to newer one. #PYTHON_ADD_MODULE(${MODULE_NAME} ${SOURCES}) From cb60878e99cc687503bfb1f9f192ff5d8c2ae778 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 15:42:06 -0700 Subject: [PATCH 43/92] fixes --- tuplex/python/CMakeLists.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index e0b7bfb2c..c1589e3ed 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -16,9 +16,16 @@ SET(MODULE_NAME tuplex) message(STATUS "Pybind11 uses python version ${Python3_VERSION}") set(PYBIND11_FINDPYTHON OFF CACHE INTERNAL "") set(PYBIND11_PYTHON_VERSION "${Python3_VERSION}" CACHE INTERNAL "") -# 3.0.0 requires python 3.10+ + +# 3.0.0 requires python 3.10+, 2.13.6 supports python 3.9. +if(Python3_VERSION VERSION_LESS "3.10") + set(pybind11_version v2.13.6) +else() + set(pybind11_version v3.0.1) +endif() + FetchContent_Declare(pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v3.0.0) + GIT_TAG ${pybind11_version}) FetchContent_GetProperties(pybind11) if(NOT pybind11_POPULATED) # Disable cmake warning, need to update to FetchContent_MakeAvailable eventually. From c59400772b1d3b52d84f09d502c779c5b0818540 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 15:45:25 -0700 Subject: [PATCH 44/92] update cibuildwheel --- .github/workflows/build_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9b5ad82fb..af602817c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -114,7 +114,8 @@ jobs: python-version: '3.11' - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.22.0 + # by default use the latest version, if there are issues use 3.1.4. + run: python -m pip install cibuildwheel - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse From bf43f4c024eca871637af90264a367a4d1be9717 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 16:10:11 -0700 Subject: [PATCH 45/92] force reinstall --- scripts/macos/brew_dependencies.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index b0f6f341c..60470b76f 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -16,6 +16,10 @@ brew install openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre echo "Linking packages..." brew link --overwrite cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil + +# protobuf seems to have an issue on github actions, force reinstall therefore. +brew reinstall -f protobuf + # Verify protobuf installation echo "Verifying protobuf installation..." protoc --version || echo "Warning: protoc not found in PATH" From 1cdae249ba5f40386faf173f5d3970dcd417c62f Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 16:22:18 -0700 Subject: [PATCH 46/92] requirements --- .github/workflows/build_wheels.yml | 5 ++++- setup.py | 2 ++ tuplex/python/requirements.txt | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index af602817c..54d4126ea 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -139,8 +139,11 @@ jobs: # For github actions, $HOME is /Users/runner/ CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + # ensure build requirements are available during wheel building + CIBW_BUILD_REQUIRES: "cloudpickle numpy" + # run all python tests to make sure wheels are not defunct - CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter" + CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter cloudpickle" # Use following test command when segfaults happen to better pinpoint: # python3 -X faulthandler -m pytest -p no:faulthandler diff --git a/setup.py b/setup.py index 3d26dada7..842da6a6e 100644 --- a/setup.py +++ b/setup.py @@ -134,6 +134,7 @@ def run_command(cmd, cwd, env): "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", "cloudpickle>=2.1.0;python_version=='3.10'", "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', 'PyYAML>=3.13', 'psutil', 'pymongo', @@ -156,6 +157,7 @@ def run_command(cmd, cwd, env): "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", "cloudpickle>=2.1.0;python_version=='3.10'", "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', 'PyYAML>=3.13', 'psutil', 'pymongo', diff --git a/tuplex/python/requirements.txt b/tuplex/python/requirements.txt index 1c53763a5..587553299 100644 --- a/tuplex/python/requirements.txt +++ b/tuplex/python/requirements.txt @@ -17,6 +17,7 @@ jedi>=0.13.2 cloudpickle>=0.6.1,<2.0.0;python_version<'3.10' cloudpickle>=2.1.0;python_version=='3.10' cloudpickle>=2.2.1;python_version>='3.11' +numpy>=1.15.0 PyYAML>=3.13 psutil pymongo From 475df81f57a7197390147f64f80263bc713baa62 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 17:05:29 -0700 Subject: [PATCH 47/92] test --- .github/workflows/build_wheels.yml | 41 +++- scripts/macos/brew_dependencies.sh | 165 ++++++++++++++-- scripts/macos/test_build_environment.sh | 244 ++++++++++++++++++++++++ 3 files changed, 437 insertions(+), 13 deletions(-) create mode 100755 scripts/macos/test_build_environment.sh diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 54d4126ea..4caed19a2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -117,6 +117,13 @@ jobs: # by default use the latest version, if there are issues use 3.1.4. run: python -m pip install cibuildwheel + - name: Test build environment (macOS only) + if: runner.os == 'macOS' + run: | + echo "=== Pre-build environment test ===" + bash ./scripts/macos/test_build_environment.sh + echo "=== Pre-build test completed ===" + - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse env: @@ -127,7 +134,37 @@ jobs: CIBW_BUILD: ${{ matrix.cibw-build }} # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile && echo "Verifying protobuf installation:" && protoc --version && echo "Protobuf cmake files:" && find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -3 + CIBW_BEFORE_ALL_MACOS: | + set -euxo pipefail + echo "=== Starting macOS dependency installation ===" + + # Install dependencies with comprehensive verification + bash ./scripts/macos/install_antlr4_cpp_runtime.sh + bash ./scripts/macos/brew_dependencies.sh + bash ./scripts/macos/install_aws-sdk-cpp.sh + + # Set up Java environment + echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile + export PATH="/usr/local/opt/openjdk@11/bin:$PATH" + + # Update environment variables for the build + export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" + export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" + export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" + + # Run comprehensive build environment test + echo "=== Running build environment test ===" + bash ./scripts/macos/test_build_environment.sh + + # Final verification + echo "=== Final verification ===" + echo "Protobuf version: $(protoc --version)" + echo "Protobuf cmake files:" + find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -5 + echo "CMake version: $(cmake --version | head -1)" + echo "Java version: $(java -version 2>&1 | head -1)" + + echo "=== macOS dependency installation completed ===" # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory # the OpenSSL3 lib is stored under /usr/local/lib64. @@ -137,7 +174,7 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DLLVM_ROOT=/opt/homebrew/Cellar/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" # ensure build requirements are available during wheel building CIBW_BUILD_REQUIRES: "cloudpickle numpy" diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 60470b76f..79fc42dbc 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -4,25 +4,168 @@ set -euxo pipefail +# Function to verify a command exists and is working +verify_command() { + local cmd="$1" + local expected_output="$2" + echo "Verifying $cmd..." + + if command -v "$cmd" >/dev/null 2>&1; then + if [ -n "$expected_output" ]; then + local actual_output + actual_output=$($cmd --version 2>/dev/null || echo "version check failed") + echo "✅ $cmd found: $actual_output" + else + echo "✅ $cmd found" + fi + return 0 + else + echo "❌ $cmd not found in PATH" + return 1 + fi +} + +# Function to verify library exists +verify_library() { + local lib_name="$1" + local search_paths="$2" + echo "Verifying $lib_name library..." + + for path in $search_paths; do + if find "$path" -name "*${lib_name}*" -type f 2>/dev/null | grep -q .; then + echo "✅ $lib_name found in $path" + find "$path" -name "*${lib_name}*" -type f 2>/dev/null | head -3 + return 0 + fi + done + + echo "❌ $lib_name not found in any of: $search_paths" + return 1 +} + +# Function to verify cmake package +verify_cmake_package() { + local package_name="$1" + echo "Verifying CMake package: $package_name" + + if find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*${package_name}*" 2>/dev/null | grep -q .; then + echo "✅ CMake package $package_name found" + find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*${package_name}*" 2>/dev/null | head -3 + return 0 + else + echo "❌ CMake package $package_name not found" + return 1 + fi +} + +echo "=== Starting macOS dependency installation ===" +echo "Current PATH: $PATH" +echo "Current architecture: $(uname -m)" +echo "macOS version: $(sw_vers -productVersion)" + +# Update brew first +echo "Updating brew..." +brew update || echo "Warning: brew update failed" + # Uninstall conflicting cmake first if it exists from a different tap echo "Checking for conflicting cmake installations..." brew uninstall cmake --ignore-dependencies || true -# brew doesn't provide llvm@16 bottle anymore for big sur, but python3.8 only works with big sur tags. use llvm@15 instead +# Install dependencies with better error handling echo "Installing dependencies..." -brew install openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf +DEPENDENCIES="openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf" + +for dep in $DEPENDENCIES; do + echo "Installing $dep..." + if brew install "$dep"; then + echo "✅ $dep installed successfully" + else + echo "❌ Failed to install $dep" + echo "Attempting to reinstall $dep..." + brew reinstall -f "$dep" || { + echo "❌ Failed to reinstall $dep, continuing..." + } + fi +done -# link (when e.g. used from restoring cache) +# Link packages with better error handling echo "Linking packages..." -brew link --overwrite cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil +LINK_PACKAGES="cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil" + +for pkg in $LINK_PACKAGES; do + echo "Linking $pkg..." + brew link --overwrite "$pkg" || echo "Warning: Failed to link $pkg" +done + +# Force reinstall protobuf as it often has issues +echo "Force reinstalling protobuf..." +brew reinstall -f protobuf || { + echo "❌ Failed to reinstall protobuf" + exit 1 +} + +# Update PATH to include brew binaries +echo "Updating PATH..." +export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" +echo "Updated PATH: $PATH" + +# Comprehensive verification +echo "=== Verifying installations ===" + +# Verify essential commands +verify_command "protoc" "libprotoc" +verify_command "cmake" "cmake version" +verify_command "java" "openjdk" +verify_command "pkg-config" "pkg-config" + +# Verify protobuf specifically +echo "=== Protobuf verification ===" +if verify_command "protoc" "libprotoc"; then + echo "Protobuf version: $(protoc --version)" + echo "Protobuf include path: $(pkg-config --variable=includedir protobuf 2>/dev/null || echo 'not found')" + echo "Protobuf lib path: $(pkg-config --variable=libdir protobuf 2>/dev/null || echo 'not found')" +else + echo "❌ Protobuf verification failed" + echo "Searching for protobuf installations..." + find /opt/homebrew /usr/local -name "*protobuf*" 2>/dev/null | head -10 + exit 1 +fi + +# Verify libraries +verify_library "protobuf" "/opt/homebrew/lib /usr/local/lib" +verify_library "boost" "/opt/homebrew/lib /usr/local/lib" +verify_library "llvm" "/opt/homebrew/lib /usr/local/lib" + +# Verify CMake packages +verify_cmake_package "protobuf" +verify_cmake_package "boost" +verify_cmake_package "llvm" +# Test protobuf compilation +echo "=== Testing protobuf compilation ===" +cat > /tmp/test.proto << 'EOF' +syntax = "proto3"; +package test; +message TestMessage { + string name = 1; + int32 value = 2; +} +EOF -# protobuf seems to have an issue on github actions, force reinstall therefore. -brew reinstall -f protobuf +if protoc --cpp_out=/tmp /tmp/test.proto; then + echo "✅ Protobuf compilation test passed" + ls -la /tmp/test.pb.* +else + echo "❌ Protobuf compilation test failed" + exit 1 +fi -# Verify protobuf installation -echo "Verifying protobuf installation..." -protoc --version || echo "Warning: protoc not found in PATH" -find /opt/homebrew /usr/local -name "protobuf*" -type d 2>/dev/null | head -5 +# Final environment summary +echo "=== Environment Summary ===" +echo "PATH: $PATH" +echo "PKG_CONFIG_PATH: ${PKG_CONFIG_PATH:-not set}" +echo "CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH:-not set}" +echo "Protobuf locations:" +find /opt/homebrew /usr/local -name "*protobuf*" 2>/dev/null | head -5 -echo "Done!" \ No newline at end of file +echo "=== Installation completed successfully! ===" \ No newline at end of file diff --git a/scripts/macos/test_build_environment.sh b/scripts/macos/test_build_environment.sh new file mode 100755 index 000000000..1a2d9a2f1 --- /dev/null +++ b/scripts/macos/test_build_environment.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# Comprehensive test script for macOS build environment +# This script verifies that all dependencies are properly installed and accessible + +set -euxo pipefail + +echo "=== macOS Build Environment Test ===" +echo "Timestamp: $(date)" +echo "Architecture: $(uname -m)" +echo "macOS version: $(sw_vers -productVersion)" +echo "Current user: $(whoami)" +echo "Working directory: $(pwd)" + +# Function to test command availability and version +test_command() { + local cmd="$1" + local min_version="$2" + local version_cmd="$3" + + echo "Testing $cmd..." + + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "❌ $cmd not found in PATH" + return 1 + fi + + local version_output + if version_output=$($version_cmd 2>&1); then + echo "✅ $cmd found: $version_output" + + if [ -n "$min_version" ]; then + # Simple version comparison (works for most tools) + if [[ "$version_output" =~ $min_version ]]; then + echo "✅ $cmd version meets requirements" + else + echo "⚠️ $cmd version may not meet requirements (expected: $min_version, got: $version_output)" + fi + fi + return 0 + else + echo "❌ $cmd found but version check failed: $version_output" + return 1 + fi +} + +# Function to test library availability +test_library() { + local lib_name="$1" + local search_paths="$2" + local pkg_config_name="$3" + + echo "Testing $lib_name library..." + + # Test via pkg-config if available + if [ -n "$pkg_config_name" ] && command -v pkg-config >/dev/null 2>&1; then + if pkg-config --exists "$pkg_config_name" 2>/dev/null; then + echo "✅ $lib_name found via pkg-config" + echo " Version: $(pkg-config --modversion "$pkg_config_name")" + echo " Cflags: $(pkg-config --cflags "$pkg_config_name")" + echo " Libs: $(pkg-config --libs "$pkg_config_name")" + return 0 + fi + fi + + # Test via file search + for path in $search_paths; do + if find "$path" -name "*${lib_name}*" -type f 2>/dev/null | grep -q .; then + echo "✅ $lib_name found in $path" + find "$path" -name "*${lib_name}*" -type f 2>/dev/null | head -3 + return 0 + fi + done + + echo "❌ $lib_name not found in any of: $search_paths" + return 1 +} + +# Function to test CMake package +test_cmake_package() { + local package_name="$1" + local cmake_name="$2" + + echo "Testing CMake package: $package_name" + + # Create a temporary CMakeLists.txt to test package finding + cat > /tmp/test_cmake_${package_name}.cmake << EOF +cmake_minimum_required(VERSION 3.16) +project(test_${package_name}) +find_package(${cmake_name} REQUIRED) +message(STATUS "Found ${cmake_name}: \${${cmake_name}_VERSION}") +EOF + + if cmake -P /tmp/test_cmake_${package_name}.cmake 2>/dev/null; then + echo "✅ CMake package $package_name found" + rm -f /tmp/test_cmake_${package_name}.cmake + return 0 + else + echo "❌ CMake package $package_name not found" + rm -f /tmp/test_cmake_${package_name}.cmake + return 1 + fi +} + +# Function to test compilation +test_compilation() { + local test_name="$1" + local source_code="$2" + local compile_cmd="$3" + + echo "Testing $test_name compilation..." + + # Create temporary source file + local temp_file="/tmp/test_${test_name}.cpp" + echo "$source_code" > "$temp_file" + + if eval "$compile_cmd" 2>/dev/null; then + echo "✅ $test_name compilation successful" + rm -f "$temp_file" "/tmp/test_${test_name}" + return 0 + else + echo "❌ $test_name compilation failed" + rm -f "$temp_file" "/tmp/test_${test_name}" + return 1 + fi +} + +echo "=== Testing Essential Commands ===" + +# Test basic commands +test_command "brew" "" "brew --version" +test_command "cmake" "3.16" "cmake --version" +test_command "protoc" "libprotoc" "protoc --version" +test_command "java" "openjdk" "java -version" +test_command "pkg-config" "" "pkg-config --version" +test_command "python3" "3.9" "python3 --version" +test_command "pip3" "" "pip3 --version" + +echo "=== Testing Libraries ===" + +# Test protobuf +test_library "protobuf" "/opt/homebrew/lib /usr/local/lib" "protobuf" + +# Test boost +test_library "boost" "/opt/homebrew/lib /usr/local/lib" "boost" + +# Test LLVM +test_library "llvm" "/opt/homebrew/lib /usr/local/lib" "llvm" + +# Test other libraries +test_library "zstd" "/opt/homebrew/lib /usr/local/lib" "libzstd" +test_library "pcre2" "/opt/homebrew/lib /usr/local/lib" "libpcre2-8" + +echo "=== Testing CMake Packages ===" + +# Test CMake package finding +test_cmake_package "protobuf" "Protobuf" +test_cmake_package "boost" "Boost" +test_cmake_package "llvm" "LLVM" + +echo "=== Testing Compilation ===" + +# Test protobuf compilation +test_compilation "protobuf" \ +"#include +#include +int main() { + std::cout << \"Protobuf test successful\" << std::endl; + return 0; +}" \ +"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lprotobuf $temp_file -o /tmp/test_protobuf" + +# Test boost compilation +test_compilation "boost" \ +"#include +#include +int main() { + std::cout << \"Boost version: \" << BOOST_VERSION << std::endl; + return 0; +}" \ +"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib $temp_file -o /tmp/test_boost" + +echo "=== Testing Environment Variables ===" + +# Test important environment variables +echo "PATH: $PATH" +echo "PKG_CONFIG_PATH: ${PKG_CONFIG_PATH:-not set}" +echo "CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH:-not set}" +echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH:-not set}" + +# Test brew paths +echo "Brew prefix: $(brew --prefix 2>/dev/null || echo 'not found')" +echo "Brew cellar: $(brew --cellar 2>/dev/null || echo 'not found')" + +echo "=== Testing Python Environment ===" + +# Test Python packages +python3 -c " +import sys +print(f'Python version: {sys.version}') +print(f'Python executable: {sys.executable}') + +# Test required packages +required_packages = ['cloudpickle', 'numpy'] +for pkg in required_packages: + try: + module = __import__(pkg) + print(f'✅ {pkg}: {module.__version__}') + except ImportError as e: + print(f'❌ {pkg}: {e}') +" + +echo "=== Testing CMake Configuration ===" + +# Test CMake configuration with our project +if [ -f "tuplex/CMakeLists.txt" ]; then + echo "Testing CMake configuration..." + + # Create a test build directory + mkdir -p /tmp/cmake_test_build + cd /tmp/cmake_test_build + + # Test basic CMake configuration + if cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON /Users/runner/work/tuplex/tuplex/tuplex 2>/dev/null; then + echo "✅ CMake configuration successful" + else + echo "❌ CMake configuration failed" + echo "CMake error output:" + cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON /Users/runner/work/tuplex/tuplex/tuplex 2>&1 | head -20 + fi + + cd - >/dev/null + rm -rf /tmp/cmake_test_build +else + echo "⚠️ tuplex/CMakeLists.txt not found, skipping CMake test" +fi + +echo "=== Final Summary ===" +echo "Build environment test completed at $(date)" + +# Count successes and failures +echo "Test results summary:" +echo "✅ Successful tests: $(grep -c '✅' <<< "$(cat)")" +echo "❌ Failed tests: $(grep -c '❌' <<< "$(cat)")" +echo "⚠️ Warnings: $(grep -c '⚠️' <<< "$(cat)")" From 5e4474898de3438da08aa9e8aeded97e2b7be13b Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 17:12:12 -0700 Subject: [PATCH 48/92] check again --- scripts/macos/brew_dependencies.sh | 3 +++ scripts/macos/install_aws-sdk-cpp.sh | 8 +++++++- scripts/macos/test_build_environment.sh | 16 +++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 79fc42dbc..6c8d75e08 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -62,6 +62,9 @@ echo "=== Starting macOS dependency installation ===" echo "Current PATH: $PATH" echo "Current architecture: $(uname -m)" echo "macOS version: $(sw_vers -productVersion)" +echo "Current working directory: $(pwd)" +echo "User: $(whoami)" +echo "Home directory: $HOME" # Update brew first echo "Updating brew..." diff --git a/scripts/macos/install_aws-sdk-cpp.sh b/scripts/macos/install_aws-sdk-cpp.sh index bdf7fadd9..ea53d00d6 100755 --- a/scripts/macos/install_aws-sdk-cpp.sh +++ b/scripts/macos/install_aws-sdk-cpp.sh @@ -38,5 +38,11 @@ cd /tmp && make -j${CPU_CORES} && make install && popd && - cd - || echo ">> error: AWS SDK failed" + cd - || { + echo ">> error: AWS SDK installation failed" + echo "Current directory: $(pwd)" + echo "Build directory contents:" + ls -la build/ 2>/dev/null || echo "Build directory not found" + exit 1 + } diff --git a/scripts/macos/test_build_environment.sh b/scripts/macos/test_build_environment.sh index 1a2d9a2f1..54b93c12a 100755 --- a/scripts/macos/test_build_environment.sh +++ b/scripts/macos/test_build_environment.sh @@ -220,12 +220,18 @@ if [ -f "tuplex/CMakeLists.txt" ]; then cd /tmp/cmake_test_build # Test basic CMake configuration - if cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON /Users/runner/work/tuplex/tuplex/tuplex 2>/dev/null; then - echo "✅ CMake configuration successful" + local project_dir + project_dir=$(find /Users/runner/work -name "tuplex" -type d 2>/dev/null | head -1 || echo ".") + if [ -d "$project_dir/tuplex" ]; then + if cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON "$project_dir/tuplex" 2>/dev/null; then + echo "✅ CMake configuration successful" + else + echo "❌ CMake configuration failed" + echo "CMake error output:" + cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON "$project_dir/tuplex" 2>&1 | head -20 + fi else - echo "❌ CMake configuration failed" - echo "CMake error output:" - cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON /Users/runner/work/tuplex/tuplex/tuplex 2>&1 | head -20 + echo "⚠️ Project directory not found, skipping CMake test" fi cd - >/dev/null From 8c23555b976376317e60f159cf2b389c839f3503 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 17:16:14 -0700 Subject: [PATCH 49/92] another test --- .github/workflows/build_wheels.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 4caed19a2..863196410 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -113,17 +113,15 @@ jobs: with: python-version: '3.11' + - name: Setup protobuf + uses: arduino/setup-protoc@v3 + with: + version: "32.x" + - name: Install cibuildwheel # by default use the latest version, if there are issues use 3.1.4. run: python -m pip install cibuildwheel - - name: Test build environment (macOS only) - if: runner.os == 'macOS' - run: | - echo "=== Pre-build environment test ===" - bash ./scripts/macos/test_build_environment.sh - echo "=== Pre-build test completed ===" - - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse env: From b271557bad36a03e858150284d2338f5c7fa00bd Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 17:19:31 -0700 Subject: [PATCH 50/92] update --- .github/workflows/build_wheels.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 863196410..85cf1a1b2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -113,11 +113,6 @@ jobs: with: python-version: '3.11' - - name: Setup protobuf - uses: arduino/setup-protoc@v3 - with: - version: "32.x" - - name: Install cibuildwheel # by default use the latest version, if there are issues use 3.1.4. run: python -m pip install cibuildwheel From d3888fbb49881b5abf0e02722385677a820455ae Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 17:32:08 -0700 Subject: [PATCH 51/92] test again --- .github/workflows/build_wheels.yml | 2 +- scripts/macos/brew_dependencies.sh | 98 ++++++++++++++++++++++++- scripts/macos/test_build_environment.sh | 14 +++- 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 85cf1a1b2..84a4c82f1 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -167,7 +167,7 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DLLVM_ROOT=/opt/homebrew/Cellar/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib -DLLVM_ROOT=/opt/homebrew/Cellar/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" # ensure build requirements are available during wheel building CIBW_BUILD_REQUIRES: "cloudpickle numpy" diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 6c8d75e08..10f123236 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -76,9 +76,11 @@ brew uninstall cmake --ignore-dependencies || true # Install dependencies with better error handling echo "Installing dependencies..." -DEPENDENCIES="openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf" -for dep in $DEPENDENCIES; do +# Install core dependencies first +CORE_DEPENDENCIES="openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf" + +for dep in $CORE_DEPENDENCIES; do echo "Installing $dep..." if brew install "$dep"; then echo "✅ $dep installed successfully" @@ -91,6 +93,46 @@ for dep in $DEPENDENCIES; do fi done +# Install boost separately with special handling +echo "Installing boost..." +if brew install boost; then + echo "✅ boost installed successfully" +else + echo "❌ Failed to install boost" + echo "Attempting to reinstall boost..." + brew reinstall -f boost || { + echo "❌ Failed to reinstall boost, trying alternative approach..." + # Try installing boost with specific options + brew install boost --build-from-source || { + echo "❌ Failed to install boost from source, continuing..." + } + } +fi + +# Install LLVM separately (can be problematic) +echo "Installing llvm@15..." +if brew install llvm@15; then + echo "✅ llvm@15 installed successfully" +else + echo "❌ Failed to install llvm@15" + echo "Attempting to reinstall llvm@15..." + brew reinstall -f llvm@15 || { + echo "❌ Failed to reinstall llvm@15, continuing..." + } +fi + +# Install protobuf last (often has issues) +echo "Installing protobuf..." +if brew install protobuf; then + echo "✅ protobuf installed successfully" +else + echo "❌ Failed to install protobuf" + echo "Attempting to reinstall protobuf..." + brew reinstall -f protobuf || { + echo "❌ Failed to reinstall protobuf, continuing..." + } +fi + # Link packages with better error handling echo "Linking packages..." LINK_PACKAGES="cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil" @@ -144,6 +186,24 @@ verify_cmake_package "protobuf" verify_cmake_package "boost" verify_cmake_package "llvm" +# Special boost verification +echo "=== Boost verification ===" +if verify_library "boost" "/opt/homebrew/lib /usr/local/lib"; then + echo "Boost library found" + # Check for specific boost libraries + for lib in libboost_system libboost_filesystem libboost_thread libboost_iostreams; do + if find /opt/homebrew/lib /usr/local/lib -name "${lib}*" 2>/dev/null | grep -q .; then + echo "✅ $lib found" + else + echo "⚠️ $lib not found" + fi + done +else + echo "❌ Boost verification failed" + echo "Searching for boost installations..." + find /opt/homebrew /usr/local -name "*boost*" 2>/dev/null | head -10 +fi + # Test protobuf compilation echo "=== Testing protobuf compilation ===" cat > /tmp/test.proto << 'EOF' @@ -163,6 +223,40 @@ else exit 1 fi +# Test boost compilation +echo "=== Testing boost compilation ===" +cat > /tmp/test_boost.cpp << 'EOF' +#include +#include +#include +#include +#include + +int main() { + std::cout << "Boost version: " << BOOST_VERSION << std::endl; + std::cout << "Boost filesystem test: " << boost::filesystem::current_path() << std::endl; + std::cout << "Boost test successful" << std::endl; + return 0; +} +EOF + +if g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lboost_system -lboost_filesystem -lboost_thread -lboost_iostreams /tmp/test_boost.cpp -o /tmp/test_boost && /tmp/test_boost; then + echo "✅ Boost compilation test passed" + rm -f /tmp/test_boost.cpp /tmp/test_boost +else + echo "❌ Boost compilation test failed" + echo "Trying with different boost libraries..." + # Try with just system library + if g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lboost_system /tmp/test_boost.cpp -o /tmp/test_boost && /tmp/test_boost; then + echo "✅ Boost compilation test passed (system library only)" + rm -f /tmp/test_boost.cpp /tmp/test_boost + else + echo "❌ Boost compilation test failed completely" + rm -f /tmp/test_boost.cpp /tmp/test_boost + exit 1 + fi +fi + # Final environment summary echo "=== Environment Summary ===" echo "PATH: $PATH" diff --git a/scripts/macos/test_build_environment.sh b/scripts/macos/test_build_environment.sh index 54b93c12a..750b0fd41 100755 --- a/scripts/macos/test_build_environment.sh +++ b/scripts/macos/test_build_environment.sh @@ -143,6 +143,16 @@ test_library "protobuf" "/opt/homebrew/lib /usr/local/lib" "protobuf" # Test boost test_library "boost" "/opt/homebrew/lib /usr/local/lib" "boost" +# Test specific boost libraries +echo "Testing specific boost libraries..." +for lib in libboost_system libboost_filesystem libboost_thread libboost_iostreams; do + if find /opt/homebrew/lib /usr/local/lib -name "${lib}*" 2>/dev/null | grep -q .; then + echo "✅ $lib found" + else + echo "❌ $lib not found" + fi +done + # Test LLVM test_library "llvm" "/opt/homebrew/lib /usr/local/lib" "llvm" @@ -172,12 +182,14 @@ int main() { # Test boost compilation test_compilation "boost" \ "#include +#include #include int main() { std::cout << \"Boost version: \" << BOOST_VERSION << std::endl; + std::cout << \"Boost filesystem test: \" << boost::filesystem::current_path() << std::endl; return 0; }" \ -"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib $temp_file -o /tmp/test_boost" +"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lboost_system -lboost_filesystem $temp_file -o /tmp/test_boost" echo "=== Testing Environment Variables ===" From 1fde2d84f0daa0f4da4852fe9f1f948b62aceec0 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 17:55:04 -0700 Subject: [PATCH 52/92] update --- .github/workflows/build_wheels.yml | 89 +++++++++++++----------------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 84a4c82f1..9e9e1ceb8 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -88,25 +88,42 @@ jobs: macos-target: "14.0" arch: "arm64" steps: - - uses: actions/checkout@v4 - - # disable cache for now. - #- name: Cache brew dependencies - # if: runner.os == 'macOS' - # uses: actions/cache@v4.2.1 - # with: - # # Paths to cache: - # # /usr/local/Homebrew - installation folder of Homebrew - # # /usr/local/Cellar - installation folder of Homebrew formulae - # # /usr/local/Frameworks, /usr/local/bin, /usr/local/opt - contain (links to) binaries installed by Homebrew formulae - # path: | - # /usr/local/Homebrew - # /usr/local/Cellar - # /usr/local/Frameworks - # /usr/local/bin - # /usr/local/opt - # # macos13 runners are x86, macos14 are arm. --> use os therefore as cache key. - # key: ${{ matrix.os }}-build-cache-${{ hashFiles('./scripts/macos/brew_dependencies.sh') }}-v2 + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install macOS dependencies + if: runner.os == 'macOS' + run: | + set -euxo pipefail + echo "=== Starting macOS dependency installation ===" + + # Install dependencies with comprehensive verification + bash ./scripts/macos/install_antlr4_cpp_runtime.sh + bash ./scripts/macos/brew_dependencies.sh + bash ./scripts/macos/install_aws-sdk-cpp.sh + + # Set up Java environment + echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile + export PATH="/usr/local/opt/openjdk@11/bin:$PATH" + + # Update environment variables for the build + export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" + export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" + export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" + + # Run comprehensive build environment test + echo "=== Running build environment test ===" + bash ./scripts/macos/test_build_environment.sh + + # Final verification + echo "=== Final verification ===" + echo "Protobuf version: $(protoc --version)" + echo "Protobuf cmake files:" + find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -5 + echo "CMake version: $(cmake --version | head -1)" + echo "Java version: $(java -version 2>&1 | head -1)" + + echo "=== macOS dependency installation completed ===" - name: Setup python uses: actions/setup-python@v5 @@ -126,38 +143,6 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" CIBW_BUILD: ${{ matrix.cibw-build }} - # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: | - set -euxo pipefail - echo "=== Starting macOS dependency installation ===" - - # Install dependencies with comprehensive verification - bash ./scripts/macos/install_antlr4_cpp_runtime.sh - bash ./scripts/macos/brew_dependencies.sh - bash ./scripts/macos/install_aws-sdk-cpp.sh - - # Set up Java environment - echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile - export PATH="/usr/local/opt/openjdk@11/bin:$PATH" - - # Update environment variables for the build - export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" - export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" - export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" - - # Run comprehensive build environment test - echo "=== Running build environment test ===" - bash ./scripts/macos/test_build_environment.sh - - # Final verification - echo "=== Final verification ===" - echo "Protobuf version: $(protoc --version)" - echo "Protobuf cmake files:" - find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -5 - echo "CMake version: $(cmake --version | head -1)" - echo "Java version: $(java -version 2>&1 | head -1)" - - echo "=== macOS dependency installation completed ===" # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory # the OpenSSL3 lib is stored under /usr/local/lib64. @@ -167,7 +152,7 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib -DLLVM_ROOT=/opt/homebrew/Cellar/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" # ensure build requirements are available during wheel building CIBW_BUILD_REQUIRES: "cloudpickle numpy" From f43a8e7919bf777879e9de0b2621cc9b2d281000 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 18:26:40 -0700 Subject: [PATCH 53/92] fixes --- .github/workflows/build_wheels.yml | 8 -- scripts/macos/brew_dependencies.sh | 161 ++++++------------------ scripts/macos/test_build_environment.sh | 10 +- 3 files changed, 43 insertions(+), 136 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9e9e1ceb8..640ab51f4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -115,14 +115,6 @@ jobs: echo "=== Running build environment test ===" bash ./scripts/macos/test_build_environment.sh - # Final verification - echo "=== Final verification ===" - echo "Protobuf version: $(protoc --version)" - echo "Protobuf cmake files:" - find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*protobuf*" 2>/dev/null | head -5 - echo "CMake version: $(cmake --version | head -1)" - echo "Java version: $(java -version 2>&1 | head -1)" - echo "=== macOS dependency installation completed ===" - name: Setup python diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 10f123236..1bf889a2b 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -31,14 +31,37 @@ verify_library() { local search_paths="$2" echo "Verifying $lib_name library..." + # Disable pipefail for this function since grep -q returns 1 when no matches found + set +o pipefail + + # Expand all paths including wildcards + local expanded_paths="" for path in $search_paths; do + if [[ "$path" == *"*"* ]]; then + # Expand wildcards + for expanded_path in $path; do + if [ -d "$expanded_path" ]; then + expanded_paths="$expanded_paths $expanded_path" + fi + done + else + expanded_paths="$expanded_paths $path" + fi + done + + # Search in all expanded paths + for path in $expanded_paths; do if find "$path" -name "*${lib_name}*" -type f 2>/dev/null | grep -q .; then echo "✅ $lib_name found in $path" find "$path" -name "*${lib_name}*" -type f 2>/dev/null | head -3 + # Re-enable pipefail before returning + set -o pipefail return 0 fi done + # Re-enable pipefail before returning + set -o pipefail echo "❌ $lib_name not found in any of: $search_paths" return 1 } @@ -48,11 +71,21 @@ verify_cmake_package() { local package_name="$1" echo "Verifying CMake package: $package_name" - if find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*${package_name}*" 2>/dev/null | grep -q .; then + # Disable pipefail for this function since grep -q returns 1 when no matches found + set +o pipefail + + # Search in standard locations and Cellar paths + local search_paths="/opt/homebrew/lib/cmake /usr/local/lib/cmake /opt/homebrew/Cellar/*/lib/cmake" + + if find $search_paths -name "*${package_name}*" 2>/dev/null | grep -q .; then echo "✅ CMake package $package_name found" - find /opt/homebrew/lib/cmake /usr/local/lib/cmake -name "*${package_name}*" 2>/dev/null | head -3 + find $search_paths -name "*${package_name}*" 2>/dev/null | head -3 + # Re-enable pipefail before returning + set -o pipefail return 0 else + # Re-enable pipefail before returning + set -o pipefail echo "❌ CMake package $package_name not found" return 1 fi @@ -93,46 +126,6 @@ for dep in $CORE_DEPENDENCIES; do fi done -# Install boost separately with special handling -echo "Installing boost..." -if brew install boost; then - echo "✅ boost installed successfully" -else - echo "❌ Failed to install boost" - echo "Attempting to reinstall boost..." - brew reinstall -f boost || { - echo "❌ Failed to reinstall boost, trying alternative approach..." - # Try installing boost with specific options - brew install boost --build-from-source || { - echo "❌ Failed to install boost from source, continuing..." - } - } -fi - -# Install LLVM separately (can be problematic) -echo "Installing llvm@15..." -if brew install llvm@15; then - echo "✅ llvm@15 installed successfully" -else - echo "❌ Failed to install llvm@15" - echo "Attempting to reinstall llvm@15..." - brew reinstall -f llvm@15 || { - echo "❌ Failed to reinstall llvm@15, continuing..." - } -fi - -# Install protobuf last (often has issues) -echo "Installing protobuf..." -if brew install protobuf; then - echo "✅ protobuf installed successfully" -else - echo "❌ Failed to install protobuf" - echo "Attempting to reinstall protobuf..." - brew reinstall -f protobuf || { - echo "❌ Failed to reinstall protobuf, continuing..." - } -fi - # Link packages with better error handling echo "Linking packages..." LINK_PACKAGES="cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil" @@ -142,13 +135,6 @@ for pkg in $LINK_PACKAGES; do brew link --overwrite "$pkg" || echo "Warning: Failed to link $pkg" done -# Force reinstall protobuf as it often has issues -echo "Force reinstalling protobuf..." -brew reinstall -f protobuf || { - echo "❌ Failed to reinstall protobuf" - exit 1 -} - # Update PATH to include brew binaries echo "Updating PATH..." export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" @@ -177,86 +163,15 @@ else fi # Verify libraries -verify_library "protobuf" "/opt/homebrew/lib /usr/local/lib" -verify_library "boost" "/opt/homebrew/lib /usr/local/lib" -verify_library "llvm" "/opt/homebrew/lib /usr/local/lib" +verify_library "protobuf" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/protobuf/*/lib" +verify_library "boost" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib" +verify_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llvm@15/*/lib" # Verify CMake packages verify_cmake_package "protobuf" verify_cmake_package "boost" verify_cmake_package "llvm" -# Special boost verification -echo "=== Boost verification ===" -if verify_library "boost" "/opt/homebrew/lib /usr/local/lib"; then - echo "Boost library found" - # Check for specific boost libraries - for lib in libboost_system libboost_filesystem libboost_thread libboost_iostreams; do - if find /opt/homebrew/lib /usr/local/lib -name "${lib}*" 2>/dev/null | grep -q .; then - echo "✅ $lib found" - else - echo "⚠️ $lib not found" - fi - done -else - echo "❌ Boost verification failed" - echo "Searching for boost installations..." - find /opt/homebrew /usr/local -name "*boost*" 2>/dev/null | head -10 -fi - -# Test protobuf compilation -echo "=== Testing protobuf compilation ===" -cat > /tmp/test.proto << 'EOF' -syntax = "proto3"; -package test; -message TestMessage { - string name = 1; - int32 value = 2; -} -EOF - -if protoc --cpp_out=/tmp /tmp/test.proto; then - echo "✅ Protobuf compilation test passed" - ls -la /tmp/test.pb.* -else - echo "❌ Protobuf compilation test failed" - exit 1 -fi - -# Test boost compilation -echo "=== Testing boost compilation ===" -cat > /tmp/test_boost.cpp << 'EOF' -#include -#include -#include -#include -#include - -int main() { - std::cout << "Boost version: " << BOOST_VERSION << std::endl; - std::cout << "Boost filesystem test: " << boost::filesystem::current_path() << std::endl; - std::cout << "Boost test successful" << std::endl; - return 0; -} -EOF - -if g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lboost_system -lboost_filesystem -lboost_thread -lboost_iostreams /tmp/test_boost.cpp -o /tmp/test_boost && /tmp/test_boost; then - echo "✅ Boost compilation test passed" - rm -f /tmp/test_boost.cpp /tmp/test_boost -else - echo "❌ Boost compilation test failed" - echo "Trying with different boost libraries..." - # Try with just system library - if g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lboost_system /tmp/test_boost.cpp -o /tmp/test_boost && /tmp/test_boost; then - echo "✅ Boost compilation test passed (system library only)" - rm -f /tmp/test_boost.cpp /tmp/test_boost - else - echo "❌ Boost compilation test failed completely" - rm -f /tmp/test_boost.cpp /tmp/test_boost - exit 1 - fi -fi - # Final environment summary echo "=== Environment Summary ===" echo "PATH: $PATH" diff --git a/scripts/macos/test_build_environment.sh b/scripts/macos/test_build_environment.sh index 750b0fd41..23d2dae75 100755 --- a/scripts/macos/test_build_environment.sh +++ b/scripts/macos/test_build_environment.sh @@ -138,15 +138,15 @@ test_command "pip3" "" "pip3 --version" echo "=== Testing Libraries ===" # Test protobuf -test_library "protobuf" "/opt/homebrew/lib /usr/local/lib" "protobuf" +test_library "protobuf" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/protobuf/*/lib" "protobuf" # Test boost -test_library "boost" "/opt/homebrew/lib /usr/local/lib" "boost" +test_library "boost" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib" "boost" # Test specific boost libraries echo "Testing specific boost libraries..." for lib in libboost_system libboost_filesystem libboost_thread libboost_iostreams; do - if find /opt/homebrew/lib /usr/local/lib -name "${lib}*" 2>/dev/null | grep -q .; then + if find /opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib -name "${lib}*" 2>/dev/null | grep -q .; then echo "✅ $lib found" else echo "❌ $lib not found" @@ -154,7 +154,7 @@ for lib in libboost_system libboost_filesystem libboost_thread libboost_iostream done # Test LLVM -test_library "llvm" "/opt/homebrew/lib /usr/local/lib" "llvm" +test_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llvm@15/*/lib" "llvm" # Test other libraries test_library "zstd" "/opt/homebrew/lib /usr/local/lib" "libzstd" @@ -189,7 +189,7 @@ int main() { std::cout << \"Boost filesystem test: \" << boost::filesystem::current_path() << std::endl; return 0; }" \ -"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lboost_system -lboost_filesystem $temp_file -o /tmp/test_boost" +"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -L/opt/homebrew/Cellar/boost/*/lib -lboost_system -lboost_filesystem $temp_file -o /tmp/test_boost" echo "=== Testing Environment Variables ===" From 825c2f3d8073e206615478ba1fa0dc6e87582100 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 18:50:32 -0700 Subject: [PATCH 54/92] less verbose --- .github/workflows/build_wheels.yml | 2 +- scripts/macos/brew_dependencies.sh | 15 +++------------ scripts/macos/install_antlr4_cpp_runtime.sh | 2 +- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 640ab51f4..4cc3c941f 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -98,8 +98,8 @@ jobs: echo "=== Starting macOS dependency installation ===" # Install dependencies with comprehensive verification - bash ./scripts/macos/install_antlr4_cpp_runtime.sh bash ./scripts/macos/brew_dependencies.sh + bash ./scripts/macos/install_antlr4_cpp_runtime.sh bash ./scripts/macos/install_aws-sdk-cpp.sh # Set up Java environment diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 1bf889a2b..01367946a 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -111,7 +111,7 @@ brew uninstall cmake --ignore-dependencies || true echo "Installing dependencies..." # Install core dependencies first -CORE_DEPENDENCIES="openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf" +CORE_DEPENDENCIES="openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf protobuf boost" for dep in $CORE_DEPENDENCIES; do echo "Installing $dep..." @@ -119,23 +119,14 @@ for dep in $CORE_DEPENDENCIES; do echo "✅ $dep installed successfully" else echo "❌ Failed to install $dep" - echo "Attempting to reinstall $dep..." + echo "Attempting to reinstall $dep with --force ..." brew reinstall -f "$dep" || { echo "❌ Failed to reinstall $dep, continuing..." } fi done -# Link packages with better error handling -echo "Linking packages..." -LINK_PACKAGES="cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil" - -for pkg in $LINK_PACKAGES; do - echo "Linking $pkg..." - brew link --overwrite "$pkg" || echo "Warning: Failed to link $pkg" -done - -# Update PATH to include brew binaries +# # Update PATH to include brew binaries echo "Updating PATH..." export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" echo "Updated PATH: $PATH" diff --git a/scripts/macos/install_antlr4_cpp_runtime.sh b/scripts/macos/install_antlr4_cpp_runtime.sh index 777a83495..1355e9590 100644 --- a/scripts/macos/install_antlr4_cpp_runtime.sh +++ b/scripts/macos/install_antlr4_cpp_runtime.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This is a script to install the antlr4 runtime. -set -euxo pipefail +set -euo pipefail # Specify here target location. PREFIX=${PREFIX:-/usr/local} From 212c0bd79699fcef4e657f6431d3de22dedb8aee Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 18:53:22 -0700 Subject: [PATCH 55/92] update --- .github/workflows/build_wheels.yml | 2 +- scripts/macos/brew_dependencies.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 4cc3c941f..29ff6d4af 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -94,7 +94,7 @@ jobs: - name: Install macOS dependencies if: runner.os == 'macOS' run: | - set -euxo pipefail + set -euo pipefail echo "=== Starting macOS dependency installation ===" # Install dependencies with comprehensive verification diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 01367946a..30d30edbe 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -2,7 +2,7 @@ # This script installs all required dependencies via brew # for instructions on how to install brew, visit https://brew.sh/ -set -euxo pipefail +set -euo pipefail # Function to verify a command exists and is working verify_command() { @@ -101,7 +101,7 @@ echo "Home directory: $HOME" # Update brew first echo "Updating brew..." -brew update || echo "Warning: brew update failed" +brew update --quiet || echo "Warning: brew update failed" # Uninstall conflicting cmake first if it exists from a different tap echo "Checking for conflicting cmake installations..." @@ -111,7 +111,7 @@ brew uninstall cmake --ignore-dependencies || true echo "Installing dependencies..." # Install core dependencies first -CORE_DEPENDENCIES="openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf protobuf boost" +CORE_DEPENDENCIES="llvm@15 openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf protobuf boost" for dep in $CORE_DEPENDENCIES; do echo "Installing $dep..." From 4ebd88728ea88262a547c08d7c2fd181096c2044 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:06:37 -0700 Subject: [PATCH 56/92] fixes --- scripts/macos/brew_dependencies.sh | 71 +++++++++++++++++++----------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 30d30edbe..98ea1b3da 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -31,9 +31,6 @@ verify_library() { local search_paths="$2" echo "Verifying $lib_name library..." - # Disable pipefail for this function since grep -q returns 1 when no matches found - set +o pipefail - # Expand all paths including wildcards local expanded_paths="" for path in $search_paths; do @@ -51,17 +48,23 @@ verify_library() { # Search in all expanded paths for path in $expanded_paths; do - if find "$path" -name "*${lib_name}*" -type f 2>/dev/null | grep -q .; then + echo " Searching in: $path" + # Check if directory exists before searching + if [ ! -d "$path" ]; then + echo " Directory does not exist, skipping" + continue + fi + # Use a temporary variable to capture output and check length + local found_files + found_files=$(find "$path" -name "*${lib_name}*" -type f 2>/dev/null) + echo " Found $(echo "$found_files" | wc -l) files" + if [ -n "$found_files" ]; then echo "✅ $lib_name found in $path" - find "$path" -name "*${lib_name}*" -type f 2>/dev/null | head -3 - # Re-enable pipefail before returning - set -o pipefail + echo "$found_files" | head -3 return 0 fi done - # Re-enable pipefail before returning - set -o pipefail echo "❌ $lib_name not found in any of: $search_paths" return 1 } @@ -71,24 +74,42 @@ verify_cmake_package() { local package_name="$1" echo "Verifying CMake package: $package_name" - # Disable pipefail for this function since grep -q returns 1 when no matches found - set +o pipefail - # Search in standard locations and Cellar paths local search_paths="/opt/homebrew/lib/cmake /usr/local/lib/cmake /opt/homebrew/Cellar/*/lib/cmake" - if find $search_paths -name "*${package_name}*" 2>/dev/null | grep -q .; then - echo "✅ CMake package $package_name found" - find $search_paths -name "*${package_name}*" 2>/dev/null | head -3 - # Re-enable pipefail before returning - set -o pipefail - return 0 - else - # Re-enable pipefail before returning - set -o pipefail - echo "❌ CMake package $package_name not found" - return 1 - fi + # Expand all paths including wildcards + local expanded_paths="" + for path in $search_paths; do + if [[ "$path" == *"*"* ]]; then + # Expand wildcards + for expanded_path in $path; do + if [ -d "$expanded_path" ]; then + expanded_paths="$expanded_paths $expanded_path" + fi + done + else + expanded_paths="$expanded_paths $path" + fi + done + + # Search in all expanded paths + for path in $expanded_paths; do + # Check if directory exists before searching + if [ ! -d "$path" ]; then + continue + fi + # Use a temporary variable to capture output and check length + local found_packages + found_packages=$(find "$path" -name "*${package_name}*" 2>/dev/null) + if [ -n "$found_packages" ]; then + echo "✅ CMake package $package_name found" + echo "$found_packages" | head -3 + return 0 + fi + done + + echo "❌ CMake package $package_name not found" + return 1 } echo "=== Starting macOS dependency installation ===" @@ -158,7 +179,7 @@ verify_library "protobuf" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar verify_library "boost" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib" verify_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llvm@15/*/lib" -# Verify CMake packages +# Verify cmake packages verify_cmake_package "protobuf" verify_cmake_package "boost" verify_cmake_package "llvm" From 34e23579972e42dd905710b65bef68215ef01afa Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:11:26 -0700 Subject: [PATCH 57/92] more concise script --- scripts/macos/brew_dependencies.sh | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 98ea1b3da..380c95d9e 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -122,11 +122,11 @@ echo "Home directory: $HOME" # Update brew first echo "Updating brew..." -brew update --quiet || echo "Warning: brew update failed" +brew update --quiet 2>/dev/null || echo "Warning: brew update failed" # Uninstall conflicting cmake first if it exists from a different tap echo "Checking for conflicting cmake installations..." -brew uninstall cmake --ignore-dependencies || true +brew uninstall cmake --ignore-dependencies 2>/dev/null || true # Install dependencies with better error handling echo "Installing dependencies..." @@ -135,15 +135,24 @@ echo "Installing dependencies..." CORE_DEPENDENCIES="llvm@15 openjdk@11 cmake coreutils zstd zlib libmagic pcre2 gflags yaml-cpp celero wget googletest libdwarf libelf protobuf boost" for dep in $CORE_DEPENDENCIES; do - echo "Installing $dep..." - if brew install "$dep"; then - echo "✅ $dep installed successfully" + # Check if package is already installed + if brew list "$dep" &>/dev/null; then + version=$(brew list --versions "$dep" | awk '{print $2}') + echo "✅ $dep $version already installed, skipping." else - echo "❌ Failed to install $dep" - echo "Attempting to reinstall $dep with --force ..." - brew reinstall -f "$dep" || { - echo "❌ Failed to reinstall $dep, continuing..." - } + if brew install "$dep" 2>/dev/null; then + version=$(brew list --versions "$dep" | awk '{print $2}') + echo "✅ $dep $version installed successfully." + else + echo "❌ Failed to install $dep" + echo "Attempting to reinstall $dep with --force ..." + if brew reinstall -f "$dep" 2>/dev/null; then + version=$(brew list --versions "$dep" | awk '{print $2}') + echo "✅ $dep $version reinstalled successfully." + else + echo "❌ Failed to reinstall $dep, continuing..." + fi + fi fi done From 501d527d929b8b5921bfd9911b684dedbdd71fbe Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:25:18 -0700 Subject: [PATCH 58/92] good state --- scripts/macos/brew_dependencies.sh | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 380c95d9e..112ed044d 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -75,22 +75,14 @@ verify_cmake_package() { echo "Verifying CMake package: $package_name" # Search in standard locations and Cellar paths - local search_paths="/opt/homebrew/lib/cmake /usr/local/lib/cmake /opt/homebrew/Cellar/*/lib/cmake" + # Use find to get all CMake directories + local expanded_paths="/opt/homebrew/lib/cmake /usr/local/lib/cmake" - # Expand all paths including wildcards - local expanded_paths="" - for path in $search_paths; do - if [[ "$path" == *"*"* ]]; then - # Expand wildcards - for expanded_path in $path; do - if [ -d "$expanded_path" ]; then - expanded_paths="$expanded_paths $expanded_path" - fi - done - else - expanded_paths="$expanded_paths $path" - fi - done + # Add Cellar paths using find to get actual paths + cellar_cmake_paths=$(find /opt/homebrew/Cellar -name "cmake" -type d -path "*/lib/cmake" 2>/dev/null) + if [ -n "$cellar_cmake_paths" ]; then + expanded_paths="$expanded_paths $cellar_cmake_paths" + fi # Search in all expanded paths for path in $expanded_paths; do @@ -124,10 +116,6 @@ echo "Home directory: $HOME" echo "Updating brew..." brew update --quiet 2>/dev/null || echo "Warning: brew update failed" -# Uninstall conflicting cmake first if it exists from a different tap -echo "Checking for conflicting cmake installations..." -brew uninstall cmake --ignore-dependencies 2>/dev/null || true - # Install dependencies with better error handling echo "Installing dependencies..." @@ -191,7 +179,6 @@ verify_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llv # Verify cmake packages verify_cmake_package "protobuf" verify_cmake_package "boost" -verify_cmake_package "llvm" # Final environment summary echo "=== Environment Summary ===" From 5cc13a6a6a3aed717d30335fc754617e85b9c2ac Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:27:31 -0700 Subject: [PATCH 59/92] cleanup --- .github/workflows/build_wheels.yml | 12 +- scripts/macos/brew_dependencies.sh | 20 +- scripts/macos/test_build_environment.sh | 262 ------------------------ 3 files changed, 8 insertions(+), 286 deletions(-) delete mode 100755 scripts/macos/test_build_environment.sh diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 29ff6d4af..7e39d1ac4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -96,7 +96,13 @@ jobs: run: | set -euo pipefail echo "=== Starting macOS dependency installation ===" - + echo "Current PATH: $PATH" + echo "Current architecture: $(uname -m)" + echo "macOS version: $(sw_vers -productVersion)" + echo "Current working directory: $(pwd)" + echo "User: $(whoami)" + echo "Home directory: $HOME" + # Install dependencies with comprehensive verification bash ./scripts/macos/brew_dependencies.sh bash ./scripts/macos/install_antlr4_cpp_runtime.sh @@ -111,10 +117,6 @@ jobs: export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" - # Run comprehensive build environment test - echo "=== Running build environment test ===" - bash ./scripts/macos/test_build_environment.sh - echo "=== macOS dependency installation completed ===" - name: Setup python diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 112ed044d..3b5ae26ea 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -104,14 +104,6 @@ verify_cmake_package() { return 1 } -echo "=== Starting macOS dependency installation ===" -echo "Current PATH: $PATH" -echo "Current architecture: $(uname -m)" -echo "macOS version: $(sw_vers -productVersion)" -echo "Current working directory: $(pwd)" -echo "User: $(whoami)" -echo "Home directory: $HOME" - # Update brew first echo "Updating brew..." brew update --quiet 2>/dev/null || echo "Warning: brew update failed" @@ -178,14 +170,4 @@ verify_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llv # Verify cmake packages verify_cmake_package "protobuf" -verify_cmake_package "boost" - -# Final environment summary -echo "=== Environment Summary ===" -echo "PATH: $PATH" -echo "PKG_CONFIG_PATH: ${PKG_CONFIG_PATH:-not set}" -echo "CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH:-not set}" -echo "Protobuf locations:" -find /opt/homebrew /usr/local -name "*protobuf*" 2>/dev/null | head -5 - -echo "=== Installation completed successfully! ===" \ No newline at end of file +verify_cmake_package "boost" \ No newline at end of file diff --git a/scripts/macos/test_build_environment.sh b/scripts/macos/test_build_environment.sh deleted file mode 100755 index 23d2dae75..000000000 --- a/scripts/macos/test_build_environment.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env bash -# Comprehensive test script for macOS build environment -# This script verifies that all dependencies are properly installed and accessible - -set -euxo pipefail - -echo "=== macOS Build Environment Test ===" -echo "Timestamp: $(date)" -echo "Architecture: $(uname -m)" -echo "macOS version: $(sw_vers -productVersion)" -echo "Current user: $(whoami)" -echo "Working directory: $(pwd)" - -# Function to test command availability and version -test_command() { - local cmd="$1" - local min_version="$2" - local version_cmd="$3" - - echo "Testing $cmd..." - - if ! command -v "$cmd" >/dev/null 2>&1; then - echo "❌ $cmd not found in PATH" - return 1 - fi - - local version_output - if version_output=$($version_cmd 2>&1); then - echo "✅ $cmd found: $version_output" - - if [ -n "$min_version" ]; then - # Simple version comparison (works for most tools) - if [[ "$version_output" =~ $min_version ]]; then - echo "✅ $cmd version meets requirements" - else - echo "⚠️ $cmd version may not meet requirements (expected: $min_version, got: $version_output)" - fi - fi - return 0 - else - echo "❌ $cmd found but version check failed: $version_output" - return 1 - fi -} - -# Function to test library availability -test_library() { - local lib_name="$1" - local search_paths="$2" - local pkg_config_name="$3" - - echo "Testing $lib_name library..." - - # Test via pkg-config if available - if [ -n "$pkg_config_name" ] && command -v pkg-config >/dev/null 2>&1; then - if pkg-config --exists "$pkg_config_name" 2>/dev/null; then - echo "✅ $lib_name found via pkg-config" - echo " Version: $(pkg-config --modversion "$pkg_config_name")" - echo " Cflags: $(pkg-config --cflags "$pkg_config_name")" - echo " Libs: $(pkg-config --libs "$pkg_config_name")" - return 0 - fi - fi - - # Test via file search - for path in $search_paths; do - if find "$path" -name "*${lib_name}*" -type f 2>/dev/null | grep -q .; then - echo "✅ $lib_name found in $path" - find "$path" -name "*${lib_name}*" -type f 2>/dev/null | head -3 - return 0 - fi - done - - echo "❌ $lib_name not found in any of: $search_paths" - return 1 -} - -# Function to test CMake package -test_cmake_package() { - local package_name="$1" - local cmake_name="$2" - - echo "Testing CMake package: $package_name" - - # Create a temporary CMakeLists.txt to test package finding - cat > /tmp/test_cmake_${package_name}.cmake << EOF -cmake_minimum_required(VERSION 3.16) -project(test_${package_name}) -find_package(${cmake_name} REQUIRED) -message(STATUS "Found ${cmake_name}: \${${cmake_name}_VERSION}") -EOF - - if cmake -P /tmp/test_cmake_${package_name}.cmake 2>/dev/null; then - echo "✅ CMake package $package_name found" - rm -f /tmp/test_cmake_${package_name}.cmake - return 0 - else - echo "❌ CMake package $package_name not found" - rm -f /tmp/test_cmake_${package_name}.cmake - return 1 - fi -} - -# Function to test compilation -test_compilation() { - local test_name="$1" - local source_code="$2" - local compile_cmd="$3" - - echo "Testing $test_name compilation..." - - # Create temporary source file - local temp_file="/tmp/test_${test_name}.cpp" - echo "$source_code" > "$temp_file" - - if eval "$compile_cmd" 2>/dev/null; then - echo "✅ $test_name compilation successful" - rm -f "$temp_file" "/tmp/test_${test_name}" - return 0 - else - echo "❌ $test_name compilation failed" - rm -f "$temp_file" "/tmp/test_${test_name}" - return 1 - fi -} - -echo "=== Testing Essential Commands ===" - -# Test basic commands -test_command "brew" "" "brew --version" -test_command "cmake" "3.16" "cmake --version" -test_command "protoc" "libprotoc" "protoc --version" -test_command "java" "openjdk" "java -version" -test_command "pkg-config" "" "pkg-config --version" -test_command "python3" "3.9" "python3 --version" -test_command "pip3" "" "pip3 --version" - -echo "=== Testing Libraries ===" - -# Test protobuf -test_library "protobuf" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/protobuf/*/lib" "protobuf" - -# Test boost -test_library "boost" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib" "boost" - -# Test specific boost libraries -echo "Testing specific boost libraries..." -for lib in libboost_system libboost_filesystem libboost_thread libboost_iostreams; do - if find /opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib -name "${lib}*" 2>/dev/null | grep -q .; then - echo "✅ $lib found" - else - echo "❌ $lib not found" - fi -done - -# Test LLVM -test_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llvm@15/*/lib" "llvm" - -# Test other libraries -test_library "zstd" "/opt/homebrew/lib /usr/local/lib" "libzstd" -test_library "pcre2" "/opt/homebrew/lib /usr/local/lib" "libpcre2-8" - -echo "=== Testing CMake Packages ===" - -# Test CMake package finding -test_cmake_package "protobuf" "Protobuf" -test_cmake_package "boost" "Boost" -test_cmake_package "llvm" "LLVM" - -echo "=== Testing Compilation ===" - -# Test protobuf compilation -test_compilation "protobuf" \ -"#include -#include -int main() { - std::cout << \"Protobuf test successful\" << std::endl; - return 0; -}" \ -"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -lprotobuf $temp_file -o /tmp/test_protobuf" - -# Test boost compilation -test_compilation "boost" \ -"#include -#include -#include -int main() { - std::cout << \"Boost version: \" << BOOST_VERSION << std::endl; - std::cout << \"Boost filesystem test: \" << boost::filesystem::current_path() << std::endl; - return 0; -}" \ -"g++ -std=c++17 -I/opt/homebrew/include -I/usr/local/include -L/opt/homebrew/lib -L/usr/local/lib -L/opt/homebrew/Cellar/boost/*/lib -lboost_system -lboost_filesystem $temp_file -o /tmp/test_boost" - -echo "=== Testing Environment Variables ===" - -# Test important environment variables -echo "PATH: $PATH" -echo "PKG_CONFIG_PATH: ${PKG_CONFIG_PATH:-not set}" -echo "CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH:-not set}" -echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH:-not set}" - -# Test brew paths -echo "Brew prefix: $(brew --prefix 2>/dev/null || echo 'not found')" -echo "Brew cellar: $(brew --cellar 2>/dev/null || echo 'not found')" - -echo "=== Testing Python Environment ===" - -# Test Python packages -python3 -c " -import sys -print(f'Python version: {sys.version}') -print(f'Python executable: {sys.executable}') - -# Test required packages -required_packages = ['cloudpickle', 'numpy'] -for pkg in required_packages: - try: - module = __import__(pkg) - print(f'✅ {pkg}: {module.__version__}') - except ImportError as e: - print(f'❌ {pkg}: {e}') -" - -echo "=== Testing CMake Configuration ===" - -# Test CMake configuration with our project -if [ -f "tuplex/CMakeLists.txt" ]; then - echo "Testing CMake configuration..." - - # Create a test build directory - mkdir -p /tmp/cmake_test_build - cd /tmp/cmake_test_build - - # Test basic CMake configuration - local project_dir - project_dir=$(find /Users/runner/work -name "tuplex" -type d 2>/dev/null | head -1 || echo ".") - if [ -d "$project_dir/tuplex" ]; then - if cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON "$project_dir/tuplex" 2>/dev/null; then - echo "✅ CMake configuration successful" - else - echo "❌ CMake configuration failed" - echo "CMake error output:" - cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON "$project_dir/tuplex" 2>&1 | head -20 - fi - else - echo "⚠️ Project directory not found, skipping CMake test" - fi - - cd - >/dev/null - rm -rf /tmp/cmake_test_build -else - echo "⚠️ tuplex/CMakeLists.txt not found, skipping CMake test" -fi - -echo "=== Final Summary ===" -echo "Build environment test completed at $(date)" - -# Count successes and failures -echo "Test results summary:" -echo "✅ Successful tests: $(grep -c '✅' <<< "$(cat)")" -echo "❌ Failed tests: $(grep -c '❌' <<< "$(cat)")" -echo "⚠️ Warnings: $(grep -c '⚠️' <<< "$(cat)")" From 2945c53e1df653e1f86a1c39669a1ef81933fd0f Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:33:32 -0700 Subject: [PATCH 60/92] another try --- scripts/macos/brew_dependencies.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 3b5ae26ea..22c9f1b30 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -144,6 +144,10 @@ echo "Updated PATH: $PATH" # Comprehensive verification echo "=== Verifying installations ===" +brew install tree +tree /opt/homebrew/lib + + # Verify essential commands verify_command "protoc" "libprotoc" verify_command "cmake" "cmake version" From f0f6d139be6137df9f73f28dc91323fd966446c1 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:39:53 -0700 Subject: [PATCH 61/92] test again... --- .github/workflows/build_wheels.yml | 12 ++++++------ scripts/macos/brew_dependencies.sh | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7e39d1ac4..aed09ec2d 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -104,18 +104,18 @@ jobs: echo "Home directory: $HOME" # Install dependencies with comprehensive verification - bash ./scripts/macos/brew_dependencies.sh - bash ./scripts/macos/install_antlr4_cpp_runtime.sh - bash ./scripts/macos/install_aws-sdk-cpp.sh + PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh + PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh + PREFIX=/opt bash ./scripts/macos/install_aws-sdk-cpp.sh # Set up Java environment echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile export PATH="/usr/local/opt/openjdk@11/bin:$PATH" # Update environment variables for the build - export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" - export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" - export CMAKE_PREFIX_PATH="/opt/homebrew:/usr/local:$CMAKE_PREFIX_PATH" + export PATH="/opt/homebrew/bin:/opt/bin:/usr/local/bin:$PATH" + export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/opt/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" + export CMAKE_PREFIX_PATH="/opt/homebrew:/opt:/usr/local:$CMAKE_PREFIX_PATH" echo "=== macOS dependency installation completed ===" diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 22c9f1b30..974427c75 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -144,9 +144,9 @@ echo "Updated PATH: $PATH" # Comprehensive verification echo "=== Verifying installations ===" -brew install tree -tree /opt/homebrew/lib - +# For debugging +# brew install tree +# tree /opt/homebrew/lib # Verify essential commands verify_command "protoc" "libprotoc" From f9d71e8b35646ce27b988c7f984afe09317a329a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:45:01 -0700 Subject: [PATCH 62/92] fix --- .github/workflows/build_wheels.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index aed09ec2d..e47f43df1 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -103,6 +103,12 @@ jobs: echo "User: $(whoami)" echo "Home directory: $HOME" + # Create /opt directory with correct permissions + USER=$(whoami) + sudo mkdir -p /opt + sudo chown $USER /opt + touch /opt/test + # Install dependencies with comprehensive verification PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh From 55c2f594df108edc165d9e368334bc1124ea2f75 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:55:44 -0700 Subject: [PATCH 63/92] less build --- .github/workflows/build_wheels.yml | 120 ++++++++++++++--------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e47f43df1..1f253a297 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -22,66 +22,66 @@ jobs: os: [ ubuntu-latest, macos-13, macos-14 ] python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] include: - - os: ubuntu-latest - python-version: "3.9" - cibw-build: "cp39-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.10" - cibw-build: "cp310-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.11" - cibw-build: "cp311-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.12" - cibw-build: "cp312-manylinux_x86_64" - - os: ubuntu-latest - python-version: "3.13" - cibw-build: "cp313-manylinux_x86_64" - - os: macos-13 - python-version: "3.9" - cibw-build: "cp39-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.10" - cibw-build: "cp310-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.11" - cibw-build: "cp311-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.12" - cibw-build: "cp312-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-13 - python-version: "3.13" - cibw-build: "cp313-macosx_x86_64" - macos-target: "13.6" - arch: "x86_64" - - os: macos-14 - python-version: "3.9" - cibw-build: "cp39-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.10" - cibw-build: "cp310-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.11" - cibw-build: "cp311-macosx_arm64" - macos-target: "14.0" - arch: "arm64" - - os: macos-14 - python-version: "3.12" - cibw-build: "cp312-macosx_arm64" - macos-target: "14.0" - arch: "arm64" + # - os: ubuntu-latest + # python-version: "3.9" + # cibw-build: "cp39-manylinux_x86_64" + # - os: ubuntu-latest + # python-version: "3.10" + # cibw-build: "cp310-manylinux_x86_64" + # - os: ubuntu-latest + # python-version: "3.11" + # cibw-build: "cp311-manylinux_x86_64" + # - os: ubuntu-latest + # python-version: "3.12" + # cibw-build: "cp312-manylinux_x86_64" + # - os: ubuntu-latest + # python-version: "3.13" + # cibw-build: "cp313-manylinux_x86_64" + # - os: macos-13 + # python-version: "3.9" + # cibw-build: "cp39-macosx_x86_64" + # macos-target: "13.6" + # arch: "x86_64" + # - os: macos-13 + # python-version: "3.10" + # cibw-build: "cp310-macosx_x86_64" + # macos-target: "13.6" + # arch: "x86_64" + # - os: macos-13 + # python-version: "3.11" + # cibw-build: "cp311-macosx_x86_64" + # macos-target: "13.6" + # arch: "x86_64" + # - os: macos-13 + # python-version: "3.12" + # cibw-build: "cp312-macosx_x86_64" + # macos-target: "13.6" + # arch: "x86_64" + # - os: macos-13 + # python-version: "3.13" + # cibw-build: "cp313-macosx_x86_64" + # macos-target: "13.6" + # arch: "x86_64" + # - os: macos-14 + # python-version: "3.9" + # cibw-build: "cp39-macosx_arm64" + # macos-target: "14.0" + # arch: "arm64" + # - os: macos-14 + # python-version: "3.10" + # cibw-build: "cp310-macosx_arm64" + # macos-target: "14.0" + # arch: "arm64" + # - os: macos-14 + # python-version: "3.11" + # cibw-build: "cp311-macosx_arm64" + # macos-target: "14.0" + # arch: "arm64" + # - os: macos-14 + # python-version: "3.12" + # cibw-build: "cp312-macosx_arm64" + # macos-target: "14.0" + # arch: "arm64" - os: macos-14 python-version: "3.13" cibw-build: "cp313-macosx_arm64" From 298b1ff0800e75818cb3c61cf2e01ff829b9c95a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 19:58:15 -0700 Subject: [PATCH 64/92] restrict --- .github/workflows/build_wheels.yml | 65 +----------------------------- 1 file changed, 1 insertion(+), 64 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 1f253a297..a42776006 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,71 +17,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-14 (which is macos-latest) is ARM only. macos-13 is the latest intel runner. - # libunwind from brew has 13.6 as requirement right now, update the 13 runners accordingly. - os: [ ubuntu-latest, macos-13, macos-14 ] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # Build only Python 3.13 for macOS 14 include: - # - os: ubuntu-latest - # python-version: "3.9" - # cibw-build: "cp39-manylinux_x86_64" - # - os: ubuntu-latest - # python-version: "3.10" - # cibw-build: "cp310-manylinux_x86_64" - # - os: ubuntu-latest - # python-version: "3.11" - # cibw-build: "cp311-manylinux_x86_64" - # - os: ubuntu-latest - # python-version: "3.12" - # cibw-build: "cp312-manylinux_x86_64" - # - os: ubuntu-latest - # python-version: "3.13" - # cibw-build: "cp313-manylinux_x86_64" - # - os: macos-13 - # python-version: "3.9" - # cibw-build: "cp39-macosx_x86_64" - # macos-target: "13.6" - # arch: "x86_64" - # - os: macos-13 - # python-version: "3.10" - # cibw-build: "cp310-macosx_x86_64" - # macos-target: "13.6" - # arch: "x86_64" - # - os: macos-13 - # python-version: "3.11" - # cibw-build: "cp311-macosx_x86_64" - # macos-target: "13.6" - # arch: "x86_64" - # - os: macos-13 - # python-version: "3.12" - # cibw-build: "cp312-macosx_x86_64" - # macos-target: "13.6" - # arch: "x86_64" - # - os: macos-13 - # python-version: "3.13" - # cibw-build: "cp313-macosx_x86_64" - # macos-target: "13.6" - # arch: "x86_64" - # - os: macos-14 - # python-version: "3.9" - # cibw-build: "cp39-macosx_arm64" - # macos-target: "14.0" - # arch: "arm64" - # - os: macos-14 - # python-version: "3.10" - # cibw-build: "cp310-macosx_arm64" - # macos-target: "14.0" - # arch: "arm64" - # - os: macos-14 - # python-version: "3.11" - # cibw-build: "cp311-macosx_arm64" - # macos-target: "14.0" - # arch: "arm64" - # - os: macos-14 - # python-version: "3.12" - # cibw-build: "cp312-macosx_arm64" - # macos-target: "14.0" - # arch: "arm64" - os: macos-14 python-version: "3.13" cibw-build: "cp313-macosx_arm64" From 7c9c3d2d6796ab2ebe96d49698197ff31f5c1f1c Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:03:50 -0700 Subject: [PATCH 65/92] now test macos13 --- .github/workflows/build_wheels.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a42776006..8bdd1723b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,13 +17,18 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # Build only Python 3.13 for macOS 14 + # Build only Python 3.13 for macOS 13 include: - - os: macos-14 + # - os: macos-14 + # python-version: "3.13" + # cibw-build: "cp313-macosx_arm64" + # macos-target: "14.0" + # arch: "arm64" + - os: macos-13 python-version: "3.13" - cibw-build: "cp313-macosx_arm64" - macos-target: "14.0" - arch: "arm64" + cibw-build: "cp313-macosx_x86_64" + macos-target: "13.6" + arch: "x86_64" steps: - name: Checkout repository uses: actions/checkout@v4 From 3532a88b805c65de4c85efd69f7c77f14770a73b Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:04:12 -0700 Subject: [PATCH 66/92] test macos13 --- scripts/macos/brew_dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 974427c75..8239421e3 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -145,8 +145,8 @@ echo "Updated PATH: $PATH" echo "=== Verifying installations ===" # For debugging -# brew install tree -# tree /opt/homebrew/lib +brew install tree +tree /opt/homebrew/lib # Verify essential commands verify_command "protoc" "libprotoc" From 1624de62839fcd397398ba9fb62c59718e59b59a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:10:03 -0700 Subject: [PATCH 67/92] suuuuudooooo --- .github/workflows/build_wheels.yml | 3 +++ scripts/macos/brew_dependencies.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8bdd1723b..30cc8fd5b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -51,6 +51,9 @@ jobs: sudo chown $USER /opt touch /opt/test + brew install tree + sudo tree /opt/homebrew/lib + # Install dependencies with comprehensive verification PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 8239421e3..f90cbcd23 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -146,7 +146,7 @@ echo "=== Verifying installations ===" # For debugging brew install tree -tree /opt/homebrew/lib +sudo tree /opt/homebrew/lib # Verify essential commands verify_command "protoc" "libprotoc" From 9ccd7778928cd27625ad884398f3334d40ed2d9a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:14:25 -0700 Subject: [PATCH 68/92] debug --- .github/workflows/build_wheels.yml | 5 +++-- scripts/macos/brew_dependencies.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 30cc8fd5b..39517c764 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -51,8 +51,9 @@ jobs: sudo chown $USER /opt touch /opt/test - brew install tree - sudo tree /opt/homebrew/lib + ls /opt/homebrew + ls /opt/homebrew/lib + find /opt/homebrew/lib -maxdepth 3 -mindepth 1 -print # Install dependencies with comprehensive verification PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index f90cbcd23..8f84122da 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -145,8 +145,9 @@ echo "Updated PATH: $PATH" echo "=== Verifying installations ===" # For debugging -brew install tree -sudo tree /opt/homebrew/lib +echo "Listing /opt/homebrew/lib (2-3 levels deep):" +find /opt/homebrew/lib -maxdepth 3 -mindepth 1 -print + # Verify essential commands verify_command "protoc" "libprotoc" From a05aa2e1fcb82a3ad2d33fee2f84910b79aecf46 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:17:02 -0700 Subject: [PATCH 69/92] moar --- .github/workflows/build_wheels.yml | 4 ---- scripts/macos/brew_dependencies.sh | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 39517c764..8bdd1723b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -51,10 +51,6 @@ jobs: sudo chown $USER /opt touch /opt/test - ls /opt/homebrew - ls /opt/homebrew/lib - find /opt/homebrew/lib -maxdepth 3 -mindepth 1 -print - # Install dependencies with comprehensive verification PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 8f84122da..e7a9349e1 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -147,6 +147,8 @@ echo "=== Verifying installations ===" # For debugging echo "Listing /opt/homebrew/lib (2-3 levels deep):" find /opt/homebrew/lib -maxdepth 3 -mindepth 1 -print +echo "printing /usr/local/lib:" +find /usr/local/lib -maxdepth 3 -mindepth 1 -print # Verify essential commands From f2262d21af98627002ee97a50593cdf393959835 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:27:42 -0700 Subject: [PATCH 70/92] intel based brew --- .github/workflows/build_wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8bdd1723b..d74affae8 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -44,6 +44,8 @@ jobs: echo "Current working directory: $(pwd)" echo "User: $(whoami)" echo "Home directory: $HOME" + echo "Brew cellar directory: $(brew --cellar)" + echo "Brew prefix directory: $(brew --prefix)" # Create /opt directory with correct permissions USER=$(whoami) @@ -51,6 +53,8 @@ jobs: sudo chown $USER /opt touch /opt/test + # Under intel based macs, the cellar dir is /usr/local/Cellar whereas Apple Silicon based macs have it in /opt/homebrew/Cellar + # Install dependencies with comprehensive verification PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh PREFIX=/opt bash ./scripts/macos/install_antlr4_cpp_runtime.sh From 1003c6218ac58cebfffb05c5203aa3ade7cd3026 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:48:03 -0700 Subject: [PATCH 71/92] test this --- .github/workflows/build_wheels.yml | 43 +++++++++++++++++++++++++++++- scripts/macos/brew_dependencies.sh | 42 +++++++++++++++++++---------- 2 files changed, 70 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d74affae8..278ad6f44 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -53,7 +53,48 @@ jobs: sudo chown $USER /opt touch /opt/test - # Under intel based macs, the cellar dir is /usr/local/Cellar whereas Apple Silicon based macs have it in /opt/homebrew/Cellar + # Handle different Homebrew cellar directories between Apple Silicon and x86_64 + # Apple Silicon: /opt/homebrew/Cellar + # Intel x86_64: /usr/local/Cellar + + HOMEBREW_PREFIX="$(brew --prefix)" + HOMEBREW_CELLAR="$(brew --cellar)" + ARCH="$(uname -m)" + + echo "Homebrew prefix: $HOMEBREW_PREFIX" + echo "Homebrew cellar: $HOMEBREW_CELLAR" + echo "Architecture: $ARCH" + + # Ensure /opt/homebrew exists and points to the correct location + if [[ "$ARCH" == "arm64" ]]; then + # Apple Silicon - Homebrew should already be in /opt/homebrew + echo "Apple Silicon detected - using native /opt/homebrew" + if [ ! -d /opt/homebrew ]; then + echo "ERROR: /opt/homebrew not found on Apple Silicon!" + exit 1 + fi + else + # Intel x86_64 - Homebrew is in /usr/local, create symlink to /opt/homebrew + echo "Intel x86_64 detected - creating /opt/homebrew symlink" + if [ -L /opt/homebrew ]; then + echo "/opt/homebrew already exists as a symlink" + elif [ -d /opt/homebrew ]; then + echo "Removing existing /opt/homebrew directory" + sudo rm -rf /opt/homebrew + fi + + # Create symlink from /opt/homebrew to actual Homebrew prefix + sudo ln -s "$HOMEBREW_PREFIX" /opt/homebrew + echo "Created symlink: /opt/homebrew -> $HOMEBREW_PREFIX" + fi + + # Verify the setup + echo "Verifying Homebrew setup:" + echo " /opt/homebrew exists: $([ -e /opt/homebrew ] && echo 'YES' || echo 'NO')" + echo " /opt/homebrew/bin exists: $([ -d /opt/homebrew/bin ] && echo 'YES' || echo 'NO')" + echo " /opt/homebrew/lib exists: $([ -d /opt/homebrew/lib ] && echo 'YES' || echo 'NO')" + echo " /opt/homebrew/Cellar exists: $([ -d /opt/homebrew/Cellar ] && echo 'YES' || echo 'NO')" + # Install dependencies with comprehensive verification PREFIX=/opt bash ./scripts/macos/brew_dependencies.sh diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index e7a9349e1..6fa36ab03 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -74,12 +74,16 @@ verify_cmake_package() { local package_name="$1" echo "Verifying CMake package: $package_name" + # Get Homebrew paths dynamically + local homebrew_prefix="${HOMEBREW_PREFIX:-$(brew --prefix)}" + local homebrew_cellar="${HOMEBREW_CELLAR:-$(brew --cellar)}" + # Search in standard locations and Cellar paths # Use find to get all CMake directories - local expanded_paths="/opt/homebrew/lib/cmake /usr/local/lib/cmake" + local expanded_paths="$homebrew_prefix/lib/cmake /usr/local/lib/cmake" # Add Cellar paths using find to get actual paths - cellar_cmake_paths=$(find /opt/homebrew/Cellar -name "cmake" -type d -path "*/lib/cmake" 2>/dev/null) + cellar_cmake_paths=$(find "$homebrew_cellar" -name "cmake" -type d -path "*/lib/cmake" 2>/dev/null) if [ -n "$cellar_cmake_paths" ]; then expanded_paths="$expanded_paths $cellar_cmake_paths" fi @@ -136,19 +140,29 @@ for dep in $CORE_DEPENDENCIES; do fi done -# # Update PATH to include brew binaries +# Detect Homebrew paths dynamically +HOMEBREW_PREFIX="$(brew --prefix)" +HOMEBREW_CELLAR="$(brew --cellar)" +ARCH="$(uname -m)" + +echo "Detected Homebrew configuration:" +echo " Architecture: $ARCH" +echo " Homebrew prefix: $HOMEBREW_PREFIX" +echo " Homebrew cellar: $HOMEBREW_CELLAR" + +# Update PATH to include brew binaries echo "Updating PATH..." -export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" +export PATH="$HOMEBREW_PREFIX/bin:/usr/local/bin:$PATH" echo "Updated PATH: $PATH" # Comprehensive verification echo "=== Verifying installations ===" -# For debugging -echo "Listing /opt/homebrew/lib (2-3 levels deep):" -find /opt/homebrew/lib -maxdepth 3 -mindepth 1 -print -echo "printing /usr/local/lib:" -find /usr/local/lib -maxdepth 3 -mindepth 1 -print +# For debugging - show library paths for both architectures +echo "Listing $HOMEBREW_PREFIX/lib (2-3 levels deep):" +find "$HOMEBREW_PREFIX/lib" -maxdepth 3 -mindepth 1 -print 2>/dev/null || echo "No libraries found in $HOMEBREW_PREFIX/lib" +echo "Listing /usr/local/lib:" +find /usr/local/lib -maxdepth 3 -mindepth 1 -print 2>/dev/null || echo "No libraries found in /usr/local/lib" # Verify essential commands @@ -166,14 +180,14 @@ if verify_command "protoc" "libprotoc"; then else echo "❌ Protobuf verification failed" echo "Searching for protobuf installations..." - find /opt/homebrew /usr/local -name "*protobuf*" 2>/dev/null | head -10 + find "$HOMEBREW_PREFIX" /usr/local -name "*protobuf*" 2>/dev/null | head -10 exit 1 fi -# Verify libraries -verify_library "protobuf" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/protobuf/*/lib" -verify_library "boost" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/boost/*/lib" -verify_library "llvm" "/opt/homebrew/lib /usr/local/lib /opt/homebrew/Cellar/llvm@15/*/lib" +# Verify libraries using dynamic paths +verify_library "protobuf" "$HOMEBREW_PREFIX/lib /usr/local/lib $HOMEBREW_CELLAR/protobuf/*/lib" +verify_library "boost" "$HOMEBREW_PREFIX/lib /usr/local/lib $HOMEBREW_CELLAR/boost/*/lib" +verify_library "llvm" "$HOMEBREW_PREFIX/lib /usr/local/lib $HOMEBREW_CELLAR/llvm@15/*/lib" # Verify cmake packages verify_cmake_package "protobuf" From b77d0b6439c4b314bb7a878a766edcbaf30afdb0 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 20:57:36 -0700 Subject: [PATCH 72/92] test on both macos --- .github/workflows/build_wheels.yml | 11 +++++------ scripts/macos/brew_dependencies.sh | 27 ++++++++++----------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 278ad6f44..a269bb58a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,13 +17,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # Build only Python 3.13 for macOS 13 include: - # - os: macos-14 - # python-version: "3.13" - # cibw-build: "cp313-macosx_arm64" - # macos-target: "14.0" - # arch: "arm64" + - os: macos-14 + python-version: "3.13" + cibw-build: "cp313-macosx_arm64" + macos-target: "14.0" + arch: "arm64" - os: macos-13 python-version: "3.13" cibw-build: "cp313-macosx_x86_64" diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index 6fa36ab03..c5ef7ff7d 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -108,6 +108,16 @@ verify_cmake_package() { return 1 } +# Detect Homebrew paths dynamically +HOMEBREW_PREFIX="$(brew --prefix)" +HOMEBREW_CELLAR="$(brew --cellar)" +ARCH="$(uname -m)" + +echo "Detected Homebrew configuration:" +echo " Architecture: $ARCH" +echo " Homebrew prefix: $HOMEBREW_PREFIX" +echo " Homebrew cellar: $HOMEBREW_CELLAR" + # Update brew first echo "Updating brew..." brew update --quiet 2>/dev/null || echo "Warning: brew update failed" @@ -140,16 +150,6 @@ for dep in $CORE_DEPENDENCIES; do fi done -# Detect Homebrew paths dynamically -HOMEBREW_PREFIX="$(brew --prefix)" -HOMEBREW_CELLAR="$(brew --cellar)" -ARCH="$(uname -m)" - -echo "Detected Homebrew configuration:" -echo " Architecture: $ARCH" -echo " Homebrew prefix: $HOMEBREW_PREFIX" -echo " Homebrew cellar: $HOMEBREW_CELLAR" - # Update PATH to include brew binaries echo "Updating PATH..." export PATH="$HOMEBREW_PREFIX/bin:/usr/local/bin:$PATH" @@ -158,13 +158,6 @@ echo "Updated PATH: $PATH" # Comprehensive verification echo "=== Verifying installations ===" -# For debugging - show library paths for both architectures -echo "Listing $HOMEBREW_PREFIX/lib (2-3 levels deep):" -find "$HOMEBREW_PREFIX/lib" -maxdepth 3 -mindepth 1 -print 2>/dev/null || echo "No libraries found in $HOMEBREW_PREFIX/lib" -echo "Listing /usr/local/lib:" -find /usr/local/lib -maxdepth 3 -mindepth 1 -print 2>/dev/null || echo "No libraries found in /usr/local/lib" - - # Verify essential commands verify_command "protoc" "libprotoc" verify_command "cmake" "cmake version" From fed0550e9266a52028ff11e601b55c835dd93a33 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 21:12:00 -0700 Subject: [PATCH 73/92] try it out --- .github/workflows/build_wheels.yml | 8 +- scripts/macos/install_aws-sdk-cpp.sh | 143 +++++++++++++++++++++------ 2 files changed, 118 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a269bb58a..7323ac786 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -104,10 +104,10 @@ jobs: echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile export PATH="/usr/local/opt/openjdk@11/bin:$PATH" - # Update environment variables for the build - export PATH="/opt/homebrew/bin:/opt/bin:/usr/local/bin:$PATH" - export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/opt/lib/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" - export CMAKE_PREFIX_PATH="/opt/homebrew:/opt:/usr/local:$CMAKE_PREFIX_PATH" + # Update environment variables for the build (handle unbound variables with set -u) + export PATH="/opt/homebrew/bin:/opt/bin:/usr/local/bin:${PATH:-}" + export PKG_CONFIG_PATH="/opt/homebrew/lib/pkgconfig:/opt/lib/pkgconfig:/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + export CMAKE_PREFIX_PATH="/opt/homebrew:/opt:/usr/local:${CMAKE_PREFIX_PATH:-}" echo "=== macOS dependency installation completed ===" diff --git a/scripts/macos/install_aws-sdk-cpp.sh b/scripts/macos/install_aws-sdk-cpp.sh index ea53d00d6..a6eeeca7f 100755 --- a/scripts/macos/install_aws-sdk-cpp.sh +++ b/scripts/macos/install_aws-sdk-cpp.sh @@ -1,48 +1,133 @@ #!/usr/bin/env bash -set -euxo pipefail +set -euo pipefail PREFIX=${PREFIX:-/usr/local} AWSSDK_CPP_VERSION=1.11.524 # need at least 1.11.267 because of pyarrow bugs... -# check if dir exists (i.e. restored from cache, then skip) +# Check if AWS SDK is already installed if [ -d "${PREFIX}/include/aws" ]; then - echo ">> Skip aws-sdk-cpp compile from source, already exists." + echo "✅ AWS SDK C++ already installed, skipping." exit 0 fi -echo ">> installing AWS SDK from source" -CPU_CORES=$(sysctl -n hw.physicalcpu) - -# if macOS is 10.x -> use this as minimum -MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" +echo "📦 Installing AWS SDK C++ v${AWSSDK_CPP_VERSION} from source..." +# Get system info +CPU_CORES=$(sysctl -n hw.physicalcpu) MACOS_VERSION=$(sw_vers -productVersion) -echo "-- processing on MacOS ${MACOS_VERSION}" -function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } - MACOS_VERSION_MAJOR=$(echo "$MACOS_VERSION" | awk -F \. {'print $1'}) -if (( $MACOS_VERSION_MAJOR >= 11 )); then - echo "-- Newer MacOS detected (>=11.0), using more recent base target." - echo "-- Using minimum target ${MACOS_VERSION_MAJOR}.0" + +# Determine minimum deployment target +if (( MACOS_VERSION_MAJOR >= 11 )); then MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_VERSION_MAJOR}.0" + echo " Using macOS ${MACOS_VERSION_MAJOR}.0 as minimum target" else - # keep as is - echo "defaulting build to use as minimum target ${MINIMUM_TARGET}" + MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" + echo " Using macOS 10.13 as minimum target" +fi + +# Create temporary directory +TEMP_DIR="/tmp/aws-sdk-cpp-$$" +mkdir -p "$TEMP_DIR" +cd "$TEMP_DIR" + +# Cleanup function +cleanup() { + echo "🧹 Cleaning up temporary files..." + cd /tmp + rm -rf "$TEMP_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +echo "📥 Cloning AWS SDK C++ repository..." +if ! git clone --recurse-submodules --quiet https://github.com/aws/aws-sdk-cpp.git; then + echo "❌ Failed to clone AWS SDK C++ repository" + exit 1 +fi + +cd aws-sdk-cpp + +echo "🏷️ Checking out version ${AWSSDK_CPP_VERSION}..." +if ! git checkout --quiet "tags/${AWSSDK_CPP_VERSION}"; then + echo "❌ Failed to checkout version ${AWSSDK_CPP_VERSION}" + exit 1 +fi + +echo "🔨 Configuring build..." +mkdir -p build +cd build + +CMAKE_ARGS=( + ${MINIMUM_TARGET} + "-DCMAKE_INSTALL_PREFIX=${PREFIX}" + "-DCMAKE_BUILD_TYPE=Release" + "-DUSE_OPENSSL=ON" + "-DENABLE_TESTING=OFF" + "-DENABLE_UNITY_BUILD=ON" + "-DCPP_STANDARD=17" + "-DBUILD_SHARED_LIBS=OFF" + "-DBUILD_ONLY=s3;core;lambda;transfer" + ".." +) + +if ! cmake "${CMAKE_ARGS[@]}" >/dev/null 2>&1; then + echo "❌ CMake configuration failed" + echo "CMake arguments: ${CMAKE_ARGS[*]}" + exit 1 fi -cd /tmp && - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git && - cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && mkdir build && pushd build && - cmake ${MINIMUM_TARGET} -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;core;lambda;transfer" .. && - make -j${CPU_CORES} && - make install && - popd && - cd - || { - echo ">> error: AWS SDK installation failed" - echo "Current directory: $(pwd)" - echo "Build directory contents:" - ls -la build/ 2>/dev/null || echo "Build directory not found" +echo "🔨 Building AWS SDK C++ (using ${CPU_CORES} cores)..." +echo " This may take several minutes. Building in progress..." + +# Function to show progress dots +show_progress() { + local pid=$1 + local dots=0 + while kill -0 "$pid" 2>/dev/null; do + sleep 10 + dots=$((dots + 1)) + if [ $dots -eq 1 ]; then + echo " ⏳ Still building... (10s elapsed)" + elif [ $dots -eq 3 ]; then + echo " ⏳ Still building... (30s elapsed)" + elif [ $dots -eq 6 ]; then + echo " ⏳ Still building... (1m elapsed)" + elif [ $dots -eq 12 ]; then + echo " ⏳ Still building... (2m elapsed)" + elif [ $dots -eq 18 ]; then + echo " ⏳ Still building... (3m elapsed)" + elif [ $dots -eq 30 ]; then + echo " ⏳ Still building... (5m elapsed)" + elif [ $((dots % 12)) -eq 0 ]; then + echo " ⏳ Still building... ($((dots * 10 / 60))m elapsed)" + fi + done +} + +# Start the build in background and show progress +make -j"${CPU_CORES}" >/dev/null 2>&1 & +BUILD_PID=$! + +# Show progress while building +show_progress $BUILD_PID + +# Wait for build to complete and check result +wait $BUILD_PID +BUILD_EXIT_CODE=$? + +if [ $BUILD_EXIT_CODE -ne 0 ]; then + echo "❌ Build failed" exit 1 - } +fi + +echo " ✅ Build completed successfully!" + +echo "📦 Installing AWS SDK C++..." +if ! make install >/dev/null 2>&1; then + echo "❌ Installation failed" + exit 1 +fi + +echo "✅ AWS SDK C++ v${AWSSDK_CPP_VERSION} installed successfully!" From 653c64a5d52a8fb675b2073bd5c6a78b3c1bdbf3 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 21:20:54 -0700 Subject: [PATCH 74/92] less vibe coding --- scripts/macos/brew_dependencies.sh | 26 ++++++++-------- scripts/macos/install_aws-sdk-cpp.sh | 44 ++++++++++++++-------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh index c5ef7ff7d..167caa7e9 100755 --- a/scripts/macos/brew_dependencies.sh +++ b/scripts/macos/brew_dependencies.sh @@ -14,13 +14,13 @@ verify_command() { if [ -n "$expected_output" ]; then local actual_output actual_output=$($cmd --version 2>/dev/null || echo "version check failed") - echo "✅ $cmd found: $actual_output" + echo "$cmd found: $actual_output" else - echo "✅ $cmd found" + echo "$cmd found" fi return 0 else - echo "❌ $cmd not found in PATH" + echo "ERROR: $cmd not found in PATH" return 1 fi } @@ -59,13 +59,13 @@ verify_library() { found_files=$(find "$path" -name "*${lib_name}*" -type f 2>/dev/null) echo " Found $(echo "$found_files" | wc -l) files" if [ -n "$found_files" ]; then - echo "✅ $lib_name found in $path" + echo "$lib_name found in $path" echo "$found_files" | head -3 return 0 fi done - echo "❌ $lib_name not found in any of: $search_paths" + echo "ERROR: $lib_name not found in any of: $search_paths" return 1 } @@ -98,13 +98,13 @@ verify_cmake_package() { local found_packages found_packages=$(find "$path" -name "*${package_name}*" 2>/dev/null) if [ -n "$found_packages" ]; then - echo "✅ CMake package $package_name found" + echo "CMake package $package_name found" echo "$found_packages" | head -3 return 0 fi done - echo "❌ CMake package $package_name not found" + echo "ERROR: CMake package $package_name not found" return 1 } @@ -132,19 +132,19 @@ for dep in $CORE_DEPENDENCIES; do # Check if package is already installed if brew list "$dep" &>/dev/null; then version=$(brew list --versions "$dep" | awk '{print $2}') - echo "✅ $dep $version already installed, skipping." + echo "$dep $version already installed, skipping." else if brew install "$dep" 2>/dev/null; then version=$(brew list --versions "$dep" | awk '{print $2}') - echo "✅ $dep $version installed successfully." + echo "$dep $version installed successfully." else - echo "❌ Failed to install $dep" + echo "ERROR: Failed to install $dep" echo "Attempting to reinstall $dep with --force ..." if brew reinstall -f "$dep" 2>/dev/null; then version=$(brew list --versions "$dep" | awk '{print $2}') - echo "✅ $dep $version reinstalled successfully." + echo "$dep $version reinstalled successfully." else - echo "❌ Failed to reinstall $dep, continuing..." + echo "ERROR: Failed to reinstall $dep, continuing..." fi fi fi @@ -171,7 +171,7 @@ if verify_command "protoc" "libprotoc"; then echo "Protobuf include path: $(pkg-config --variable=includedir protobuf 2>/dev/null || echo 'not found')" echo "Protobuf lib path: $(pkg-config --variable=libdir protobuf 2>/dev/null || echo 'not found')" else - echo "❌ Protobuf verification failed" + echo "ERROR: Protobuf verification failed" echo "Searching for protobuf installations..." find "$HOMEBREW_PREFIX" /usr/local -name "*protobuf*" 2>/dev/null | head -10 exit 1 diff --git a/scripts/macos/install_aws-sdk-cpp.sh b/scripts/macos/install_aws-sdk-cpp.sh index a6eeeca7f..04c8a86cd 100755 --- a/scripts/macos/install_aws-sdk-cpp.sh +++ b/scripts/macos/install_aws-sdk-cpp.sh @@ -7,11 +7,11 @@ AWSSDK_CPP_VERSION=1.11.524 # need at least 1.11.267 because of pyarrow bugs... # Check if AWS SDK is already installed if [ -d "${PREFIX}/include/aws" ]; then - echo "✅ AWS SDK C++ already installed, skipping." + echo "AWS SDK C++ already installed, skipping." exit 0 fi -echo "📦 Installing AWS SDK C++ v${AWSSDK_CPP_VERSION} from source..." +echo "Installing AWS SDK C++ v${AWSSDK_CPP_VERSION} from source..." # Get system info CPU_CORES=$(sysctl -n hw.physicalcpu) @@ -34,27 +34,27 @@ cd "$TEMP_DIR" # Cleanup function cleanup() { - echo "🧹 Cleaning up temporary files..." + echo "Cleaning up temporary files..." cd /tmp rm -rf "$TEMP_DIR" 2>/dev/null || true } trap cleanup EXIT -echo "📥 Cloning AWS SDK C++ repository..." +echo "Cloning AWS SDK C++ repository..." if ! git clone --recurse-submodules --quiet https://github.com/aws/aws-sdk-cpp.git; then - echo "❌ Failed to clone AWS SDK C++ repository" + echo "ERROR: Failed to clone AWS SDK C++ repository" exit 1 fi cd aws-sdk-cpp -echo "🏷️ Checking out version ${AWSSDK_CPP_VERSION}..." +echo "Checking out version ${AWSSDK_CPP_VERSION}..." if ! git checkout --quiet "tags/${AWSSDK_CPP_VERSION}"; then - echo "❌ Failed to checkout version ${AWSSDK_CPP_VERSION}" + echo "ERROR: Failed to checkout version ${AWSSDK_CPP_VERSION}" exit 1 fi -echo "🔨 Configuring build..." +echo "Configuring build..." mkdir -p build cd build @@ -72,12 +72,12 @@ CMAKE_ARGS=( ) if ! cmake "${CMAKE_ARGS[@]}" >/dev/null 2>&1; then - echo "❌ CMake configuration failed" + echo "ERROR: CMake configuration failed" echo "CMake arguments: ${CMAKE_ARGS[*]}" exit 1 fi -echo "🔨 Building AWS SDK C++ (using ${CPU_CORES} cores)..." +echo "Building AWS SDK C++ (using ${CPU_CORES} cores)..." echo " This may take several minutes. Building in progress..." # Function to show progress dots @@ -88,19 +88,19 @@ show_progress() { sleep 10 dots=$((dots + 1)) if [ $dots -eq 1 ]; then - echo " ⏳ Still building... (10s elapsed)" + echo " Still building... (10s elapsed)" elif [ $dots -eq 3 ]; then - echo " ⏳ Still building... (30s elapsed)" + echo " Still building... (30s elapsed)" elif [ $dots -eq 6 ]; then - echo " ⏳ Still building... (1m elapsed)" + echo " Still building... (1m elapsed)" elif [ $dots -eq 12 ]; then - echo " ⏳ Still building... (2m elapsed)" + echo " Still building... (2m elapsed)" elif [ $dots -eq 18 ]; then - echo " ⏳ Still building... (3m elapsed)" + echo " Still building... (3m elapsed)" elif [ $dots -eq 30 ]; then - echo " ⏳ Still building... (5m elapsed)" + echo " Still building... (5m elapsed)" elif [ $((dots % 12)) -eq 0 ]; then - echo " ⏳ Still building... ($((dots * 10 / 60))m elapsed)" + echo " Still building... ($((dots * 10 / 60))m elapsed)" fi done } @@ -117,17 +117,17 @@ wait $BUILD_PID BUILD_EXIT_CODE=$? if [ $BUILD_EXIT_CODE -ne 0 ]; then - echo "❌ Build failed" + echo "ERROR: Build failed" exit 1 fi -echo " ✅ Build completed successfully!" +echo " Build completed successfully!" -echo "📦 Installing AWS SDK C++..." +echo "Installing AWS SDK C++..." if ! make install >/dev/null 2>&1; then - echo "❌ Installation failed" + echo "ERROR: Installation failed" exit 1 fi -echo "✅ AWS SDK C++ v${AWSSDK_CPP_VERSION} installed successfully!" +echo "AWS SDK C++ v${AWSSDK_CPP_VERSION} installed successfully!" From a7c0e1a886c90b85241103cc9e580b3050076627 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 21:22:54 -0700 Subject: [PATCH 75/92] another try --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7323ac786..f4d1a69c6 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -138,7 +138,7 @@ jobs: # To avoid issues, simply use 13.6 for now. # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ - CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" + CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib -DLLVM_ROOT=/opt/homebrew/opt/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" # ensure build requirements are available during wheel building CIBW_BUILD_REQUIRES: "cloudpickle numpy" From ba4471a2bd714be5df83baebd3ba21b8d4a4b76b Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 21:41:53 -0700 Subject: [PATCH 76/92] fix --- .github/workflows/build_wheels.yml | 3 --- pyproject.toml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f4d1a69c6..36a33d9be 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -139,9 +139,6 @@ jobs: # Fix for Java home from https://github.com/actions/runner-images/discussions/9266. # For github actions, $HOME is /Users/runner/ CIBW_ENVIRONMENT_MACOS: "ARCH=${{ matrix.arch }} PREFIX=${HOME}/.local MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos-target }} PATH=/opt/homebrew/bin:/usr/local/bin:/usr/local/opt/openjdk@11/bin:$PATH PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig CMAKE_PREFIX_PATH=/opt/homebrew:/usr/local:/Users/runner/.local CMAKE_ARGS='-DCMAKE_PREFIX_PATH=/Users/runner/.local:/opt/homebrew:/usr/local -DCMAKE_MODULE_PATH=/Users/runner/.local/cmake/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DProtobuf_ROOT=/opt/homebrew -DBoost_ROOT=/opt/homebrew -DBoost_INCLUDE_DIR=/opt/homebrew/include -DBoost_LIBRARY_DIR=/opt/homebrew/lib -DLLVM_ROOT=/opt/homebrew/opt/llvm@15' JAVA_HOME=${JAVA_HOME_11_X64:-$JAVA_HOME_11_arm64}" - - # ensure build requirements are available during wheel building - CIBW_BUILD_REQUIRES: "cloudpickle numpy" # run all python tests to make sure wheels are not defunct CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter cloudpickle" diff --git a/pyproject.toml b/pyproject.toml index aefc4e5dc..157878123 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,3 +12,7 @@ requires = [ "requests" ] build-backend = "setuptools.build_meta" + +# Ensure cloudpickle and numpy are installed as build-time dependencies when building the wheel. +[tool.cibuildwheel] +before-build = "pip install cloudpickle numpy" \ No newline at end of file From 106ff992d3c5d780bc3a7b57827c9897b6430692 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 5 Sep 2025 23:35:45 -0700 Subject: [PATCH 77/92] more verbosity to find issue --- pyproject.toml | 15 +++++---- tuplex/test/CMakeLists.txt | 62 ++++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 157878123..fdc917f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,21 @@ [build-system] requires = [ - "setuptools>=42", - "pip>9", + "setuptools", + "pip", "wheel", "cloudpickle", "numpy", - "ninja; sys_platform != 'win32'", + "ninja;sys_platform!='win32'", "cmake>=3.25", - "delocate; sys.platform == 'darwin'", - "auditwheel; sys.platform == 'linux'", + "delocate;sys.platform=='darwin'", + "auditwheel;sys.platform=='linux'", "requests" ] build-backend = "setuptools.build_meta" # Ensure cloudpickle and numpy are installed as build-time dependencies when building the wheel. [tool.cibuildwheel] -before-build = "pip install cloudpickle numpy" \ No newline at end of file +before-build = [ + "pip install --upgrade pip setuptools wheel", + "pip install cloudpickle numpy" +] diff --git a/tuplex/test/CMakeLists.txt b/tuplex/test/CMakeLists.txt index 12d571a07..c20c484bb 100755 --- a/tuplex/test/CMakeLists.txt +++ b/tuplex/test/CMakeLists.txt @@ -16,18 +16,68 @@ find_package(Threads REQUIRED) #endif() if (Python3_FOUND) + message(STATUS "Python3 executable: ${Python3_EXECUTABLE}") + message(STATUS "Python3 version: ${Python3_VERSION}") + + # First, let's see what packages are actually installed + message(STATUS "Checking installed Python packages...") + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import pkg_resources; installed = [d.project_name for d in pkg_resources.working_set]; print('\\n'.join(sorted(installed)))" + OUTPUT_VARIABLE installed_packages + RESULT_VARIABLE ret_packages + ) + if("${ret_packages}" STREQUAL "0") + message(STATUS "Installed packages:") + message(STATUS "${installed_packages}") + else() + message(WARNING "Could not list installed packages") + endif() + + # Check Python path and site-packages + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import sys; print('\\n'.join(sys.path))" + OUTPUT_VARIABLE python_path + RESULT_VARIABLE ret_path + ) + if("${ret_path}" STREQUAL "0") + message(STATUS "Python sys.path:") + message(STATUS "${python_path}") + endif() + # check that cloudpickle is installed via import - set(cmd -c "import cloudpickle") - execute_process(COMMAND ${Python3_EXECUTABLE} ${cmd} RESULT_VARIABLE ret) + message(STATUS "Testing cloudpickle import...") + set(cmd -c "import cloudpickle; print('cloudpickle version:', cloudpickle.__version__)") + execute_process( + COMMAND ${Python3_EXECUTABLE} ${cmd} + OUTPUT_VARIABLE cloudpickle_output + ERROR_VARIABLE cloudpickle_error + RESULT_VARIABLE ret + ) if(NOT "${ret}" STREQUAL "0") - message(FATAL_ERROR "Could not find cloudpickle module, please install via pip3 install cloudpickle.") + message(STATUS "cloudpickle import failed with return code: ${ret}") + message(STATUS "cloudpickle stdout: ${cloudpickle_output}") + message(STATUS "cloudpickle stderr: ${cloudpickle_error}") + message(FATAL_ERROR "Could not find cloudpickle module. Please install via 'pip3 install cloudpickle'. The module is required for testing. Check the output above for details about what packages are currently installed.") + else() + message(STATUS "cloudpickle import successful: ${cloudpickle_output}") endif() # check that numpy is installed too for testing purposes... - set(cmd -c "import numpy") - execute_process(COMMAND ${Python3_EXECUTABLE} ${cmd} RESULT_VARIABLE ret) + message(STATUS "Testing numpy import...") + set(cmd -c "import numpy; print('numpy version:', numpy.__version__)") + execute_process( + COMMAND ${Python3_EXECUTABLE} ${cmd} + OUTPUT_VARIABLE numpy_output + ERROR_VARIABLE numpy_error + RESULT_VARIABLE ret + ) if(NOT "${ret}" STREQUAL "0") - message(FATAL_ERROR "Could not find numpy module, please install via pip3 install numpy.") + message(STATUS "numpy import failed with return code: ${ret}") + message(STATUS "numpy stdout: ${numpy_output}") + message(STATUS "numpy stderr: ${numpy_error}") + message(FATAL_ERROR "Could not find numpy module. Please install via 'pip3 install numpy'. The module is required for testing. Check the output above for details about what packages are currently installed.") + else() + message(STATUS "numpy import successful: ${numpy_output}") endif() else() From e9dc1156f966a6c3f5768225ff73967d869a0dea Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 6 Sep 2025 00:06:54 -0700 Subject: [PATCH 78/92] try older cibuildwheel --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 36a33d9be..df5a9d663 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -118,7 +118,7 @@ jobs: - name: Install cibuildwheel # by default use the latest version, if there are issues use 3.1.4. - run: python -m pip install cibuildwheel + run: python -m pip install cibuildwheel==2.22.0 - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse From 5789ce848c53014fe955ba46210122be038467e2 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 02:42:40 -0700 Subject: [PATCH 79/92] try --- scripts/docker/Dockerfile.alpine.minimal | 11 +++++++++++ tuplex/python/CMakeLists.txt | 20 ++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 scripts/docker/Dockerfile.alpine.minimal diff --git a/scripts/docker/Dockerfile.alpine.minimal b/scripts/docker/Dockerfile.alpine.minimal new file mode 100644 index 000000000..7adb48882 --- /dev/null +++ b/scripts/docker/Dockerfile.alpine.minimal @@ -0,0 +1,11 @@ +FROM python:3.11-alpine + +RUN apk update +RUN apk add gcc musl-dev build-base linux-headers + +WORKDIR /code +COPY tuplex-0.3.7-cp311-cp311-musllinux_1_2_aarch64.whl /code/wheelhouse/ + +RUN python3 -m pip install /code/wheelhouse/tuplex-0.3.7-cp311-cp311-musllinux_1_2_aarch64.whl + + diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index c1589e3ed..75acc738d 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -60,11 +60,23 @@ message(STATUS "includes: ${Python3_INCLUDE_DIRS}") #include(${pybind11_SOURCE_DIR}/tools/pybind11Tools.cmake) #include(${pybind11_SOURCE_DIR}/tools/pybind11NewTools.cmake) +## Method 1: +## cf. https://pybind11.readthedocs.io/en/stable/cmake/index.html +#pybind11_add_module(${MODULE_NAME} ${SOURCES}) +#set(PYMODULE "PYBIND11_MODULE(${MODULE_NAME}, m)") +#target_compile_definitions(${MODULE_NAME} PRIVATE PYMODULE=${PYMODULE}) + +# Method 2: +# Python method: +find_package(Python COMPONENTS Interpreter Development) +Python3_add_library(${MODULE_NAME} ${SOURCES}) +target_link_libraries(${MODULE_NAME} PUBLIC pybind11::headers) +set_target_properties(${MODULE_NAME} PROPERTIES + INTERPROCEDURAL_OPTIMIZATION ON + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) + -# cf. https://pybind11.readthedocs.io/en/stable/cmake/index.html -pybind11_add_module(${MODULE_NAME} ${SOURCES}) -set(PYMODULE "PYBIND11_MODULE(${MODULE_NAME}, m)") -target_compile_definitions(${MODULE_NAME} PRIVATE PYMODULE=${PYMODULE}) add_dependencies(${MODULE_NAME} libcore libcodegen) target_include_directories(${MODULE_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include From 40dfb3c6ff9beecbee72777931e09344332498f0 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 14:33:56 -0700 Subject: [PATCH 80/92] fix --- tuplex/python/CMakeLists.txt | 74 ++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 75acc738d..3616ccfd3 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -71,12 +71,78 @@ message(STATUS "includes: ${Python3_INCLUDE_DIRS}") find_package(Python COMPONENTS Interpreter Development) Python3_add_library(${MODULE_NAME} ${SOURCES}) target_link_libraries(${MODULE_NAME} PUBLIC pybind11::headers) -set_target_properties(${MODULE_NAME} PROPERTIES - INTERPROCEDURAL_OPTIMIZATION ON - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON) +# Macro/function to detect musl libc or Alpine Linux +function(detect_musl OUT_VAR) + include(CheckCXXSourceCompiles) + set(_musl_detected OFF) + + # Method 1: Check for musl-specific functions that don't exist in glibc + check_cxx_source_compiles(" + #include + #include + int main() { + // musl has __progname, glibc doesn't + extern char *__progname; + return 0; + } + " _musl_progname_detected) + + if(_musl_progname_detected) + set(_musl_detected ON) + else() + # Method 2: Check for musl-specific header behavior + check_cxx_source_compiles(" + #include + #include + int main() { + // musl defines MSG_NOSIGNAL, glibc doesn't always + int flag = MSG_NOSIGNAL; + return 0; + } + " _musl_socket_detected) + + if(_musl_socket_detected) + set(_musl_detected ON) + else() + # Method 3: Check for Alpine Linux specific behavior + check_cxx_source_compiles(" + #include + int main() { + // Check if we're on Alpine/musl by looking for specific defines + #ifdef __GLIBC__ + #error glibc detected + #endif + return 0; + } + " _musl_features_detected) + + if(_musl_features_detected) + set(_musl_detected ON) + endif() + endif() + endif() + + if(APPLE) + set(_musl_detected OFF) + endif() + + set(${OUT_VAR} ${_musl_detected} PARENT_SCOPE) +endfunction() + +# Detect whether muslc is being used. +detect_musl(MUSL_DETECTED) + +if(MUSL_DETECTED) + # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. + message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") + set_target_properties(${MODULE_NAME} PROPERTIES + INTERPROCEDURAL_OPTIMIZATION OFF + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) +endif() + add_dependencies(${MODULE_NAME} libcore libcodegen) target_include_directories(${MODULE_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include From 3c3411a7c3b2e1da2f88c1ee5e0b938c644f41c9 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 14:44:36 -0700 Subject: [PATCH 81/92] fixes --- tuplex/python/CMakeLists.txt | 8 ++++++++ tuplex/python/preserve_symbols.txt | 1 + 2 files changed, 9 insertions(+) create mode 100644 tuplex/python/preserve_symbols.txt diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 3616ccfd3..1e4d798aa 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -135,12 +135,20 @@ endfunction() detect_musl(MUSL_DETECTED) if(MUSL_DETECTED) + # Option 1: # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") set_target_properties(${MODULE_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION OFF CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) + + # Option 2: + # The alternative is to have a dedicated file to tell which symbols are required to be preserved during the LTO phase. + # Add linker flags to preserve PyInit_ symbols during LTO + target_link_options(${MODULE_NAME} PRIVATE + "-Wl,--export-dynamic" + "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt") endif() add_dependencies(${MODULE_NAME} libcore libcodegen) diff --git a/tuplex/python/preserve_symbols.txt b/tuplex/python/preserve_symbols.txt new file mode 100644 index 000000000..f84e68323 --- /dev/null +++ b/tuplex/python/preserve_symbols.txt @@ -0,0 +1 @@ +PyInit_tuplex From 860e5cd9b0b700709a7488670ddd8a9547f726e7 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 16:59:29 -0700 Subject: [PATCH 82/92] fix --- tuplex/python/CMakeLists.txt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 1e4d798aa..4d7a906b0 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -135,20 +135,21 @@ endfunction() detect_musl(MUSL_DETECTED) if(MUSL_DETECTED) - # Option 1: - # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. - message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") - set_target_properties(${MODULE_NAME} PROPERTIES - INTERPROCEDURAL_OPTIMIZATION OFF - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON) + # # Option 1: + # # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. + # message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") + # set_target_properties(${MODULE_NAME} PROPERTIES + # INTERPROCEDURAL_OPTIMIZATION OFF + # CXX_VISIBILITY_PRESET hidden + # VISIBILITY_INLINES_HIDDEN ON) # Option 2: # The alternative is to have a dedicated file to tell which symbols are required to be preserved during the LTO phase. # Add linker flags to preserve PyInit_ symbols during LTO target_link_options(${MODULE_NAME} PRIVATE "-Wl,--export-dynamic" - "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt") + "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt" + "-Wl,--allow-shlib-undefined") endif() add_dependencies(${MODULE_NAME} libcore libcodegen) From 85a4f9083312679d5f340ae85734d25bda86db6d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 17:06:35 -0700 Subject: [PATCH 83/92] settings sufficient --- tuplex/python/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 4d7a906b0..9bb38e268 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -144,11 +144,15 @@ if(MUSL_DETECTED) # VISIBILITY_INLINES_HIDDEN ON) # Option 2: - # The alternative is to have a dedicated file to tell which symbols are required to be preserved during the LTO phase. - # Add linker flags to preserve PyInit_ symbols during LTO + # Pass the right linker flags for Alpine. + # If there are still missing symbols, can also add an option + # "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt" + # And have a file with a list of symbols to preserve. + # E.g., + # PyInit_ + # ... target_link_options(${MODULE_NAME} PRIVATE "-Wl,--export-dynamic" - "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt" "-Wl,--allow-shlib-undefined") endif() From 650753e30083de80e850c95e770e2f9a7b3c7c0e Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 17:15:41 -0700 Subject: [PATCH 84/92] scripts --- scripts/alpine/README.md | 5 +++++ scripts/alpine/example/Dockerfile | 2 ++ scripts/alpine/make-wheels.sh | 37 +++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 scripts/alpine/README.md create mode 100644 scripts/alpine/example/Dockerfile create mode 100755 scripts/alpine/make-wheels.sh diff --git a/scripts/alpine/README.md b/scripts/alpine/README.md new file mode 100644 index 000000000..7be927775 --- /dev/null +++ b/scripts/alpine/README.md @@ -0,0 +1,5 @@ +Building a minimal alpine image with Tuplex. + +Images like python:3.11-alpine use muslc and are extremtly small. However, building for them is not trivial. + +This folder here holds a script to create whl files to be used in a python:3.11-alpine image. \ No newline at end of file diff --git a/scripts/alpine/example/Dockerfile b/scripts/alpine/example/Dockerfile new file mode 100644 index 000000000..f8f0533a2 --- /dev/null +++ b/scripts/alpine/example/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-alpine + diff --git a/scripts/alpine/make-wheels.sh b/scripts/alpine/make-wheels.sh new file mode 100755 index 000000000..d86ca7af0 --- /dev/null +++ b/scripts/alpine/make-wheels.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Script to build Tuplex wheels for musl-based Alpine Linux images +# This script checks for the required tuplex/musl Docker image and provides +# instructions if it doesn't exist. + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "Checking for tuplex/musl Docker image..." + +# Check if the tuplex/musl image exists locally +if ! docker image inspect tuplex/musl >/dev/null 2>&1; then + echo -e "${RED}Error: tuplex/musl Docker image not found locally.${NC}" + echo "" + echo -e "${YELLOW}To build the required tuplex/musl image, please run:${NC}" + echo "" + echo " docker build -t tuplex/musl -f scripts/docker/musllinux/Dockerfile ." + echo "" + echo "This will create the musl-based image needed for building Alpine-compatible wheels." + echo "" + echo -e "${YELLOW}Note:${NC} The build process may take some time as it needs to compile" + echo "Tuplex and all its dependencies for the musl libc environment." + echo "" + exit 1 +fi + +echo -e "${GREEN}✓ tuplex/musl Docker image found locally.${NC}" +echo "Proceeding with wheel building..." + +# TODO: Add the actual wheel building logic here +echo "Wheel building functionality to be implemented..." From 4589e42b734ea6de585dea0ecf071b41fa012154 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 17:27:54 -0700 Subject: [PATCH 85/92] builder image setup --- scripts/alpine/builder/Dockerfile | 12 ++++++++++ scripts/alpine/builder/build.sh | 7 ++++++ scripts/alpine/make-wheels.sh | 34 +++++++++++++++++++++++++++++ scripts/docker/benchmark/Dockerfile | 1 - scripts/ubuntu2004/Dockerfile | 2 -- scripts/ubuntu2204/Dockerfile | 2 -- 6 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 scripts/alpine/builder/Dockerfile create mode 100644 scripts/alpine/builder/build.sh diff --git a/scripts/alpine/builder/Dockerfile b/scripts/alpine/builder/Dockerfile new file mode 100644 index 000000000..52971e5fb --- /dev/null +++ b/scripts/alpine/builder/Dockerfile @@ -0,0 +1,12 @@ +FROM tuplex/musl:latest + +# mount tuplex in /code. + +# mount wheelhouse in /wheelhouse. + +WORKDIR /work + +COPY build.sh /work/build.sh +RUN chmod +x /work/build.sh + +ENTRYPOINT ["/work/build.sh"] diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh new file mode 100644 index 000000000..98f09cab9 --- /dev/null +++ b/scripts/alpine/builder/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Alpine wheel builder script +# This script will be executed when the container starts + +echo "Alpine wheel builder container started" + diff --git a/scripts/alpine/make-wheels.sh b/scripts/alpine/make-wheels.sh index d86ca7af0..6e4a36022 100755 --- a/scripts/alpine/make-wheels.sh +++ b/scripts/alpine/make-wheels.sh @@ -6,6 +6,9 @@ set -e +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' @@ -33,5 +36,36 @@ fi echo -e "${GREEN}✓ tuplex/musl Docker image found locally.${NC}" echo "Proceeding with wheel building..." + + +# Build the tuplex/alpine-wheel-builder image based on scripts/alpine/builder/Dockerfile +echo "Building tuplex/alpine-wheel-builder Docker image..." +docker build -t tuplex/alpine-wheel-builder -f "$SCRIPT_DIR/builder/Dockerfile" "$SCRIPT_DIR/builder" + +echo -e "${GREEN}✓ tuplex/alpine-wheel-builder Docker image built successfully.${NC}" + +# Prepare Docker mount options +# Exclude build/ and other temp files using .dockerignore if present, otherwise use :ro as fallback +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")/.." +MOUNT_OPTS="-v $PROJECT_ROOT/tuplex:/code/tuplex:ro" + +echo "Launching tuplex/alpine-wheel-builder container with tuplex/ mounted..." + +# Ensure wheelhouse/ exists in the project root directory +if [ ! -d "$PROJECT_ROOT/wheelhouse" ]; then + echo "Creating wheelhouse/ directory for output wheels..." + mkdir -p "$PROJECT_ROOT/wheelhouse" +fi + +# Add wheelhouse mount to Docker options +MOUNT_OPTS="$MOUNT_OPTS -v $PROJECT_ROOT/wheelhouse:/wheelhouse" + + +# You can add --rm to auto-remove the container after exit +docker run -it --rm \ + $MOUNT_OPTS \ + --name tuplex-alpine-wheel-builder \ + tuplex/alpine-wheel-builder + # TODO: Add the actual wheel building logic here echo "Wheel building functionality to be implemented..." diff --git a/scripts/docker/benchmark/Dockerfile b/scripts/docker/benchmark/Dockerfile index 4188a4aee..a1b86d2f6 100644 --- a/scripts/docker/benchmark/Dockerfile +++ b/scripts/docker/benchmark/Dockerfile @@ -2,7 +2,6 @@ # (c) 2017 - 2023 L.Spiegelberg, R. Yesantharao FROM ubuntu:20.04 -MAINTAINER Leonhard Spiegelberg "leonhard_spiegelberg@brown.edu" # Use /tmp folder to build everything... ENV DEBIAN_FRONTEND=noninteractive diff --git a/scripts/ubuntu2004/Dockerfile b/scripts/ubuntu2004/Dockerfile index 20f4a72b1..597e497fa 100644 --- a/scripts/ubuntu2004/Dockerfile +++ b/scripts/ubuntu2004/Dockerfile @@ -3,8 +3,6 @@ FROM ubuntu:20.04 # build using docker build -t tuplex/ubuntu:20.04 . -MAINTAINER Leonhard Spiegelberg "leonhard@brown.edu" - ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /opt/sbin diff --git a/scripts/ubuntu2204/Dockerfile b/scripts/ubuntu2204/Dockerfile index d977f8b0c..9d9974690 100644 --- a/scripts/ubuntu2204/Dockerfile +++ b/scripts/ubuntu2204/Dockerfile @@ -3,8 +3,6 @@ FROM ubuntu:22.04 # build using docker build -t tuplex/ubuntu:22.04 . -MAINTAINER Leonhard Spiegelberg "leonhard@brown.edu" - ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /opt/sbin From 687c5598c72ff77990a783afe4ffd62f163d9b4e Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 18:01:30 -0700 Subject: [PATCH 86/92] updates --- scripts/alpine/builder/build.sh | 88 +++++++++++++++++++++++++++++++++ scripts/alpine/make-wheels.sh | 11 +++-- 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh index 98f09cab9..255de70d4 100644 --- a/scripts/alpine/builder/build.sh +++ b/scripts/alpine/builder/build.sh @@ -3,5 +3,93 @@ # Alpine wheel builder script # This script will be executed when the container starts +set -e + echo "Alpine wheel builder container started" +# Use Python 3.11 from the tuplex/musl image +PYTHON="/opt/_internal/cpython-3.11.13/bin/python3.11" +PIP="/opt/_internal/cpython-3.11.13/bin/pip" + +echo "Using Python: $PYTHON" +echo "Python version: $($PYTHON --version)" + +# Check if the project root is mounted +if [ ! -f "/code/setup.py" ]; then + echo "Error: /code/setup.py not found. Make sure the project root is mounted." + exit 1 +fi + +# Check if wheelhouse directory is mounted +if [ ! -d "/wheelhouse" ]; then + echo "Error: /wheelhouse directory not found. Make sure the wheelhouse is mounted." + exit 1 +fi + +echo "Installing required dependencies..." +$PIP install setuptools wheel cloudpickle numpy attrs dill pluggy py pygments six wcwidth astor prompt_toolkit jedi PyYAML psutil pymongo iso8601 + +export LLVM_ROOT_DIR="/opt/llvm-16.0.6" +export LLVM_CONFIG="/opt/llvm-16.0.6/bin/llvm-config" +export PATH="/opt/llvm-16.0.6/bin:$PATH" + + +echo "Building wheel for tuplex..." + +# Change to the tuplex source directory +cd /code + +export CMAKE_ARGS=" + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_WITH_AWS=OFF \ + -DBUILD_WITH_ORC=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSKIP_AWS_TESTS=ON \ + -DBUILD_FOR_CI=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ + -DCMAKE_CXX_FLAGS=\"-O3 -DNDEBUG\" \ + -DCMAKE_C_FLAGS=\"-O3 -DNDEBUG\" \ + -DLLVM_ROOT=/opt/llvm-16.0.6 \ + -DLLVM_ROOT_DIR=/opt/llvm-16.0.6 \ + -DLLVM_CONFIG=/opt/llvm-16.0.6/bin/llvm-config \ + -DPython3_EXECUTABLE=/opt/_internal/cpython-3.11.13/bin/python3.11 \ + -DPython3_ROOT_DIR=/opt/_internal/cpython-3.11.13 \ + -DPython3_LIBRARY=/opt/_internal/cpython-3.11.13/lib/libpython3.11.so \ + -DPython3_INCLUDE_DIR=/opt/python/cp311-cp311/include/python3.11 \ + -DPYTHON3_VERSION=3.11 \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 +" + + + +# Build the wheel using setup.py +echo "Building wheel with setup.py..." +CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel + +# Find the built wheel +WHEEL_FILE=$(find dist/ -name "*.whl" | head -1) + +if [ -z "$WHEEL_FILE" ]; then + echo "Error: No wheel file found in dist/ directory" + exit 1 +fi + +echo "Built wheel: $WHEEL_FILE" + +# Use auditwheel to repair/delocate the wheel +echo "Repairing wheel with auditwheel..." +$PIP install auditwheel +auditwheel repair "$WHEEL_FILE" -w /wheelhouse + +# Find the repaired wheel +REPAIRED_WHEEL=$(find /wheelhouse -name "*.whl" | head -1) + +if [ -z "$REPAIRED_WHEEL" ]; then + echo "Error: No repaired wheel found in /wheelhouse" + exit 1 +fi + +echo "Successfully built and repaired wheel: $REPAIRED_WHEEL" +echo "Wheel stored in: /wheelhouse" + diff --git a/scripts/alpine/make-wheels.sh b/scripts/alpine/make-wheels.sh index 6e4a36022..19b316c42 100755 --- a/scripts/alpine/make-wheels.sh +++ b/scripts/alpine/make-wheels.sh @@ -45,20 +45,21 @@ docker build -t tuplex/alpine-wheel-builder -f "$SCRIPT_DIR/builder/Dockerfile" echo -e "${GREEN}✓ tuplex/alpine-wheel-builder Docker image built successfully.${NC}" # Prepare Docker mount options -# Exclude build/ and other temp files using .dockerignore if present, otherwise use :ro as fallback +# Mount the entire project root to /code so setup.py and other files are accessible +# Note: Must be writable as setup.py needs to create directories during build PROJECT_ROOT="$(dirname "$SCRIPT_DIR")/.." -MOUNT_OPTS="-v $PROJECT_ROOT/tuplex:/code/tuplex:ro" +MOUNT_OPTS="-v $PROJECT_ROOT:/code" echo "Launching tuplex/alpine-wheel-builder container with tuplex/ mounted..." # Ensure wheelhouse/ exists in the project root directory -if [ ! -d "$PROJECT_ROOT/wheelhouse" ]; then +if [ ! -d "$SCRIPT_DIR/wheelhouse" ]; then echo "Creating wheelhouse/ directory for output wheels..." - mkdir -p "$PROJECT_ROOT/wheelhouse" + mkdir -p "$SCRIPT_DIR/wheelhouse" fi # Add wheelhouse mount to Docker options -MOUNT_OPTS="$MOUNT_OPTS -v $PROJECT_ROOT/wheelhouse:/wheelhouse" +MOUNT_OPTS="$MOUNT_OPTS -v $SCRIPT_DIR/wheelhouse:/wheelhouse" # You can add --rm to auto-remove the container after exit From c3653e675fb248eb3c09f57dca27f4b8f266adcc Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 18:08:09 -0700 Subject: [PATCH 87/92] list --- scripts/alpine/builder/build.sh | 163 ++++++++++++++++-------- scripts/alpine/make-wheels.sh | 28 ++++- tuplex/python/CMakeLists.txt | 10 +- tuplex/python/preserve_symbols.txt | 192 +++++++++++++++++++++++++++++ 4 files changed, 328 insertions(+), 65 deletions(-) diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh index 255de70d4..bbd425c61 100644 --- a/scripts/alpine/builder/build.sh +++ b/scripts/alpine/builder/build.sh @@ -2,9 +2,17 @@ # Alpine wheel builder script # This script will be executed when the container starts +# Usage: build.sh [--test-only] set -e +# Check for --test-only parameter +TEST_ONLY=false +if [ "$1" = "--test-only" ]; then + TEST_ONLY=true + echo "Test-only mode: will skip wheel building and test existing wheel" +fi + echo "Alpine wheel builder container started" # Use Python 3.11 from the tuplex/musl image @@ -26,70 +34,121 @@ if [ ! -d "/wheelhouse" ]; then exit 1 fi -echo "Installing required dependencies..." -$PIP install setuptools wheel cloudpickle numpy attrs dill pluggy py pygments six wcwidth astor prompt_toolkit jedi PyYAML psutil pymongo iso8601 - -export LLVM_ROOT_DIR="/opt/llvm-16.0.6" -export LLVM_CONFIG="/opt/llvm-16.0.6/bin/llvm-config" -export PATH="/opt/llvm-16.0.6/bin:$PATH" +if [ "$TEST_ONLY" = false ]; then + echo "Installing required dependencies..." + $PIP install setuptools wheel cloudpickle numpy attrs dill pluggy py pygments six wcwidth astor prompt_toolkit jedi PyYAML psutil pymongo iso8601 + + export LLVM_ROOT_DIR="/opt/llvm-16.0.6" + export LLVM_CONFIG="/opt/llvm-16.0.6/bin/llvm-config" + export PATH="/opt/llvm-16.0.6/bin:$PATH" + + echo "Building wheel for tuplex..." + + # Change to the tuplex source directory + cd /code + + export CMAKE_ARGS=" + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_WITH_AWS=OFF \ + -DBUILD_WITH_ORC=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSKIP_AWS_TESTS=ON \ + -DBUILD_FOR_CI=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ + -DCMAKE_CXX_FLAGS=\"-O3 -DNDEBUG\" \ + -DCMAKE_C_FLAGS=\"-O3 -DNDEBUG\" \ + -DLLVM_ROOT=/opt/llvm-16.0.6 \ + -DLLVM_ROOT_DIR=/opt/llvm-16.0.6 \ + -DLLVM_CONFIG=/opt/llvm-16.0.6/bin/llvm-config \ + -DPython3_EXECUTABLE=/opt/_internal/cpython-3.11.13/bin/python3.11 \ + -DPython3_ROOT_DIR=/opt/_internal/cpython-3.11.13 \ + -DPython3_LIBRARY=/opt/_internal/cpython-3.11.13/lib/libpython3.11.so \ + -DPython3_INCLUDE_DIR=/opt/python/cp311-cp311/include/python3.11 \ + -DPYTHON3_VERSION=3.11 \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + " + + # Build the wheel using setup.py + echo "Building wheel with setup.py..." + CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel + + # Find the built wheel + WHEEL_FILE=$(find dist/ -name "*.whl" | head -1) + + if [ -z "$WHEEL_FILE" ]; then + echo "Error: No wheel file found in dist/ directory" + exit 1 + fi + + echo "Built wheel: $WHEEL_FILE" + + # Use auditwheel to repair/delocate the wheel + echo "Repairing wheel with auditwheel..." + $PIP install auditwheel + auditwheel repair "$WHEEL_FILE" -w /wheelhouse + + # Find the repaired wheel + REPAIRED_WHEEL=$(find /wheelhouse -name "*.whl" | head -1) + + if [ -z "$REPAIRED_WHEEL" ]; then + echo "Error: No repaired wheel found in /wheelhouse" + exit 1 + fi + + echo "Successfully built and repaired wheel: $REPAIRED_WHEEL" + echo "Wheel stored in: /wheelhouse" +else + echo "Skipping wheel building (test-only mode)" + # Find the existing repaired wheel + REPAIRED_WHEEL=$(find /wheelhouse -name "*.whl" | head -1) + + if [ -z "$REPAIRED_WHEEL" ]; then + echo "Error: No wheel file found in /wheelhouse for testing" + exit 1 + fi + + echo "Found existing wheel for testing: $REPAIRED_WHEEL" +fi -echo "Building wheel for tuplex..." -# Change to the tuplex source directory -cd /code -export CMAKE_ARGS=" - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DBUILD_WITH_AWS=OFF \ - -DBUILD_WITH_ORC=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DSKIP_AWS_TESTS=ON \ - -DBUILD_FOR_CI=ON \ - -DBUILD_TESTING=OFF \ - -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ - -DCMAKE_CXX_FLAGS=\"-O3 -DNDEBUG\" \ - -DCMAKE_C_FLAGS=\"-O3 -DNDEBUG\" \ - -DLLVM_ROOT=/opt/llvm-16.0.6 \ - -DLLVM_ROOT_DIR=/opt/llvm-16.0.6 \ - -DLLVM_CONFIG=/opt/llvm-16.0.6/bin/llvm-config \ - -DPython3_EXECUTABLE=/opt/_internal/cpython-3.11.13/bin/python3.11 \ - -DPython3_ROOT_DIR=/opt/_internal/cpython-3.11.13 \ - -DPython3_LIBRARY=/opt/_internal/cpython-3.11.13/lib/libpython3.11.so \ - -DPython3_INCLUDE_DIR=/opt/python/cp311-cp311/include/python3.11 \ - -DPYTHON3_VERSION=3.11 \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -" +echo "==============================" +echo "Testing built wheel in a fresh virtual environment..." +echo "==============================" +# Create a temporary directory for the venv +VENV_DIR=$(mktemp -d) +echo "Created temporary venv directory: $VENV_DIR" +# Create a new virtual environment +$PYTHON -m venv "$VENV_DIR/venv" +source "$VENV_DIR/venv/bin/activate" -# Build the wheel using setup.py -echo "Building wheel with setup.py..." -CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel +# Upgrade pip and install wheel +pip install --upgrade pip wheel -# Find the built wheel -WHEEL_FILE=$(find dist/ -name "*.whl" | head -1) +# Install the repaired wheel +pip install "$REPAIRED_WHEEL" -if [ -z "$WHEEL_FILE" ]; then - echo "Error: No wheel file found in dist/ directory" - exit 1 -fi +# Install test dependencies (pytest, etc.) +pip install pytest -echo "Built wheel: $WHEEL_FILE" +# Run the tests in tuplex/python/tests +echo "Running tests in tuplex/python/tests..." +pytest /code/tuplex/python/tests -# Use auditwheel to repair/delocate the wheel -echo "Repairing wheel with auditwheel..." -$PIP install auditwheel -auditwheel repair "$WHEEL_FILE" -w /wheelhouse +TEST_EXIT_CODE=$? -# Find the repaired wheel -REPAIRED_WHEEL=$(find /wheelhouse -name "*.whl" | head -1) +# Deactivate and remove the virtual environment +deactivate +rm -rf "$VENV_DIR" -if [ -z "$REPAIRED_WHEEL" ]; then - echo "Error: No repaired wheel found in /wheelhouse" - exit 1 +if [ $TEST_EXIT_CODE -eq 0 ]; then + echo "All tests passed successfully!" +else + echo "Some tests failed. Exit code: $TEST_EXIT_CODE" + exit $TEST_EXIT_CODE fi -echo "Successfully built and repaired wheel: $REPAIRED_WHEEL" -echo "Wheel stored in: /wheelhouse" - diff --git a/scripts/alpine/make-wheels.sh b/scripts/alpine/make-wheels.sh index 19b316c42..79ae977ec 100755 --- a/scripts/alpine/make-wheels.sh +++ b/scripts/alpine/make-wheels.sh @@ -3,9 +3,17 @@ # Script to build Tuplex wheels for musl-based Alpine Linux images # This script checks for the required tuplex/musl Docker image and provides # instructions if it doesn't exist. +# Usage: make-wheels.sh [--test-only] set -e +# Check for --test-only parameter +TEST_ONLY=false +if [ "$1" = "--test-only" ]; then + TEST_ONLY=true + echo "Test-only mode: will test existing wheel without rebuilding" +fi + # Get the directory where this script is located SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -63,10 +71,18 @@ MOUNT_OPTS="$MOUNT_OPTS -v $SCRIPT_DIR/wheelhouse:/wheelhouse" # You can add --rm to auto-remove the container after exit -docker run -it --rm \ - $MOUNT_OPTS \ - --name tuplex-alpine-wheel-builder \ - tuplex/alpine-wheel-builder +if [ "$TEST_ONLY" = true ]; then + echo "Running container in test-only mode..." + docker run -it --rm \ + $MOUNT_OPTS \ + --name tuplex-alpine-wheel-builder \ + tuplex/alpine-wheel-builder --test-only +else + echo "Running container in build mode..." + docker run -it --rm \ + $MOUNT_OPTS \ + --name tuplex-alpine-wheel-builder \ + tuplex/alpine-wheel-builder +fi -# TODO: Add the actual wheel building logic here -echo "Wheel building functionality to be implemented..." +echo "Container execution completed." diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index 9bb38e268..ef64f59e3 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -144,15 +144,11 @@ if(MUSL_DETECTED) # VISIBILITY_INLINES_HIDDEN ON) # Option 2: - # Pass the right linker flags for Alpine. - # If there are still missing symbols, can also add an option - # "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt" - # And have a file with a list of symbols to preserve. - # E.g., - # PyInit_ - # ... + # Pass the right linker flags for Alpine. In addition, need to have a list of symbols to preserve as + # they need to be present when using the C-extension module. target_link_options(${MODULE_NAME} PRIVATE "-Wl,--export-dynamic" + "-Wl,--retain-symbols-file=${CMAKE_CURRENT_SOURCE_DIR}/preserve_symbols.txt" "-Wl,--allow-shlib-undefined") endif() diff --git a/tuplex/python/preserve_symbols.txt b/tuplex/python/preserve_symbols.txt index f84e68323..2dcc2267b 100644 --- a/tuplex/python/preserve_symbols.txt +++ b/tuplex/python/preserve_symbols.txt @@ -1 +1,193 @@ PyInit_tuplex +PyInit_tuplex +_Py_Dealloc +PyLong_FromLong +PyModuleDef_Init +PyUnicode_AsUTF8String +PyDict_SetItem +PyEval_GetBuiltins +PyUnicode_FromFormat +PySlice_New +PyDict_SetItemString +PyCapsule_GetName +PyImport_ImportModule +PyType_IsSubtype +PyException_SetTraceback +PyModule_GetDict +PyObject_SetAttrString +PyErr_Fetch +_PyThreadState_UncheckedGet +PyThread_acquire_lock +PyDict_Copy +PyUnicode_AsWideCharString +PyUnicode_FromString +PyObject_GetAttrString +PyUnicode_AsUTF8 +PyLong_AsLong +PyObject_CallFunctionObjArgs +PyObject_GetItem +PyObject_SetItem +PyObject_Size +PyObject_Str +PyObject_Repr +PyObject_Hash +PyObject_IsTrue +PyObject_Not +PyObject_RichCompare +PyObject_RichCompareBool +PyObject_GetIter +PyIter_Next +PyObject_GenericGetAttr +PyObject_GenericSetAttr +PyObject_GetAttr +PyObject_SetAttr +PyObject_DelAttr +PyObject_DelAttrString +PyObject_GetAttrString +PyObject_HasAttr +PyObject_HasAttrString +PyObject_GetItem +PyObject_SetItem +PyObject_DelItem +PyObject_Contains +PyObject_Length +PyObject_Size +PyObject_GetSlice +PyObject_SetSlice +PyObject_DelSlice +PySequence_Check +PySequence_Size +PySequence_Length +PySequence_Concat +PySequence_Repeat +PySequence_GetItem +PySequence_SetItem +PySequence_DelItem +PySequence_GetSlice +PySequence_SetSlice +PySequence_DelSlice +PySequence_Count +PySequence_Index +PySequence_List +PySequence_Tuple +PySequence_Fast +PySequence_Fast_GET_ITEM +PySequence_Fast_GET_SIZE +PySequence_InPlaceConcat +PySequence_InPlaceRepeat +PyMapping_Check +PyMapping_Size +PyMapping_Length +PyMapping_HasKey +PyMapping_HasKeyString +PyMapping_Keys +PyMapping_Values +PyMapping_Items +PyMapping_GetItemString +PyMapping_SetItemString +PyMapping_DelItemString +PyMapping_Clear +PyNumber_Check +PyNumber_Add +PyNumber_Subtract +PyNumber_Multiply +PyNumber_MatrixMultiply +PyNumber_FloorDivide +PyNumber_TrueDivide +PyNumber_Remainder +PyNumber_Divmod +PyNumber_Power +PyNumber_Negative +PyNumber_Positive +PyNumber_Absolute +PyNumber_Invert +PyNumber_Lshift +PyNumber_Rshift +PyNumber_And +PyNumber_Xor +PyNumber_Or +PyNumber_Index +PyNumber_ToBase +PyNumber_InPlaceAdd +PyNumber_InPlaceSubtract +PyNumber_InPlaceMultiply +PyNumber_InPlaceMatrixMultiply +PyNumber_InPlaceFloorDivide +PyNumber_InPlaceTrueDivide +PyNumber_InPlaceRemainder +PyNumber_InPlacePower +PyNumber_InPlaceLshift +PyNumber_InPlaceRshift +PyNumber_InPlaceAnd +PyNumber_InPlaceXor +PyNumber_InPlaceOr +PyNumber_Long +PyNumber_Float +PyNumber_Int +PyNumber_Complex +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode +PyNumber_AsSsize_t +PyNumber_AsSize_t +PyNumber_AsLong +PyNumber_AsUnsignedLong +PyNumber_AsUnsignedLongLong +PyNumber_AsLongLong +PyNumber_AsDouble +PyNumber_AsComplex +PyNumber_AsCString +PyNumber_AsString +PyNumber_AsUnicode From 9cb248116cac2a5f9b75ea82811657ca86de76d6 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 18:18:07 -0700 Subject: [PATCH 88/92] update --- tuplex/python/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index ef64f59e3..dc5a3498c 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -135,6 +135,8 @@ endfunction() detect_musl(MUSL_DETECTED) if(MUSL_DETECTED) + message(STATUS "muslc detected, adjusting linker flags") + # # Option 1: # # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. # message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") From 256e67e8d62f4c5e57d6e2e4daf37bba1d627e14 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 18:32:51 -0700 Subject: [PATCH 89/92] refactor --- scripts/alpine/builder/build.sh | 44 ++++++++- setup.py | 170 +++++++++++++++++++++++--------- 2 files changed, 164 insertions(+), 50 deletions(-) diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh index bbd425c61..04d184dd4 100644 --- a/scripts/alpine/builder/build.sh +++ b/scripts/alpine/builder/build.sh @@ -36,7 +36,7 @@ fi if [ "$TEST_ONLY" = false ]; then echo "Installing required dependencies..." - $PIP install setuptools wheel cloudpickle numpy attrs dill pluggy py pygments six wcwidth astor prompt_toolkit jedi PyYAML psutil pymongo iso8601 + $PIP install setuptools wheel cloudpickle numpy attrs dill pluggy py pygments six wcwidth astor prompt_toolkit jedi PyYAML psutil iso8601 export LLVM_ROOT_DIR="/opt/llvm-16.0.6" export LLVM_CONFIG="/opt/llvm-16.0.6/bin/llvm-config" @@ -71,7 +71,9 @@ if [ "$TEST_ONLY" = false ]; then # Build the wheel using setup.py echo "Building wheel with setup.py..." - CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel + # Exclude history server from the wheel to reduce size and dependencies + echo "Setting TUPLEX_INCLUDE_HISTORYSERVER=False to exclude history server" + TUPLEX_INCLUDE_HISTORYSERVER=False CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel # Find the built wheel WHEEL_FILE=$(find dist/ -name "*.whl" | head -1) @@ -83,6 +85,44 @@ if [ "$TEST_ONLY" = false ]; then echo "Built wheel: $WHEEL_FILE" + check_pyinit_tuplex_symbol() { + # Check that the symbol PyInit_tuplex is present in the tuplex .so + echo "Checking for PyInit_tuplex symbol in the built .so..." + + # Find the .so file in the wheel + local so_file + so_file=$(unzip -l "$WHEEL_FILE" | grep '\.so$' | awk '{print $4}' | head -1) + if [ -z "$so_file" ]; then + echo "Error: No .so file found in the wheel." + exit 1 + fi + + # Extract the .so file to a temp location + local tmp_so_dir + tmp_so_dir=$(mktemp -d) + unzip -j "$WHEEL_FILE" "$so_file" -d "$tmp_so_dir" >/dev/null + + local so_path="$tmp_so_dir/$(basename "$so_file")" + + if [ ! -f "$so_path" ]; then + echo "Error: Failed to extract .so file from wheel." + exit 1 + fi + + # Check for PyInit_tuplex symbol + if ! nm -D "$so_path" 2>/dev/null | grep -q 'PyInit_tuplex'; then + echo "Error: PyInit_tuplex symbol not found in $so_path" + exit 1 + else + echo "✓ PyInit_tuplex symbol found in $so_path" + fi + + # Clean up temp .so extraction directory + rm -rf "$tmp_so_dir" + } + + check_pyinit_tuplex_symbol + # Use auditwheel to repair/delocate the wheel echo "Repairing wheel with auditwheel..." $PIP install auditwheel diff --git a/setup.py b/setup.py index 842da6a6e..8a62b52f0 100644 --- a/setup.py +++ b/setup.py @@ -117,52 +117,107 @@ def run_command(cmd, cwd, env): if in_google_colab(): logging.debug('Building dependencies for Google Colab environment') - install_dependencies = [ - 'urllib3!=1.25.0,!=1.25.1,>=1.21.1', - 'folium>=0.2.1' - 'requests', - 'attrs>=19.2.0', - 'dill>=0.2.7.1', - 'pluggy', - 'py>=1.5.2', - 'pygments>=2.4.1', - 'six>=1.11.0', - 'wcwidth>=0.1.7', - 'astor', - 'prompt_toolkit', - 'jedi', - "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", - "cloudpickle>=2.1.0;python_version=='3.10'", - "cloudpickle>=2.2.1;python_version>='3.11'", - 'numpy>=1.15.0', - 'PyYAML>=3.13', - 'psutil', - 'pymongo', - 'iso8601' - ] + # Check if history server should be included + include_historyserver = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') + + if include_historyserver: + install_dependencies = [ + 'urllib3!=1.25.0,!=1.25.1,>=1.21.1', + 'folium>=0.2.1' + 'requests', + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'pymongo', + 'iso8601' + ] + else: + # Exclude webui dependencies when history server is not included + # Note: iso8601 is still needed by core tuplex code, not just history server + install_dependencies = [ + 'urllib3!=1.25.0,!=1.25.1,>=1.21.1', + 'folium>=0.2.1' + 'requests', + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'iso8601' + ] else: logging.debug('Building dependencies for non Colab environment') - install_dependencies = webui_dependencies + [ - 'attrs>=19.2.0', - 'dill>=0.2.7.1', - 'pluggy', - 'py>=1.5.2', - 'pygments>=2.4.1', - 'six>=1.11.0', - 'wcwidth>=0.1.7', - 'astor', - 'prompt_toolkit', - 'jedi', - "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", - "cloudpickle>=2.1.0;python_version=='3.10'", - "cloudpickle>=2.2.1;python_version>='3.11'", - 'numpy>=1.15.0', - 'PyYAML>=3.13', - 'psutil', - 'pymongo', - 'iso8601' - ] + aws_lambda_dependencies + # Check if history server should be included + include_historyserver = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') + + if include_historyserver: + install_dependencies = webui_dependencies + [ + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'pymongo', + 'iso8601' + ] + aws_lambda_dependencies + else: + # Exclude webui dependencies when history server is not included + # Note: iso8601 is still needed by core tuplex code, not just history server + install_dependencies = [ + 'attrs>=19.2.0', + 'dill>=0.2.7.1', + 'pluggy', + 'py>=1.5.2', + 'pygments>=2.4.1', + 'six>=1.11.0', + 'wcwidth>=0.1.7', + 'astor', + 'prompt_toolkit', + 'jedi', + "cloudpickle>=0.6.1,<2.0.0;python_version<'3.10'", + "cloudpickle>=2.1.0;python_version=='3.10'", + "cloudpickle>=2.2.1;python_version>='3.11'", + 'numpy>=1.15.0', + 'PyYAML>=3.13', + 'psutil', + 'iso8601' + ] + aws_lambda_dependencies def ninja_installed(): # check whether ninja is on the path @@ -660,7 +715,18 @@ def read_readme(): def reorg_historyserver(): """ reorganize historyserver to become part of pip package. + Can be disabled by setting TUPLEX_INCLUDE_HISTORYSERVER=False """ + # Check if history server should be included + env_value = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True') + include_historyserver = env_value.lower() in ('true', '1', 'yes', 'on') + + logging.info(f'TUPLEX_INCLUDE_HISTORYSERVER={env_value}, include_historyserver={include_historyserver}') + + if not include_historyserver: + logging.info('History server excluded from package (TUPLEX_INCLUDE_HISTORYSERVER=False)') + return [] + # get absolute path of this file's location import pathlib current_path = pathlib.Path(__file__).parent.resolve() @@ -682,12 +748,16 @@ def tplx_package_data(): package_data = { # include libs in libexec - 'tuplex.libexec' : ['*.so', '*.dylib'], - 'tuplex.historyserver': ['thserver/templates/*.html', 'thserver/static/css/*.css', 'thserver/static/css/styles/*.css', - 'thserver/static/img/*.*', 'thserver/static/js/*.js', 'thserver/static/js/modules/*.js', - 'thserver/static/js/styles/*.css'] + 'tuplex.libexec' : ['*.so', '*.dylib'] } + # Check if history server should be included + include_historyserver = os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') + if include_historyserver: + package_data['tuplex.historyserver'] = ['thserver/templates/*.html', 'thserver/static/css/*.css', 'thserver/static/css/styles/*.css', + 'thserver/static/img/*.*', 'thserver/static/js/*.js', 'thserver/static/js/modules/*.js', + 'thserver/static/js/styles/*.css'] + # package lambda as well? lambda_zip = os.environ.get('TUPLEX_LAMBDA_ZIP', None) if lambda_zip and os.path.isfile(lambda_zip): @@ -696,6 +766,10 @@ def tplx_package_data(): # The information here can also be placed in setup.cfg - better separation of # logic and declaration, and simpler if you include description/version in a file. +# +# Environment variables: +# - TUPLEX_INCLUDE_HISTORYSERVER: Set to 'False' to exclude the history server +# from the package (default: 'True'). This reduces package size and dependencies. setup(name="tuplex", python_requires='>=3.9.0', version="0.3.7", @@ -743,7 +817,7 @@ def tplx_package_data(): 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', ], - scripts=['tuplex/historyserver/bin/tuplex-webui'], + scripts=['tuplex/historyserver/bin/tuplex-webui'] if os.environ.get('TUPLEX_INCLUDE_HISTORYSERVER', 'True').lower() in ('true', '1', 'yes', 'on') else [], project_urls={ "Bug Tracker": "https://github.com/tuplex", "Documentation": "https://tuplex.cs.brown.edu/python-api.html", From d0f7710bcdf3f3a02b8bb8b5666b2ac6c8c8709d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 18:57:19 -0700 Subject: [PATCH 90/92] fix --- scripts/alpine/builder/build.sh | 6 +++++- tuplex/python/CMakeLists.txt | 19 ++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh index 04d184dd4..5eaded3f3 100644 --- a/scripts/alpine/builder/build.sh +++ b/scripts/alpine/builder/build.sh @@ -58,8 +58,10 @@ if [ "$TEST_ONLY" = false ]; then -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ -DCMAKE_CXX_FLAGS=\"-O3 -DNDEBUG\" \ -DCMAKE_C_FLAGS=\"-O3 -DNDEBUG\" \ + -DCMAKE_SHARED_LINKER_FLAGS=\"-Wl,--export-dynamic -Wl,--no-as-needed\" \ -DLLVM_ROOT=/opt/llvm-16.0.6 \ -DLLVM_ROOT_DIR=/opt/llvm-16.0.6 \ + -DLLVM_CONFIG=/opt/llvm-16.0.6/bin/llvm-config \ -DPython3_EXECUTABLE=/opt/_internal/cpython-3.11.13/bin/python3.11 \ -DPython3_ROOT_DIR=/opt/_internal/cpython-3.11.13 \ @@ -73,7 +75,9 @@ if [ "$TEST_ONLY" = false ]; then echo "Building wheel with setup.py..." # Exclude history server from the wheel to reduce size and dependencies echo "Setting TUPLEX_INCLUDE_HISTORYSERVER=False to exclude history server" - TUPLEX_INCLUDE_HISTORYSERVER=False CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel + export TUPLEX_INCLUDE_HISTORYSERVER=False + echo "Environment variable TUPLEX_INCLUDE_HISTORYSERVER is set to: $TUPLEX_INCLUDE_HISTORYSERVER" + CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel # Find the built wheel WHEEL_FILE=$(find dist/ -name "*.whl" | head -1) diff --git a/tuplex/python/CMakeLists.txt b/tuplex/python/CMakeLists.txt index dc5a3498c..a78e58945 100644 --- a/tuplex/python/CMakeLists.txt +++ b/tuplex/python/CMakeLists.txt @@ -60,18 +60,11 @@ message(STATUS "includes: ${Python3_INCLUDE_DIRS}") #include(${pybind11_SOURCE_DIR}/tools/pybind11Tools.cmake) #include(${pybind11_SOURCE_DIR}/tools/pybind11NewTools.cmake) -## Method 1: -## cf. https://pybind11.readthedocs.io/en/stable/cmake/index.html -#pybind11_add_module(${MODULE_NAME} ${SOURCES}) -#set(PYMODULE "PYBIND11_MODULE(${MODULE_NAME}, m)") -#target_compile_definitions(${MODULE_NAME} PRIVATE PYMODULE=${PYMODULE}) - -# Method 2: -# Python method: -find_package(Python COMPONENTS Interpreter Development) -Python3_add_library(${MODULE_NAME} ${SOURCES}) -target_link_libraries(${MODULE_NAME} PUBLIC pybind11::headers) - +# Method 1: +# cf. https://pybind11.readthedocs.io/en/stable/cmake/index.html +pybind11_add_module(${MODULE_NAME} ${SOURCES}) +set(PYMODULE "PYBIND11_MODULE(${MODULE_NAME}, m)") +target_compile_definitions(${MODULE_NAME} PRIVATE PYMODULE=${PYMODULE}) # Macro/function to detect musl libc or Alpine Linux function(detect_musl OUT_VAR) @@ -136,7 +129,7 @@ detect_musl(MUSL_DETECTED) if(MUSL_DETECTED) message(STATUS "muslc detected, adjusting linker flags") - + # # Option 1: # # There's something going wrong on Alpine, all the symbols get optimized away. Therefore manually disable for target. # message(STATUS "Detected musl libc or Alpine Linux, disabling interprocedural optimization for ${MODULE_NAME}") From b73b0cfa68246eaacff19fd1cd4161496aff1718 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 18:58:19 -0700 Subject: [PATCH 91/92] ro --- scripts/alpine/make-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/alpine/make-wheels.sh b/scripts/alpine/make-wheels.sh index 79ae977ec..bed5d940e 100755 --- a/scripts/alpine/make-wheels.sh +++ b/scripts/alpine/make-wheels.sh @@ -56,7 +56,7 @@ echo -e "${GREEN}✓ tuplex/alpine-wheel-builder Docker image built successfully # Mount the entire project root to /code so setup.py and other files are accessible # Note: Must be writable as setup.py needs to create directories during build PROJECT_ROOT="$(dirname "$SCRIPT_DIR")/.." -MOUNT_OPTS="-v $PROJECT_ROOT:/code" +MOUNT_OPTS="-v $PROJECT_ROOT:/code:ro" echo "Launching tuplex/alpine-wheel-builder container with tuplex/ mounted..." From cf8ffc8ccacab5da8e847c8458911eee003862ee Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 7 Sep 2025 19:02:32 -0700 Subject: [PATCH 92/92] better approach with building --- scripts/alpine/builder/build.sh | 35 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/alpine/builder/build.sh b/scripts/alpine/builder/build.sh index 5eaded3f3..e7330f53b 100644 --- a/scripts/alpine/builder/build.sh +++ b/scripts/alpine/builder/build.sh @@ -22,17 +22,21 @@ PIP="/opt/_internal/cpython-3.11.13/bin/pip" echo "Using Python: $PYTHON" echo "Python version: $($PYTHON --version)" -# Check if the project root is mounted -if [ ! -f "/code/setup.py" ]; then - echo "Error: /code/setup.py not found. Make sure the project root is mounted." - exit 1 -fi + # Check if the project root is mounted + if [ ! -f "/code/setup.py" ]; then + echo "Error: /code/setup.py not found. Make sure the project root is mounted." + exit 1 + fi -# Check if wheelhouse directory is mounted -if [ ! -d "/wheelhouse" ]; then - echo "Error: /wheelhouse directory not found. Make sure the wheelhouse is mounted." - exit 1 -fi + # Check if wheelhouse directory is mounted + if [ ! -d "/wheelhouse" ]; then + echo "Error: /wheelhouse directory not found. Make sure the wheelhouse is mounted." + exit 1 + fi + + # Create a writable build directory in /work (since /code is read-only) + echo "Setting up writable build directory in /work..." + mkdir -p /work/build if [ "$TEST_ONLY" = false ]; then echo "Installing required dependencies..." @@ -44,9 +48,6 @@ if [ "$TEST_ONLY" = false ]; then echo "Building wheel for tuplex..." - # Change to the tuplex source directory - cd /code - export CMAKE_ARGS=" -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DBUILD_WITH_AWS=OFF \ @@ -61,7 +62,6 @@ if [ "$TEST_ONLY" = false ]; then -DCMAKE_SHARED_LINKER_FLAGS=\"-Wl,--export-dynamic -Wl,--no-as-needed\" \ -DLLVM_ROOT=/opt/llvm-16.0.6 \ -DLLVM_ROOT_DIR=/opt/llvm-16.0.6 \ - -DLLVM_CONFIG=/opt/llvm-16.0.6/bin/llvm-config \ -DPython3_EXECUTABLE=/opt/_internal/cpython-3.11.13/bin/python3.11 \ -DPython3_ROOT_DIR=/opt/_internal/cpython-3.11.13 \ @@ -77,7 +77,12 @@ if [ "$TEST_ONLY" = false ]; then echo "Setting TUPLEX_INCLUDE_HISTORYSERVER=False to exclude history server" export TUPLEX_INCLUDE_HISTORYSERVER=False echo "Environment variable TUPLEX_INCLUDE_HISTORYSERVER is set to: $TUPLEX_INCLUDE_HISTORYSERVER" - CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py bdist_wheel + + # Change to the source directory and build with writable build directory + cd /code + export TMPDIR=/work/tmp + mkdir -p /work/tmp + CMAKE_ARGS="$CMAKE_ARGS" $PYTHON setup.py build --build-base /work/build bdist_wheel # Find the built wheel WHEEL_FILE=$(find dist/ -name "*.whl" | head -1)