From 0f2f3b3ae09e8f2baf28c7b2c9002ae1117daa92 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 6 Aug 2025 23:08:47 -0700 Subject: [PATCH 01/11] ubuntu 24.04 scripts --- scripts/ubuntu2404/Dockerfile | 70 ++++++ scripts/ubuntu2404/install_requirements.sh | 265 +++++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 scripts/ubuntu2404/Dockerfile create mode 100755 scripts/ubuntu2404/install_requirements.sh diff --git a/scripts/ubuntu2404/Dockerfile b/scripts/ubuntu2404/Dockerfile new file mode 100644 index 000000000..e3cab66cc --- /dev/null +++ b/scripts/ubuntu2404/Dockerfile @@ -0,0 +1,70 @@ +# Tuplex ubuntu 24.04 image, holds code in /code directory and allows to build tuplex easily. +# create with docker build -t tuplex/ubuntu:24.04 . +# Run with docker run -it tuplex/ubuntu:24.04 bash +FROM ubuntu:24.04 +LABEL authors="leonhards" + +# Versions, environment variables. +# Needs to be a pyenv supported version: pyenv install --list. +ENV PYTHON_VERSION=3.13.5 +ENV GITHUB_BRANCH=master + +WORKDIR /tmp + +# Install python version +# set the variables as per $(pyenv init -) +ENV LANG="C.UTF-8" \ + LC_ALL="C.UTF-8" \ + PATH="/opt/pyenv/shims:/opt/pyenv/bin:$PATH" \ + PYENV_ROOT="/opt/pyenv" \ + PYENV_SHELL="bash" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + libbz2-dev \ + libffi-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + liblzma-dev \ + libssl-dev \ + make \ + netbase \ + pkg-config \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + + +RUN git clone --single-branch --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT \ + && pyenv install $PYTHON_VERSION \ + && pyenv global $PYTHON_VERSION \ + && find $PYENV_ROOT/versions -type d '(' -name '__pycache__' -o -name 'test' -o -name 'tests' ')' -exec rm -rf '{}' + \ + && find $PYENV_ROOT/versions -type f '(' -name '*.pyo' -o -name '*.exe' ')' -exec rm -f '{}' + \ + && rm -rf /tmp/* + + +# --- install requirements --- + +ADD install_requirements.sh /tmp/install_requirements.sh + +RUN /tmp/install_requirements.sh + +# installs tuplex-specific dependencies into /opt +RUN mkdir -p /opt + +# downloads/clones tuplex into /code directory +RUN mkdir -p /code + +RUN rm -rf /tmp/* + +WORKDIR /code + +# Clone tuplex, and build initial version as well as install in python. +RUN git clone -b ${GITHUB_BRANCH} --single-branch https://github.com/tuplex/tuplex.git \ No newline at end of file diff --git a/scripts/ubuntu2404/install_requirements.sh b/scripts/ubuntu2404/install_requirements.sh new file mode 100755 index 000000000..107af2aa0 --- /dev/null +++ b/scripts/ubuntu2404/install_requirements.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# (c) Tuplex team 2017-2023 +# Installs all tuplex dependencies required to build tuplex. + +# Variables needed incl. defaults. +PREFIX=${PREFIX:-/opt} +WORKDIR=${WORKDIR:-/tmp} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +CMAKE_VERSION="3.27.5" +BOOST_VERSION="1.79.0" +LLVM_VERSION="16.0.6" +AWSSDK_CPP_VERSION=1.11.524 +ANTLR4_VERSION=4.13.1 +YAML_CPP_VERSION=0.8.0 +AWS_LAMBDA_CPP_VERSION=0.2.10 +PCRE2_VERSION=10.45 +PROTOBUF_VERSION=24.3 +CELERO_VERSION=2.8.3 +CC=gcc +CXX=g++ + +CPU_COUNT=$(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) + +PYTHON_VERSION=$(echo $(python3 --version) | cut -d ' ' -f2) +PYTHON_MAJMIN_VERSION=${PYTHON_VERSION%.*} +echo ">> Installing dependencies for Python version ${PYTHON_VERSION}" + +function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } + +# Start script. +set -euxo pipefail + +# need to run this with root privileges +if [[ $(id -u) -ne 0 ]]; then + echo "Please run this script with root privileges" + exit 1 +fi + +# --- helper functions --- +# Parse the major, minor and patch versions. +# You use it like this: +# semver="3.4.5+xyz" +# a=($(parse_semver "$semver")) +# major=${a[0]} +# minor=${a[1]} +# patch=${a[2]} +# printf "%-32s %4d %4d %4d\n" "$semver" $major $minor $patch +function parse_semver() { + local token="$1" + local major=0 + local minor=0 + local patch=0 + + if egrep '^[0-9]+\.[0-9]+\.[0-9]+' <<<"$token" >/dev/null 2>&1 ; then + # It has the correct syntax. + local n=${token//[!0-9]/ } + local a=(${n//\./ }) + major=${a[0]} + minor=${a[1]} + patch=${a[2]} + fi + + echo "$major $minor $patch" +} + +function install_llvm { + LLVM_VERSION=$1 + LLVM_MAJOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f1` + LLVM_MINOR_VERSION=`echo ${LLVM_VERSION} | cut -d. -f2` + LLVM_MAJMIN_VERSION="${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}" + + # list of targets available to build: AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SystemZ;VE;WebAssembly;X86;XCore + # in order to cross-compile, should use targets: + + + echo ">> building LLVM ${LLVM_VERSION}" + LLVM_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz + CLANG_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/clang-${LLVM_VERSION}.src.tar.xz + # required when LLVM version > 15 + LLVM_CMAKE_URL=https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cmake-${LLVM_VERSION}.src.tar.xz + + PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} + PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" + PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) + echo ">> Building dependencies for ${PYTHON_VERSION}" + + echo ">> Downloading prerequisites for llvm ${LLVM_VERSION}}" + LLVM_WORKDIR=${WORKDIR}/llvm${LLVM_VERSION} + mkdir -p ${LLVM_WORKDIR} + pushd "${LLVM_WORKDIR}" || exit 1 + + wget ${LLVM_URL} && tar xf llvm-${LLVM_VERSION}.src.tar.xz + wget ${CLANG_URL} && tar xf clang-${LLVM_VERSION}.src.tar.xz && mv clang-${LLVM_VERSION}.src llvm-${LLVM_VERSION}.src/../clang + + if (( LLVM_MAJOR_VERSION >= 15 )); then + wget ${LLVM_CMAKE_URL} && tar xf cmake-${LLVM_VERSION}.src.tar.xz && mv cmake-${LLVM_VERSION}.src cmake + fi + + mkdir -p llvm-${LLVM_VERSION}.src/build && cd llvm-${LLVM_VERSION}.src/build + + cmake -GNinja -DLLVM_ENABLE_RTTI=ON -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm-${LLVM_VERSION} .. + ninja install + popd +} + +export DEBIAN_FRONTEND=noninteractive + + + +echo ">> Installing packages into ${PREFIX}" +mkdir -p $PREFIX && chmod 0755 $PREFIX +mkdir -p $PREFIX/sbin +mkdir -p $PREFIX/bin +mkdir -p $PREFIX/share +mkdir -p $PREFIX/include +mkdir -p $PREFIX/lib + +echo ">> Files will be downloaded to ${WORKDIR}/tuplex-downloads" +WORKDIR=$WORKDIR/tuplex-downloads +mkdir -p $WORKDIR + +PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" +PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) +echo ">> Building dependencies for ${PYTHON_VERSION}" +echo ">> Installing all build dependencies for Tuplex under Ubuntu 22.04" + +echo ">> Installing apt dependencies" +apt update -y + +apt-get install -y apt-utils dh-autoreconf libmagic-dev curl libxml2-dev vim build-essential libssl-dev zlib1g-dev libncurses5-dev \ +libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev openssh-client \ +libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev wget git libcurl4-openssl-dev python3-dev python3-pip openjdk-11-jdk ninja-build + +ldconfig +export CC=${CC} +export CXX=${CXX} + +echo ">> Installing recent cmake" +# fetch recent cmake & install +URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz +mkdir -p ${WORKDIR}/cmake && cd ${WORKDIR}/cmake && + curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && + tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && + rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && + cd cmake-${CMAKE_VERSION}-linux-x86_64 && + cp -rp bin/* ${PREFIX}/bin/ && + cp -rp share/* ${PREFIX}/share/ && + cd / && rm -rf ${WORKDIR}/cmake + +export PATH=$PREFIX/bin:$PATH +cmake --version + + +echo ">> Installing Boost" +mkdir -p ${WORKDIR}/boost + +# -- Install Boost. +# create underscored version +# e.g. 1.79.0 -> 1_79_0 +BOOST_UNDERSCORED_VERSION=$(echo ${BOOST_VERSION} | tr . _) + +# build incl. boost python +cd ${WORKDIR}/boost && curl -L -O https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-b2-nodocs.tar.gz && tar xf boost-${BOOST_VERSION}-b2-nodocs.tar.gz && cd ${WORKDIR}/boost/boost-${BOOST_VERSION} \ + && ./bootstrap.sh --with-python=${PYTHON_EXECUTABLE} --prefix=${PREFIX} --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" \ + && ./b2 cxxflags="-fPIC" link=static -j "$(nproc)" \ + && ./b2 cxxflags="-fPIC" link=static install && sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' ${PREFIX}/include/boost/thread/pthread/thread_data.hpp + +cd $WORKDIR +rm -rf ${WORKDIR}/boost + +# -- install llvm +install_llvm $LLVM_VERSION + + +echo ">>> Installing tuplex dependencies." +# install recent zlib version (1.2.11) fork from cloudflare +# https://github.com/aws/aws-graviton-getting-started#zlib-on-linux +LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64:$LD_LIBRARY_PATH + +# Cloudflare fork is too old +#mkdir -p $WORKDIR/zlib && cd $WORKDIR && git clone https://github.com/cloudflare/zlib.git && cd zlib && ./configure --prefix=$PREFIX && make -j ${CPU_COUNT} && make install + +# note that zlib defines Z_NULL=0 whereas zlib-ng defines it as NULL, patch aws sdk accordingly +git clone https://github.com/zlib-ng/zlib-ng.git && cd zlib-ng && git checkout tags/2.1.3 && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC" -DZLIB_COMPAT=ON .. && make -j ${CPU_COUNT} && make install + +git clone https://github.com/google/googletest.git -b v1.14.0 && cd googletest && mkdir build && cd build && cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_BUILD_TYPE=Release .. && make -j ${CPU_COUNT} && make install + +# build snappy as static lib +git clone https://github.com/google/snappy.git -b 1.1.10 && cd snappy && git submodule update --init && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. && make -j ${CPU_COUNT} && make install + +# add github to known hosts +mkdir -p /root/.ssh/ && + touch /root/.ssh/known_hosts && + ssh-keyscan github.com >> /root/.ssh/known_hosts + + +echo ">> Installing YAMLCPP" +mkdir -p ${WORKDIR}/yamlcpp && cd ${WORKDIR}/yamlcpp \ +&& git clone https://github.com/jbeder/yaml-cpp.git yaml-cpp \ +&& cd yaml-cpp \ +&& git checkout tags/${YAML_CPP_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DYAML_CPP_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Installing Celero" +mkdir -p ${WORKDIR}/celero && cd ${WORKDIR}/celero \ +&& git clone https://github.com/DigitalInBlue/Celero.git celero && cd celero \ +&& git checkout tags/v${CELERO_VERSION} \ +&& mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="-fPIC -std=c++11" .. \ +&& make -j ${CPU_COUNT} && make install + +echo ">> Installing ANTLR" +mkdir -p ${WORKDIR}/antlr && cd ${WORKDIR}/antlr \ +&& curl -O https://www.antlr.org/download/antlr-${ANTLR4_VERSION}-complete.jar \ +&& cp antlr-${ANTLR4_VERSION}-complete.jar ${PREFIX}/lib/ \ +&& curl -O https://www.antlr.org/download/antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& unzip antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip -d antlr4-cpp-runtime \ +&& rm antlr4-cpp-runtime-${ANTLR4_VERSION}-source.zip \ +&& cd antlr4-cpp-runtime \ +&& mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT}&& make install + +echo ">> Installing AWS SDK" +# Note the z-lib patch here. +mkdir -p ${WORKDIR}/aws && cd ${WORKDIR}/aws \ +&& git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git \ +&& cd aws-sdk-cpp && git checkout tags/${AWSSDK_CPP_VERSION} && sed -i 's/int ret = Z_NULL;/int ret = static_cast(Z_NULL);/g' src/aws-cpp-sdk-core/source/client/RequestCompression.cpp && mkdir build && cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DUSE_CRT_HTTP_CLIENT=ON -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=17 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;s3-crt;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j ${CPU_COUNT} \ +&& make install + +# Installing AWS Lambda C++ runtime. +cd ${WORKDIR}/aws \ +&& git clone https://github.com/awslabs/aws-lambda-cpp.git \ +&& cd aws-lambda-cpp \ +&& git fetch && git fetch --tags \ +&& git checkout v${AWS_LAMBDA_CPP_VERSION} \ +&& mkdir build \ +&& cd build \ +&& cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} .. \ +&& make -j$(nproc) && make install + +echo ">> Installing PCRE2" +mkdir -p ${WORKDIR}/pcre2 && cd ${WORKDIR}/pcre2 \ +&& curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-${PCRE2_VERSION}/pcre2-${PCRE2_VERSION}.zip \ +&& unzip pcre2-${PCRE2_VERSION}.zip \ +&& rm pcre2-${PCRE2_VERSION}.zip \ +&& cd pcre2-${PCRE2_VERSION} \ +&& ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \ +&& make -j$(nproc) && make install + +echo ">> Installing protobuf" +mkdir -p ${WORKDIR}/protobuf && cd ${WORKDIR}/protobuf \ +&& git clone -b v${PROTOBUF_VERSION} https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j ${CPU_COUNT} && make install + + +# delete workdir (downloads dir) to clean up space +rm -rf ${WORKDIR} + +echo "-- Done, all Tuplex requirements installed to /opt --" From a8c18da61bf33f37b9eb50d0261d2546457c419d Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 6 Aug 2025 23:33:46 -0700 Subject: [PATCH 02/11] boost fix --- scripts/ubuntu2404/install_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ubuntu2404/install_requirements.sh b/scripts/ubuntu2404/install_requirements.sh index 107af2aa0..8dce6efb6 100755 --- a/scripts/ubuntu2404/install_requirements.sh +++ b/scripts/ubuntu2404/install_requirements.sh @@ -7,7 +7,7 @@ PREFIX=${PREFIX:-/opt} WORKDIR=${WORKDIR:-/tmp} PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} CMAKE_VERSION="3.27.5" -BOOST_VERSION="1.79.0" +BOOST_VERSION="1.88.0" LLVM_VERSION="16.0.6" AWSSDK_CPP_VERSION=1.11.524 ANTLR4_VERSION=4.13.1 From bf94c6d1f51a88b732a873d028b8479c42767243 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 7 Aug 2025 15:19:26 -0700 Subject: [PATCH 03/11] fixes --- scripts/ubuntu2404/Dockerfile | 7 ++++++- scripts/ubuntu2404/install_requirements.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/ubuntu2404/Dockerfile b/scripts/ubuntu2404/Dockerfile index e3cab66cc..7a8f516a2 100644 --- a/scripts/ubuntu2404/Dockerfile +++ b/scripts/ubuntu2404/Dockerfile @@ -67,4 +67,9 @@ RUN rm -rf /tmp/* WORKDIR /code # Clone tuplex, and build initial version as well as install in python. -RUN git clone -b ${GITHUB_BRANCH} --single-branch https://github.com/tuplex/tuplex.git \ No newline at end of file +RUN git clone -b ${GITHUB_BRANCH} --single-branch https://github.com/tuplex/tuplex.git + +# Install python dependencies +RUN python3 -m pip install cloudpickle numpy pandas + +# \ No newline at end of file diff --git a/scripts/ubuntu2404/install_requirements.sh b/scripts/ubuntu2404/install_requirements.sh index 8dce6efb6..5175a1fc2 100755 --- a/scripts/ubuntu2404/install_requirements.sh +++ b/scripts/ubuntu2404/install_requirements.sh @@ -130,7 +130,7 @@ echo ">> Installing apt dependencies" apt update -y apt-get install -y apt-utils dh-autoreconf libmagic-dev curl libxml2-dev vim build-essential libssl-dev zlib1g-dev libncurses5-dev \ -libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev openssh-client \ +libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev openssh-client unzip \ libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev wget git libcurl4-openssl-dev python3-dev python3-pip openjdk-11-jdk ninja-build ldconfig From 6aa2aef4835aa25d098be261a2dbb3f8db40a7da Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 15:56:52 -0700 Subject: [PATCH 04/11] fixes --- tuplex/CMakeLists.txt | 2 ++ tuplex/utils/include/Base.h | 1 + 2 files changed, 3 insertions(+) diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index 8abb640d0..7adf4b7ef 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -6,6 +6,8 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) message(STATUS "Using language version: C++${CMAKE_CXX_STANDARD}") +set(CMAKE_C_STANDARD 17) +set(CMAKE_C_STANDARD_REQUIRED ON) # add cmake modules from cmake folder list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/") diff --git a/tuplex/utils/include/Base.h b/tuplex/utils/include/Base.h index 2475a9514..da3db47b7 100644 --- a/tuplex/utils/include/Base.h +++ b/tuplex/utils/include/Base.h @@ -13,6 +13,7 @@ #include #include +#include #include #include From 158b3b42618f2cdd8f3ccd7c985417c467e61fd7 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 16:00:53 -0700 Subject: [PATCH 05/11] fix --- tuplex/utils/include/Field.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tuplex/utils/include/Field.h b/tuplex/utils/include/Field.h index fef968870..0ecb2d896 100644 --- a/tuplex/utils/include/Field.h +++ b/tuplex/utils/include/Field.h @@ -12,6 +12,7 @@ #define TUPLEX_FIELD_H #include +#include #include #include #include From f7fd585eae90d1809626adf15dd1f08ac8db01e5 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 16:18:55 -0700 Subject: [PATCH 06/11] missing includes --- tuplex/io/include/VirtualMappedFile.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tuplex/io/include/VirtualMappedFile.h b/tuplex/io/include/VirtualMappedFile.h index e7c65f374..0d21f30da 100644 --- a/tuplex/io/include/VirtualMappedFile.h +++ b/tuplex/io/include/VirtualMappedFile.h @@ -11,6 +11,9 @@ #ifndef TUPLEX_VIRTUALMAPPEDFILE_H #define TUPLEX_VIRTUALMAPPEDFILE_H +#include +#include + namespace tuplex { class VirtualMappedFile; class VirtualFileSystem; From dd7cb1c6093b3e3637da25260e3105366c41fbdf Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 16:19:52 -0700 Subject: [PATCH 07/11] include fix --- tuplex/io/include/VirtualFileSystemBase.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tuplex/io/include/VirtualFileSystemBase.h b/tuplex/io/include/VirtualFileSystemBase.h index 31b0087a6..d9c1c50f6 100644 --- a/tuplex/io/include/VirtualFileSystemBase.h +++ b/tuplex/io/include/VirtualFileSystemBase.h @@ -11,6 +11,8 @@ #ifndef TUPLEX_VIRTUALFILESYSTEMBASE_H #define TUPLEX_VIRTUALFILESYSTEMBASE_H +#include + namespace tuplex { enum class VirtualFileSystemStatus { VFS_OK = 0, From 79da010e6ed788a35e1647912d55adcf00cfadd8 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 16:20:33 -0700 Subject: [PATCH 08/11] fix --- tuplex/codegen/include/Token.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tuplex/codegen/include/Token.h b/tuplex/codegen/include/Token.h index 7688a4dbe..b6ca1b713 100644 --- a/tuplex/codegen/include/Token.h +++ b/tuplex/codegen/include/Token.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include From d7a2973c0ec980f102602b7158f8cec5d41e2303 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 16:21:23 -0700 Subject: [PATCH 09/11] fix --- tuplex/core/include/RESTInterface.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tuplex/core/include/RESTInterface.h b/tuplex/core/include/RESTInterface.h index ddf2d9bb8..ba1358250 100644 --- a/tuplex/core/include/RESTInterface.h +++ b/tuplex/core/include/RESTInterface.h @@ -13,6 +13,7 @@ #include #include +#include #include #include From 1ebca33b5215a5a26169e61396bc8e38dcc3e9c2 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 20:43:28 -0700 Subject: [PATCH 10/11] fixes --- scripts/ubuntu2404/Dockerfile | 9 ++++++++- scripts/ubuntu2404/install_requirements.sh | 2 +- tuplex/codegen/include/CodegenHelper.h | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/ubuntu2404/Dockerfile b/scripts/ubuntu2404/Dockerfile index 7a8f516a2..716f4c9ae 100644 --- a/scripts/ubuntu2404/Dockerfile +++ b/scripts/ubuntu2404/Dockerfile @@ -72,4 +72,11 @@ RUN git clone -b ${GITHUB_BRANCH} --single-branch https://github.com/tuplex/tupl # Install python dependencies RUN python3 -m pip install cloudpickle numpy pandas -# \ No newline at end of file +# Add paths to compile, +RUN echo "export PATH=/opt/bin:${PATH}" >> /root/.bashrc +RUN echo "export LLVM_ROOT=/opt/llvm-16.0.6" >> /root/.bashrc + +# Within the docker container, compile tuplex with +# cd /code/tuplex/tuplex && mkdir -p build && cd build && cmake -DLLVM_ROOT=/opt/llvm-16.0.6/ -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON .. && make +# You can test via ctest +# or follow instructions to test the python package. \ No newline at end of file diff --git a/scripts/ubuntu2404/install_requirements.sh b/scripts/ubuntu2404/install_requirements.sh index 5175a1fc2..3937a5056 100755 --- a/scripts/ubuntu2404/install_requirements.sh +++ b/scripts/ubuntu2404/install_requirements.sh @@ -124,7 +124,7 @@ mkdir -p $WORKDIR PYTHON_BASENAME="$(basename -- $PYTHON_EXECUTABLE)" PYTHON_VERSION=$(${PYTHON_EXECUTABLE} --version) echo ">> Building dependencies for ${PYTHON_VERSION}" -echo ">> Installing all build dependencies for Tuplex under Ubuntu 22.04" +echo ">> Installing all build dependencies for Tuplex under Ubuntu 24.04" echo ">> Installing apt dependencies" apt update -y diff --git a/tuplex/codegen/include/CodegenHelper.h b/tuplex/codegen/include/CodegenHelper.h index 63af95f38..9649dc9d7 100644 --- a/tuplex/codegen/include/CodegenHelper.h +++ b/tuplex/codegen/include/CodegenHelper.h @@ -763,7 +763,13 @@ namespace tuplex { llvm::Instruction& inst = *firstBlock.getFirstInsertionPt(); ctorBuilder.SetInsertPoint(&inst); } + + // llvm IR builder has some issues with copy disallowed. C++ compilers will issue -Wreturn-local-addr. + // Disable warning here. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wreturn-local-addr" return std::move(ctorBuilder); +#pragma GCC diagnostic pop } // in order to serialize/deserialize data properly and deal with From 712dc99d75bd2cd58076751c86750cb2daa32bc3 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Fri, 8 Aug 2025 20:49:28 -0700 Subject: [PATCH 11/11] cancel previous runs --- .github/workflows/build_wheels.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 682c184dc..f4dc95937 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -2,6 +2,11 @@ name: Build on: [push, pull_request, workflow_dispatch] +# Cancel previous runs, i.e. run only latest push. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + # For macos, at least 10.13 is required # to avoid issues and since the runners are macos-13 and macos-14: # -> use 13.6, which is Venture from 2022 and 14.0 on the arm runners.