tuplex · LeonhardFS · Feb 24, 2025 · Feb 10, 2024 · Feb 11, 2024 · Feb 11, 2024
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -2,45 +2,50 @@ name: Build
 
 on: [push, pull_request, workflow_dispatch]
 
+env:
+  # At least 10.13 is required, to avoid issues and since the runner is macos-13 -> use 13.0, which is Venture from 2022.
+  MACOSX_DEPLOYMENT_TARGET: 13.0
+
 jobs:
   build_wheels:
     name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-11 ]
+        # macos-14 (which is macos-latest) is ARM only. macos-13 is latest intel runner.
+        os: [ ubuntu-latest, macos-13 ]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             python-version: "3.8"
             cibw-build: "cp38-manylinux_x86_64"
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             python-version: "3.9"
             cibw-build: "cp39-manylinux_x86_64"
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             python-version: "3.10"
             cibw-build: "cp310-manylinux_x86_64"
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             python-version: "3.11"
             cibw-build: "cp311-manylinux_x86_64"
-          - os: macos-11
+          - os: macos-13
             python-version: "3.8"
             cibw-build: "cp38-macosx_x86_64"
-          - os: macos-11
+          - os: macos-13
             python-version: "3.9"
             cibw-build: "cp39-macosx_x86_64"
-          - os: macos-11
+          - os: macos-13
             python-version: "3.10"
             cibw-build: "cp310-macosx_x86_64"
-          - os: macos-11
+          - os: macos-13
             python-version: "3.11"
             cibw-build: "cp311-macosx_x86_64"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Cache brew dependencies
         if: runner.os == 'macOS'
-        uses: actions/cache@v3.3.2
+        uses: actions/cache@v4.2.1
         with:
           # Paths to cache:
           # /usr/local/Homebrew - installation folder of Homebrew
@@ -52,41 +57,58 @@ jobs:
             /usr/local/Frameworks
             /usr/local/bin
             /usr/local/opt
-          key: macos-11-build-cache-${{ hashFiles('./scripts/macos/brew_dependencies.sh') }}-v2
+          key: macos-13-build-cache-${{ hashFiles('./scripts/macos/brew_dependencies.sh') }}-v2
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11' 
 
-      # need to make this an intermediate step, i.e. build first the different lambda runners on Ubuntu...
-      - name: Build Lambda runner (Linux only)
-        if: runner.os != 'macOS'
-        run: docker pull registry-1.docker.io/tuplex/ci:${{ matrix.python-version }} && export PYTHON3_VERSION=${{ matrix.python-version }}.0 && bash ./scripts/create_lambda_zip.sh && mkdir -p ./tuplex/python/tuplex/other && cp ./build-lambda/tplxlam.zip ./tuplex/python/tuplex/other
-        shell: bash
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.22.0
 
-      - name: Build wheel
-        #if: runner.os != 'macOS'
-        uses: pypa/cibuildwheel@fff9ec32ed25a9c576750c91e06b410ed0c15db7 # hash corresponds to v2.16.2
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
         env:
           # configure cibuildwheel to build native archs ('auto'), and some
           # emulated ones
           CIBW_ARCHS_LINUX: native
           CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}"
           CIBW_BUILD: ${{ matrix.cibw-build }}
 
-          # macOS dependencies separate, for linux use docker tuplex/ci:3.x images.
+          # macOS dependencies separate, for Linux use docker tuplex/ci:3.x images.
           CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile
 
-          # bundle aws runner with linux wheel, remove environment variable TUPLEX_LAMBDA_ZIP to remove runner.
-          CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
+          # If CI complains about missing /usr/local/libexec/git-core/git-remote-https: error while loading shared libraries: libssl.so.3: cannot open shared object file: No such file or directory
+          # the OpenSSL3 lib is stored under /usr/local/lib64.
+          CIBW_ENVIRONMENT_LINUX: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:/opt/lib"
+
+          # Requires macOS 10.13 at least to build because of C++17 features.
+          # To avoid issues, simply use 13.0 for now.
+          CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64}"
 
-          # requires macOS 10.13 at least to build because of C++17 features.
-          CIBW_ENVIRONMENT_MACOS: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64}"
           # run all python tests to make sure wheels are not defunct
           CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter"
+
+          # Use following test command when segfaults happen to better pinpoint:
+          # python3 -X faulthandler -m pytest -p no:faulthandler
+          # else can use pytest ...
+
           # use 3min timeout per test and print top 25 slowest tests
-          CIBW_TEST_COMMAND: "cd {project} && pytest tuplex/python/tests -v --timeout 600 --durations 25"
+          CIBW_TEST_COMMAND: "cd {project} && python3 -X faulthandler -m pytest -p no:faulthandler tuplex/python/tests -v --timeout 600 --durations 25"
 
       - name: reorganize files
         run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./.github/scripts/test_pypi.sh ./wheelhouse
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: wheels-${{ matrix.os }}-${{ matrix.cibw-build }}
           path: |
             ./wheelhouse/*.whl
+
+# Note: when using download-artifact, use
+#       - uses: actions/download-artifact@v4
+#         with:
+#           path: dist
+#           merge-multiple: true
+#        # Requires 4.1
diff --git a/README.md b/README.md
@@ -133,4 +133,4 @@ series = {SIGMOD/PODS '21}
 ```
 
 ---
-(c) 2017-2023 Tuplex contributors
+(c) 2017-2025 Tuplex contributors
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -22,13 +22,13 @@ jobs:
         displayName: 'Install MongoDB'
       - script: sudo bash scripts/azure/install_azure_ci_reqs.sh
         displayName: 'Install required packages'
-      - script: sudo apt-get install -y python3-setuptools ninja-build && python3 -m pip install pytest pygments>=2.4.1 MarkupSafe==2.0 pexpect setuptools astor PyYAML jupyter nbformat pymongo eventlet==0.30.0 gunicorn pymongo && jupyter --version
+      - script: sudo apt-get install -y python3-setuptools ninja-build && python3 -m pip install pytest pygments>=2.4.1 MarkupSafe==2.0 pexpect setuptools astor PyYAML jupyter nbformat pymongo eventlet==0.30.0 gunicorn pymongo "lxml[html_clean]" && jupyter --version
         displayName: 'Install python dependencies'
       - script: cd tuplex/python && python3 -m pip install -r requirements.txt && python3 mongodb_test.py && pkill mongod || true
         displayName: 'Test local MongoDB'
       - script: TUPLEX_BUILD_ALL=1 CMAKE_ARGS="-DBUILD_WITH_ORC=ON -DLLVM_ROOT_DIR=/usr/lib/llvm-16 -DCMAKE_BUILD_TYPE=Release -DBUILD_FOR_CI=ON" python3 setup.py install --user
         displayName: 'Build Tuplex'
       - script: cd build/temp.linux-x86_64-3.10 && ctest --timeout 180 --output-on-failure --repeat until-pass:3 -j 2
         displayName: 'C++ tests'
-      - script: cd build/temp.linux-x86_64-3.10/dist/python && python3.10 -m pytest -x --full-trace -l --log-cli-level=DEBUG --capture=tee-sys
+      - script: cd build/temp.linux-x86_64-3.10/dist/python && python3.10 -m pip install lxml_html_clean && python3.10 -m pytest -x --full-trace -l --log-cli-level=DEBUG --capture=tee-sys
         displayName: 'Python tests'
diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -36,7 +36,7 @@
 # The short X.Y version
 version="0.3"
 # The full version, including alpha/beta/rc tags
-release="0.3.6"
+release="0.3.7"
 
 
 # -- General configuration ---------------------------------------------------

diff --git a/scripts/azure/install_azure_ci_reqs.sh b/scripts/azure/install_azure_ci_reqs.sh
@@ -86,7 +86,7 @@ mkdir -p /root/.ssh/ &&
 mkdir -p ${WORKDIR}/boost
 
 # build incl. boost python
-pushd ${WORKDIR}/boost && wget https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz && tar xf boost_1_79_0.tar.gz && cd ${WORKDIR}/boost/boost_1_79_0 \
+pushd ${WORKDIR}/boost && wget https://github.com/boostorg/boost/releases/download/boost-1.87.0/boost-1.87.0-b2-nodocs.tar.gz && tar xf boost-1.87.0-b2-nodocs.tar.gz && cd ${WORKDIR}/boost/boost-1.87.0 \
            && ./bootstrap.sh --with-python=${PYTHON_EXECUTABLE} --prefix=${PREFIX} --with-libraries="thread,iostreams,regex,system,filesystem,python,stacktrace,atomic,chrono,date_time" \
             && ./b2 cxxflags="-fPIC" link=static -j "$(nproc)" \
             && ./b2 cxxflags="-fPIC" link=static install && sed -i 's/#if PTHREAD_STACK_MIN > 0/#ifdef PTHREAD_STACK_MIN/g' ${PREFIX}/include/boost/thread/pthread/thread_data.hpp
@@ -138,5 +138,5 @@ mkdir -p ${WORKDIR}/pcre2 && cd ${WORKDIR}/pcre2 \
 && ./configure CFLAGS="-O2 -fPIC" --prefix=${PREFIX} --enable-jit=auto --disable-shared \
 && make -j$(nproc) && make install
 mkdir -p ${WORKDIR}/protobuf && cd ${WORKDIR}/protobuf && git clone -b v24.3 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF .. && make -j$(nproc) && make install && ldconfig
-pip3 install 'cloudpickle>2.0.0' cython numpy
+pip3 install 'cloudpickle>2.0.0' cython numpy 'lxml[html_clean]'
 echo ">>> installing reqs done."
diff --git a/scripts/build_macos_3.11_wheel_with_test.sh b/scripts/build_macos_3.11_wheel_with_test.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# (c) 2017-2023 Tuplex team
+# builds x86_64 (and arm64) wheels
+
+# add -x option for verbose output
+set -euo pipefail
+
+function fail {
+    printf '%s\n' "$1" >&2
+    exit "${2-1}"
+}
+
+function detect_instruction_set() {
+  arch="$(uname -m)"  # -i is only linux, -m is linux and apple
+  if [[ "$arch" = x86_64* ]]; then
+    if [[ "$(uname -a)" = *ARM64* ]]; then
+      echo 'arm64'
+    else
+      echo 'x86_64'
+    fi
+  elif [[ "$arch" = i*86 ]]; then
+    echo 'x86_32'
+  elif [[ "$arch" = arm* ]]; then
+    echo $arch
+  elif test "$arch" = aarch64; then
+    echo 'arm64'
+  else
+    exit 1
+	fi
+}
+
+# check from where script is invoked
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+echo " || Tuplex macOS wheel builder || "
+echo "-- Executing buildwheel script located in $CWD"
+
+# check platform is darwin
+if [ ! "$(uname -s)" = "Darwin" ]; then
+  fail "Error: Need to run script under macOS"
+fi
+
+# check which tags are supported
+arch=$(detect_instruction_set)
+echo "-- Detected arch ${arch}"
+
+# try to extract version of compiler first via command-line tools or xcode
+# either needs to be installed.
+xcode_version_str=$(pkgutil --pkg-info=com.apple.pkg.CLTools_Executables 2>/dev/null | grep version || pkgutil --pkg-info=com.apple.pkg.Xcode | grep version)
+echo "-- Detected Xcode ${xcode_version_str}"
+
+# if no param is given, use defaults to build all
+if [ "${arch}" = "arm64" ]; then
+  # build Python 3.9 - 3.11
+  CIBW_BUILD=${CIBW_BUILD-"cp311-macosx_arm64"}
+else
+  # build Python 3.8 - 3.11
+  CIBW_BUILD=${CIBW_BUILD-"cp311-macosx_x86_64"}
+fi
+
+echo "-- Building wheels for ${CIBW_BUILD}"
+
+MACOS_VERSION=$(sw_vers -productVersion)
+echo "-- Processing on MacOS ${MACOS_VERSION}"
+function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; }
+
+MACOS_VERSION_MAJOR=`echo $MACOS_VERSION | cut -d . -f1`
+
+if [ "$MACOS_VERSION_MAJOR" -ge 11 ]; then
+    echo "-- Newer MacOS detected (>=11.0), using more recent base target."
+    echo "-- Using minimum target ${MACOS_VERSION_MAJOR}.0"
+    MINIMUM_TARGET="${MACOS_VERSION_MAJOR}.0"
+else
+    # keep as is
+    echo "-- Defaulting build to use as minimum target ${MINIMUM_TARGET}"
+fi
+
+pushd $CWD > /dev/null
+cd ..
+
+# Use 13.0
+MINIMUM_TARGET=13.0
+export MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET}
+
+# Note: protobuf 3.20 - 3.21.2 is broken for MacOS, do not use those versions
+export CIBW_BEFORE_BUILD_MACOS="brew install protobuf coreutils zstd zlib libmagic llvm@16 aws-sdk-cpp pcre2 antlr4-cpp-runtime googletest gflags yaml-cpp celero wget boost ninja snappy libdwarf libelf"
+
+
+# Note: orc build breaks wheel right now...
+export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' TUPLEX_BUILD_TYPE=RelWithDebInfo"
+#export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' TUPLEX_BUILD_TYPE=Debug"
+
+export CIBW_BUILD="${CIBW_BUILD}"
+export CIBW_PROJECT_REQUIRES_PYTHON=">=3.8"
+
+# uncomment to increase verbosity of cibuildwheel
+export CIBW_BUILD_VERBOSITY=3
+
+# uncomment and set to specific identifier
+#export CIBW_BUILD="cp39-macosx_x86_64"
+
+export CIBW_TEST_REQUIRES="pytest pytest-timeout numpy nbformat jupyter"
+export CIBW_TEST_COMMAND="cd {project} && python3 -X faulthandler -m pytest -p no:faulthandler tuplex/python/tests --timeout_method thread --timeout 300 -l -v"
+
+cibuildwheel --platform macos
+
+popd
diff --git a/scripts/macos/install_antlr4_cpp_runtime.sh b/scripts/macos/install_antlr4_cpp_runtime.sh
@@ -12,6 +12,9 @@ if [ -d "${PREFIX}/include/antlr4-runtime" ]; then
   exit 0
 fi
 
+# use arm64 or x86_64.
+ARCH=x86_64
+
 # if macOS is 10.x -> use this as minimum
 MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13"
 
@@ -35,6 +38,7 @@ git clone https://github.com/antlr/antlr4.git \
 && cd antlr4 && cd runtime &&  git fetch --all --tags \
 && git checkout tags/4.13.1 -b 4.13.1 && cd Cpp/ \
 && sed -i '' "s/cmake ./cmake . ${MINIMUM_TARGET}/g" deploy-macos.sh \
+&& sed -i '' "s/CMAKE_OSX_ARCHITECTURES=\"arm64; x86_64\"/CMAKE_OSX_ARCHITECTURES=\"${ARCH}\"/g" deploy-macos.sh \
 && cat deploy-macos.sh \
 && ./deploy-macos.sh \
 && unzip -l antlr4-cpp-runtime-macos.zip && unzip antlr4-cpp-runtime-macos.zip \

diff --git a/scripts/set_version.py b/scripts/set_version.py
@@ -15,9 +15,8 @@ def LooseVersion(v):
     parts = v.split('.')
     return parts
 
-
-# to create a testpypi version use X.Y.devN
-version = '0.3.6'
+# change here and run script within its directory to update versions across the board
+version = '0.3.7'
 
 # https://pypi.org/simple/tuplex/
 # or https://test.pypi.org/simple/tuplex/

diff --git a/setup.py b/setup.py
@@ -66,12 +66,13 @@ def in_google_colab():
 'nbformat<7.0',
 'prompt_toolkit>=2.0.7',
 'pytest>=5.3.2'
+'lxml[html_clean]'
 ]
 
 # Also requires to install MongoDB
 webui_dependencies = [
-    'Flask>=2.0.2,<2.2.0',
-    'Werkzeug<2.2.0',
+    'Flask>=3.0',
+    'Werkzeug',
     'gunicorn',
     'eventlet==0.30.0', # newer versions of eventlet have a bug under MacOS
     'flask-socketio',
@@ -117,8 +118,8 @@ def run_command(cmd, cwd, env):
     logging.debug('Building dependencies for Google Colab environment')
 
     install_dependencies = [
-        'urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1',
-        'folium==0.2.1'
+        'urllib3!=1.25.0,!=1.25.1,>=1.21.1',
+        'folium>=0.2.1'
         'requests',
         'attrs>=19.2.0',
         'dill>=0.2.7.1',
@@ -695,7 +696,7 @@ def tplx_package_data():
 # logic and declaration, and simpler if you include description/version in a file.
 setup(name="tuplex",
     python_requires='>=3.8.0',
-    version="0.3.6",
+    version="0.3.7",
     author="Leonhard Spiegelberg",
     author_email="tuplex@cs.brown.edu",
     description="Tuplex is a novel big data analytics framework incorporating a Python UDF compiler based on LLVM "

diff --git a/tuplex/cmake/FindLibDwarf.cmake b/tuplex/cmake/FindLibDwarf.cmake
@@ -32,8 +32,10 @@ find_path (DWARF_INCLUDE_DIR
         ${PkgConfig_LibDwarf_INCLUDE_DIRS}
         /usr/include
         /usr/include/libdwarf
+        /usr/include/libdwarf-0
         /usr/local/include
         /usr/local/include/libdwarf
+        /usr/local/include/libdwarf-0
         /opt/local/include
         /sw/include
         ENV CPATH) # PATH and INCLUDE will also work
@@ -126,4 +128,4 @@ endif()
 
 mark_as_advanced(LIBDW_INCLUDE_DIR DWARF_INCLUDE_DIR)
 mark_as_advanced(LIBDWARF_INCLUDE_DIR LIBDWARF_LIBRARIES)
-mark_as_advanced(LIBDWARF_CONST_NAME LIBDWARF_USE_INIT_C)
+mark_as_advanced(LIBDWARF_CONST_NAME LIBDWARF_USE_INIT_C)
diff --git a/tuplex/historyserver/requirements.txt b/tuplex/historyserver/requirements.txt
@@ -4,12 +4,12 @@ jedi
 astor==0.7.1
 pandas>=0.23.4
 cloudpickle
-Werkzeug==2.0.1
+Werkzeug>=2.0.1
 flask>=2.0.1
-flask_socketio==4.3.1
-python-socketio==4.6.0
-python-engineio==3.13.2
-flask_pymongo==2.2.0
+flask_socketio>=4.3.1
+python-socketio>=4.6.0
+python-engineio>=3.13.2
+flask_pymongo>=2.2.0
 iso8601==0.1.12
 dill==0.2.8.2
 greenlet>=0.4.15
-Original file line number
+Diff line change
@@ Expand Up / @@ -133,4 +133,4 @@ series = {SIGMOD/PODS '21} @@
     ```
     ---
-    (c) 2017-2023 Tuplex contributors
+    (c) 2017-2025 Tuplex contributors