tuplex · LeonhardFS · Oct 14, 2023 · Jun 6, 2022 · Jun 6, 2022 · Jun 6, 2022
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -4,108 +4,69 @@ on: [push, pull_request, workflow_dispatch]
 
 jobs:
   build_wheels:
-    name: Build wheels on ${{ matrix.os }}
+    name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-10.15, macos-11, macos-12 ]
-
+        os: [ ubuntu-20.04, macos-11 ]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        include:
+          - os: ubuntu-20.04
+            python-version: "3.8"
+            cibw-build: "cp38-manylinux_x86_64"
+          - os: ubuntu-20.04
+            python-version: "3.9"
+            cibw-build: "cp39-manylinux_x86_64"
+          - os: ubuntu-20.04
+            python-version: "3.10"
+            cibw-build: "cp310-manylinux_x86_64"
+          - os: ubuntu-20.04
+            python-version: "3.11"
+            cibw-build: "cp311-manylinux_x86_64"
+          - os: macos-11
+            python-version: "3.8"
+            cibw-build: "cp38-macosx_x86_64"
+          - os: macos-11
+            python-version: "3.9"
+            cibw-build: "cp39-macosx_x86_64"
+          - os: macos-11
+            python-version: "3.10"
+            cibw-build: "cp310-macosx_x86_64"
+          - os: macos-11
+            python-version: "3.11"
+            cibw-build: "cp311-macosx_x86_64"
     steps:
-      - uses: actions/checkout@v2
-
-      # this will create a dummy dev version based on the current time to avoid conflicts on test.pypi.org
-      - name: Create dev version
-        if: github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true
-        run: cd ./scripts && pip3 install requests && python3 set_version.py --dev
-        shell: bash
+      - uses: actions/checkout@v3
 
       # need to make this an intermediate step, i.e. build first the different lambda runners on Ubuntu...
       - name: Build Lambda runner (Linux only)
         if: runner.os != 'macOS'
-        run: docker pull registry-1.docker.io/tuplex/ci:latest && bash ./scripts/create_lambda_zip.sh && mkdir -p ./tuplex/python/tuplex/other && cp ./build-lambda/tplxlam.zip ./tuplex/python/tuplex/other
+        run: docker pull registry-1.docker.io/tuplex/ci:${{ matrix.python-version }} && export PYTHON3_VERSION=${{ matrix.python-version }}.0 && bash ./scripts/create_lambda_zip.sh && mkdir -p ./tuplex/python/tuplex/other && cp ./build-lambda/tplxlam.zip ./tuplex/python/tuplex/other
         shell: bash
 
       - name: Build wheels
         #if: runner.os != 'macOS'
-        uses: pypa/cibuildwheel@v1.11.1.post1
+        uses: pypa/cibuildwheel@fff9ec32ed25a9c576750c91e06b410ed0c15db7 # hash corresponds to v2.16.2
         env:
           # configure cibuildwheel to build native archs ('auto'), and some
           # emulated ones
           CIBW_ARCHS_LINUX: native
-          CIBW_MANYLINUX_X86_64_IMAGE: 'registry-1.docker.io/tuplex/ci:latest'
-          # build python 3.7, 3.8, 3.9 on linux.
-          # only build python 3.9 on macos
-
-          # production version:
-          # no musllinux yet, no 3.10 support yet.
-          CIBW_BUILD: "cp3{7,8,9}-*"
-          CIBW_SKIP: "cp3{5,6}-macosx* pp* *-musllinux_*"
+          CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}"
+          CIBW_BUILD: ${{ matrix.cibw-build }}
 
-          # do not use build, b.c. it will fail on subsequent. setup once.
-          CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7"
+          # macOS dependencies separate, for linux use docker tuplex/ci:3.x images.
+          CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile
 
-          # set this environment variable to include the Lambda zip from the previous build step
-          # do not include Lambda runner in macos wheel yet. Do in future release.
+          # bundle aws runner with linux wheel, remove environment variable TUPLEX_LAMBDA_ZIP to remove runner.
+          CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
 
-          # use following line to bundle Lambda runner:
-          # CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
-          # yet, because PyPi limit hasn't been increased yet, do not bundle runner.
-          CIBW_ENVIRONMENT_LINUX: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
-          # requires 10.13 at least for macos!
-          CIBW_ENVIRONMENT_MACOS: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON'"
+          # requires macOS 10.13 at least to build because of C++17 features.
+          CIBW_ENVIRONMENT_MACOS: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64}"
 
       - name: reorganize files
         run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./.github/scripts/test_pypi.sh ./wheelhouse
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           path: |
-            ./wheelhouse/*.whl
-            ./wheelhouse/*.version
-            ./wheelhouse/test_pypi.sh
-
-  # cf. https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml
-  # potentially also create a sdist.
-  upload_pypi:
-    needs: [ build_wheels ]
-    runs-on: ubuntu-20.04
-    # remove repository url to publish to default pypi.
-    # upload to PyPI on every tag starting with 'v' ONLY on official tuplex repo.
-    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && github.repository == 'tuplex/tuplex'
-    # alternatively, to publish when a GitHub Release is created, use the following rule:
-    # if: github.event_name == 'release' && github.event.action == 'published'
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: artifact
-          path: dist
-
-      - name: remove test files
-        run: rm dist/*.version && rm dist/*.sh
-
-      - uses: pypa/gh-action-pypi-publish@v1.4.2
-        with:
-          user: ${{ secrets.pypi_user }}
-          password: ${{ secrets.pypi_password }}
-
-  upload_testpypi:
-    needs: [ build_wheels ]
-    runs-on: ubuntu-20.04
-    # inverse condition, always create test release, any repo with passwords can work with this.
-    # note, pull requests are not sharing secrets...
-    if: github.event_name != 'pull_request' && (github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true)
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: artifact
-          path: dist
-
-      - name: reorganize
-        run: mkdir -p scripts && mv dist/*.sh ./scripts/ && mv dist/*.version ./scripts/ && chmod +x ./scripts/test_pypi.sh
-
-      - uses: pypa/gh-action-pypi-publish@v1.4.2
-        with:
-          user: ${{ secrets.pypi_user }}
-          password: ${{ secrets.pypi_password }}
-          repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
+            ./wheelhouse/*.whl
diff --git a/benchmarks/zillow/Z1/baseline/fmt/include/fmt/printf.h b/benchmarks/zillow/Z1/baseline/fmt/include/fmt/printf.h
@@ -100,7 +100,7 @@ template <typename T, typename Context> class arg_converter {
     } else {
       if (is_signed) {
         // glibc's printf doesn't sign extend arguments of smaller types:
-        //   std::printf("%lld", -42);  // prints "4294967254"
+        //   std::printf(""%" PRId64", -42);  // prints "4294967254"
         // but we don't have to do the same because it's a UB.
         arg_ = internal::make_arg<Context>(static_cast<long long>(value));
       } else {

diff --git a/benchmarks/zillow/Z2/baseline/fmt/include/fmt/printf.h b/benchmarks/zillow/Z2/baseline/fmt/include/fmt/printf.h
@@ -100,7 +100,7 @@ template <typename T, typename Context> class arg_converter {
     } else {
       if (is_signed) {
         // glibc's printf doesn't sign extend arguments of smaller types:
-        //   std::printf("%lld", -42);  // prints "4294967254"
+        //   std::printf(""%" PRId64", -42);  // prints "4294967254"
         // but we don't have to do the same because it's a UB.
         arg_ = internal::make_arg<Context>(static_cast<long long>(value));
       } else {

diff --git a/doc/source/gettinginvolved.rst b/doc/source/gettinginvolved.rst
@@ -129,7 +129,7 @@ Go to ``BlockGeneratorVisitor.cc`` and edit the
 .. code-block:: c++
 
     llvm::Value *
-    BlockGeneratorVisitor::compareInst(llvm::IRBuilder<>& builder, llvm::Value *L, const python::Type &leftType, const TokenType &tt,
+    BlockGeneratorVisitor::compareInst(codegen::IRBuilder&builder, llvm::Value *L, const python::Type &leftType, const TokenType &tt,
                                            llvm::Value *R, const python::Type &rightType)
 
 function to add support for the ``is`` tokens you added. You can use ``error(...)`` to fail on bad comparison expressions involving ``is`` as discussed above.

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ requires = [
     "cloudpickle",
     "numpy",
     "ninja; sys_platform != 'win32'",
-    "cmake>=3.19,<3.22",
+    "cmake>=3.25",
     "delocate; sys.platform == 'darwin'",
     "auditwheel; sys.platform == 'linux'",
     "requests"

diff --git a/scripts/create_lambda_zip.sh b/scripts/create_lambda_zip.sh
@@ -1,27 +1,40 @@
 #!/usr/bin/env bash
-# (c) 2021 Tuplex team
+# (c) 2017-2023 Tuplex team
+# this script creates a deployable AWS Lambda zip package using docker
+
+set -euxo pipefail
 
-# exact python versions AWS uses:
-# Python 3.9 runtime --> Python 3.9.8
-# Python 3.8 runtime --> Python 3.8.11
-PYTHON3_VERSION=3.9.8
+echo ">>> Building Lambda runner"
+DEFAULT_PYTHON3_VERSION=$(python3 --version | cut -d ' ' -f2)
+echo "-- detected system python version is ${DEFAULT_PYTHON3_VERSION}"
+echo "-- to specify different Python3 version, set environment variable PYTHON3_VERSION, e.g. export PYTHON3_VERSION=3.9"
+
+PYTHON3_VERSION="${PYTHON3_VERSION:-$DEFAULT_PYTHON3_VERSION}"
 PYTHON3_MAJMIN=${PYTHON3_VERSION%.*}
+DOCKER_IMAGE=tuplex/ci:${PYTHON3_MAJMIN}
 
+# check which Python version is installed in /opt/lambda-python/bin/python3
+DOCKER_PYTHON3_VERSION=$(docker run -e LD_LIBRARY_PATH=/opt/lambda-python/lib $DOCKER_IMAGE /opt/lambda-python/bin/python3 --version | cut -d ' ' -f2)
 
-# this script creates a deployable AWS Lambda zip package using docker
+echo "-- detected docker Python3 version ${DOCKER_PYTHON3_VERSION}"
+
+## make sure maj.min version matches
+if [ "${DOCKER_PYTHON3_VERSION%.*}" -ne "${PYTHON3_VERSION%.*}" ]; then
+  echo "ERROR: Python maj.min versions do not match, Docker has ${DOCKER_PYTHON3_VERSION%.*} but desired version is ${PYTHON3_VERSION%.*}."
+  exit 1
+fi
 
 # check from where script is invoked
 CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 
-echo "Executing buildwheel script located in $CWD"
+echo "-- Executing buildwheel script located in $CWD"
 pushd $CWD > /dev/null
 cd .. # go to root of repo
 
 # start code here...
 
 LOCAL_BUILD_FOLDER=build-lambda
 SRC_FOLDER=tuplex
-DOCKER_IMAGE=tuplex/ci
 
 # convert to absolute paths
 get_abs_filename() {
@@ -31,9 +44,10 @@ get_abs_filename() {
 
 LOCAL_BUILD_FOLDER=$(get_abs_filename $LOCAL_BUILD_FOLDER)
 SRC_FOLDER=$(get_abs_filename $SRC_FOLDER)
-echo "Tuplex source: $SRC_FOLDER"
-echo "Building lambda in: $LOCAL_BUILD_FOLDER"
-
+LLVM_ROOT_PATH=/opt/llvm-16.0.6
+echo "-- Tuplex source: $SRC_FOLDER"
+echo "-- Building lambda in: $LOCAL_BUILD_FOLDER"
+echo "-- LLVM folder: ${LLVM_ROOT_PATH}"
 mkdir -p $LOCAL_BUILD_FOLDER
 
 echo "starting docker (this might take a while...)"
@@ -49,12 +63,12 @@ echo "starting docker (this might take a while...)"
 # only release works, b.c. of size restriction
 BUILD_TYPE=Release
 
-docker run --name lambda --rm -v $SRC_FOLDER:/code/tuplex -v $LOCAL_BUILD_FOLDER:/build tuplex/ci bash -c "export LD_LIBRARY_PATH=/opt/lambda-python/lib:\$LD_LIBRARY_PATH && /opt/lambda-python/bin/python${PYTHON3_MAJMIN} -m pip install cloudpickle numpy && cd /build && cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_FOR_LAMBDA=ON -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DPYTHON3_EXECUTABLE=/opt/lambda-python/bin/python${PYTHON3_MAJMIN} -DBOOST_ROOT=/opt/boost/python${PYTHON3_MAJMIN}/ -GNinja /code/tuplex && cmake --build . --target tplxlam && python${PYTHON3_MAJMIN} /code/tuplex/python/zip_cc_runtime.py --input /build/dist/bin/tplxlam --runtime /build/dist/bin/tuplex_runtime.so --python /opt/lambda-python/bin/python${PYTHON3_MAJMIN} --output /build/tplxlam.zip"
+docker run --name lambda --rm -v $SRC_FOLDER:/code/tuplex -v $LOCAL_BUILD_FOLDER:/build ${DOCKER_IMAGE} bash -c "export LD_LIBRARY_PATH=/opt/lambda-python/lib:/opt/lib:/opt/lib64:\$LD_LIBRARY_PATH && /opt/lambda-python/bin/python${PYTHON3_MAJMIN} -m pip install cloudpickle numpy && cd /build && cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DLLVM_ROOT_DIR=${LLVM_ROOT_PATH} -DBUILD_FOR_LAMBDA=ON -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DPYTHON3_EXECUTABLE=/opt/lambda-python/bin/python${PYTHON3_MAJMIN} -DBOOST_ROOT=/opt/boost/python${PYTHON3_MAJMIN}/ -GNinja /code/tuplex && cmake --build . --target runtime && cmake --build . --target tplxlam && python${PYTHON3_MAJMIN} /code/tuplex/python/zip_cc_runtime.py --input /build/dist/bin/tplxlam --runtime /build/dist/bin/tuplex_runtime.so --python /opt/lambda-python/bin/python${PYTHON3_MAJMIN} --output /build/tplxlam.zip"
 DOCKER_EXIT_CODE=$?
 if [ "${DOCKER_EXIT_CODE}" -eq "0" ]; then
-   echo "docker command run, zipped Lambda file can be found in: ${LOCAL_BUILD_FOLDER}/tplxlam.zip"
+   echo "-- docker command run, zipped Lambda file can be found in: ${LOCAL_BUILD_FOLDER}/tplxlam.zip"
 else
-   echo "build failed"
+   echo "ERROR: build failed"
    popd > /dev/null
    exit 1
 fi

diff --git a/scripts/docker/ci/Dockerfile b/scripts/docker/ci/Dockerfile
@@ -5,58 +5,54 @@
 # (c) 2017-2022 Tuplex team
 FROM quay.io/pypa/manylinux2014_x86_64
 
-
 MAINTAINER "Tuplex project@Brown"
 
+# select core versions to use when building CI image here
+ARG PYTHON_VERSION=3.11.5
+ARG CMAKE_VERSION=3.27.5
+ARG BOOST_VERSION=1.79.0
+
+# image is centos based, so use yum as package manager
+# --> install_llvm uses most recent 16 release.
+
+# set link to desired python version, note that ${PYTHON_VERSION%.*} gives for e.g. 3.x.y -> 3.x
+RUN ln -sf /usr/local/bin/python${PYTHON_VERSION%.*} /usr/local/bin/python3 && /usr/local/bin/python3 --version
+
+RUN yum update -y && yum install -y dnf && dnf install -y git autoconf zip wget
+
 # add script files from local dir
 RUN mkdir -p /opt/sbin
+
+ADD install_cmake.sh /opt/sbin/install_cmake.sh
+RUN bash /opt/sbin/install_cmake.sh amd64 linux ${CMAKE_VERSION} /usr/local
+
 ADD install_boost.sh /opt/sbin/install_boost.sh
-ADD install_tuplex_reqs.sh /opt/sbin/install_tuplex_reqs.sh
-ADD install_llvm.sh /opt/sbin/install_llvm.sh
 
-# cmake not required to be installed, because recent image has cmake 3.20
-# it uses gcc 9.3.1
+ADD install_llvm.sh /opt/sbin/install_llvm.sh
 
 # CentOS/RHEL does not use OpenSSL for the system curl, however AWSSDK must use OpenSSL backed curl.
 ADD install_curl.sh /opt/sbin/install_curl.sh
 
-# image is centos based, so use yum as package manager
-# --> install_llvm uses most recent 9 release.
-
-RUN yum update -y
-RUN yum install -y wget
-
-# llvm-9 on yum repo might be broken, use manually built llvm
-RUN bash /opt/sbin/install_llvm.sh
 
 # install curl now
 RUN bash /opt/sbin/install_curl.sh
 
-# install boost-python for 3.7, 3.8, 3.9, 3.10
-RUN bash /opt/sbin/install_boost.sh /opt/python/cp37-cp37m/bin/python3.7 /opt/boost/python3.7
-RUN bash /opt/sbin/install_boost.sh /opt/python/cp38-cp38//bin/python3.8 /opt/boost/python3.8
-RUN bash /opt/sbin/install_boost.sh /opt/python/cp39-cp39/bin/python3.9 /opt/boost/python3.9
-RUN bash /opt/sbin/install_boost.sh /opt/python/cp310-cp310/bin/python3.10 /opt/boost/python3.10
-
+# install boost-python for given version
+RUN bash /opt/sbin/install_boost.sh /usr/local/bin/python3 /opt/boost/python${PYTHON_VERSION%.*} ${BOOST_VERSION}
 
 # Update pip versions
-RUN python3.7 -m pip install --upgrade pip setuptools wheel
-RUN python3.8 -m pip install --upgrade pip setuptools wheel
-RUN python3.9 -m pip install --upgrade pip setuptools wheel
-RUN python3.10 -m pip install --upgrade pip setuptools wheel
-
-# matrix?
-RUN python3.7 -m pip install 'cloudpickle<2.0' cython numpy pandas
-RUN python3.8 -m pip install 'cloudpickle<2.0' cython numpy pandas
-RUN python3.9 -m pip install 'cloudpickle<2.0' numpy pandas
-RUN python3.10 -m pip install 'cloudpickle>2.0' numpy pandas
-
-# tuplex requirements
+RUN /usr/local/bin/python3 -m pip install --upgrade pip setuptools wheel
+
+# Tuplex requirements, installs python version specific requirements as well
+ADD install_tuplex_reqs.sh /opt/sbin/install_tuplex_reqs.sh
 RUN bash /opt/sbin/install_tuplex_reqs.sh
 
-# add lambda-specific Python 3.8 (full python install)
+# add lambda-specific Python (full python install), for correct shipping Python must be built with correct flags
 ADD install_lambda_python.sh /opt/sbin/install_lambda_python.sh
-RUN bash /opt/sbin/install_lambda_python.sh
+RUN bash /opt/sbin/install_lambda_python.sh ${PYTHON_VERSION}
+
+# install llvm, use here script which uses 16.0.6 to enable recent CPU architectures
+RUN bash /opt/sbin/install_llvm.sh
 
 ## MongoDB community edition for WebUI testing
 ADD mongodb-org-5.0.repo /etc/yum.repos.d/mongodb-org-5.0.repo
@@ -66,11 +62,10 @@ RUN yum update -y && yum install -y mongodb-org
 RUN bash /opt/sbin/install_curl.sh
 
 # remove all the tmp stuff
-RUN rm -rf /tmp/*
-
 # remove temp stuff based on https://www.getpagespeed.com/server-setup/clear-disk-space-centos
-RUN curl -Ls http://bit.ly/clean-centos-disk-space | bash
+RUN rm -rf /tmp/* && curl -Ls http://bit.ly/clean-centos-disk-space | bash && dnf clean all && rm -rf /var/cache/yum
 
-# install additional libraries for debugging
-RUN yum install -y centos-release-scl-rh devtoolset-11-libtsan-devel devtoolset-10-libtsan-devel
-RUN yum install -y devtoolset-11-libasan-devel devtoolset-10-libasan-devel
+# uncomment following to install optional packages for debugging
+# RUN yum install -y devtoolset-10-gdb
+# RUN yum install -y centos-release-scl-rh devtoolset-11-libtsan-devel devtoolset-10-libtsan-devel
+# RUN yum install -y devtoolset-11-libasan-devel devtoolset-10-libasan-devel