tuplex · LeonhardFS · Dec 14, 2023 · Oct 15, 2023 · Oct 15, 2023 · Oct 15, 2023
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -38,13 +38,29 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Cache brew dependencies
+        if: runner.os == 'macOS'
+        uses: actions/cache@v3.3.2
+        with:
+          # Paths to cache:
+          # /usr/local/Homebrew - installation folder of Homebrew
+          # /usr/local/Cellar - installation folder of Homebrew formulae
+          # /usr/local/Frameworks, /usr/local/bin, /usr/local/opt - contain (links to) binaries installed by Homebrew formulae
+          path: |
+            /usr/local/Homebrew
+            /usr/local/Cellar
+            /usr/local/Frameworks
+            /usr/local/bin
+            /usr/local/opt
+          key: macos-11-build-cache-${{ hashFiles('./scripts/macos/brew_dependencies.sh') }}-v2
+
       # need to make this an intermediate step, i.e. build first the different lambda runners on Ubuntu...
       - name: Build Lambda runner (Linux only)
         if: runner.os != 'macOS'
         run: docker pull registry-1.docker.io/tuplex/ci:${{ matrix.python-version }} && export PYTHON3_VERSION=${{ matrix.python-version }}.0 && bash ./scripts/create_lambda_zip.sh && mkdir -p ./tuplex/python/tuplex/other && cp ./build-lambda/tplxlam.zip ./tuplex/python/tuplex/other
         shell: bash
 
-      - name: Build wheels
+      - name: Build wheel
         #if: runner.os != 'macOS'
         uses: pypa/cibuildwheel@fff9ec32ed25a9c576750c91e06b410ed0c15db7 # hash corresponds to v2.16.2
         env:
@@ -62,11 +78,15 @@ jobs:
 
           # requires macOS 10.13 at least to build because of C++17 features.
           CIBW_ENVIRONMENT_MACOS: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64}"
+          # run all python tests to make sure wheels are not defunct
+          CIBW_TEST_REQUIRES: "pytest pytest-timeout numpy nbformat jupyter"
+          # use 3min timeout per test and print top 25 slowest tests
+          CIBW_TEST_COMMAND: "cd {project} && pytest tuplex/python/tests -v --timeout 600 --durations 25"
 
       - name: reorganize files
         run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./.github/scripts/test_pypi.sh ./wheelhouse
 
       - uses: actions/upload-artifact@v3
         with:
           path: |
-            ./wheelhouse/*.whl
+            ./wheelhouse/*.whl
diff --git a/scripts/azure/Dockerfile b/scripts/azure/Dockerfile
@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+ARG GIT_HASH
+ARG GIT_REMOTE
+
+ENV PATH=/opt/bin:$PATH
+
+RUN mkdir -p /opt/sbin
+
+RUN echo "Building tuplex for commit ${GIT_HASH} / ${GIT_REMOTE}"
+
+RUN echo "Install MongoDB"
+ADD install_mongodb.sh /opt/sbin/install_mongodb.sh
+RUN bash /opt/sbin/install_mongodb.sh
+
+RUN echo "Install required packages"
+ADD install_azure_ci_reqs.sh /opt/sbin/install_azure_ci_reqs.sh
+RUN bash /opt/sbin/install_azure_ci_reqs.sh
+
+RUN echo 'Install python dependencies'
+RUN apt-get update -y && apt-get install -y python3 python3-pip python3-setuptools ninja-build && python3 -m pip install pytest pygments>=2.4.1 MarkupSafe==2.0 pexpect setuptools astor PyYAML jupyter nbformat pymongo eventlet==0.30.0 gunicorn pymongo && jupyter --version
+
+RUN echo "Clone tuplex and checkout"
+# Install git & add github to known hosts
+RUN apt-get install -y git && mkdir -p /root/.ssh/ && touch /root/.ssh/known_hosts && ssh-keyscan github.com >>/root/.ssh/known_hosts
+RUN mkdir -p /code && cd /code && git clone "${GIT_REMOTE}" tuplex && cd tuplex && git checkout "${GIT_HASH}"
+
+
+RUN echo 'Test local MongoDB'
+RUN cd /code/tuplex/python && python3 -m pip install -r requirements.txt && python3 mongodb_test.py && pkill mongod || true
+
+RUN echo "Build Tuplex"
+RUN cd /code/tuplex && TUPLEX_BUILD_ALL=1 CMAKE_ARGS="-DBUILD_WITH_ORC=ON -DLLVM_ROOT_DIR=/usr/lib/llvm-16 -DCMAKE_BUILD_TYPE=Release -DBUILD_FOR_CI=ON" python3 setup.py install --user
+
+RUN echo "C++ tests"
+RUN cd /code/tuplex && cd build/temp.linux-x86_64-3.10 && ctest --timeout 180 --output-on-failure --repeat until-pass:3 -j 2
+
+RUN echo "Python tests"
+RUN cd /code/tuplex && cd build/temp.linux-x86_64-3.10/dist/python && python3.10 -m pytest -x --full-trace -l --log-cli-level=DEBUG --capture=tee-sys
diff --git a/scripts/azure/simulate_ci_locally.sh b/scripts/azure/simulate_ci_locally.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+echo "Retrieving last commit has"
+GIT_HASH=`git rev-parse --short HEAD`
+
+GIT_REMOTE=`git remote -v | cut -f2 | head -n 1 | cut -f1 -d ' '`
+GIT_REMOTE="https://github.com/$(git remote get-url origin | sed 's/https:\/\/github.com\///' | sed 's/git@github.com://')"
+echo "Building docker image for hash ${GIT_HASH}, remote ${GIT_REMOTE}"
+docker build --no-cache -t tuplex/azure --build-arg GIT_HASH=${GIT_HASH} --build-arg GIT_REMOTE=${GIT_REMOTE} .
+
diff --git a/scripts/build_linux_wheels.sh b/scripts/build_linux_wheels.sh
@@ -39,7 +39,7 @@ fi
 export CIBW_ENVIRONMENT="TUPLEX_LAMBDA_ZIP='./tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
 
 # Use the following line to build only python3.7-3.9 wheel
-export CIBW_BUILD="cp3{7,8,9}-*"
+export CIBW_BUILD="cp3{8,9,10,11}-*"
 export CIBW_ARCHS_LINUX="x86_64"
 
 # do not build musllinux yet
@@ -50,7 +50,10 @@ export CIBW_SKIP="*-musllinux_*"
 #export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
 
 export CIBW_BUILD_VERBOSITY=3
-export CIBW_PROJECT_REQUIRES_PYTHON=">=3.7"
+export CIBW_PROJECT_REQUIRES_PYTHON=">=3.8"
+
+# uncomment to increase verbosity of cibuildwheel
+# export CIBW_BUILD_VERBOSITY=3
 
 cibuildwheel --platform linux .
 

diff --git a/scripts/build_linux_wheels_with_test.sh b/scripts/build_linux_wheels_with_test.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# (c) 2023 Tuplex team
+# this script invokes the cibuildwheel process with necessary env variables to build the wheel for linux/docker
+# builds wheels for python 3.7 - 3.9
+
+# check from where script is invoked
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+echo "Executing buildwheel script located in $CWD"
+pushd $CWD > /dev/null
+cd ..
+
+# delete dir if exists
+rm -rf wheelhouse
+# delete in tree build files
+rm -rf tuplex/python/tuplex/libexec/tuplex*.so
+
+
+# CIBUILDWHEEL CONFIGURATION
+export CIBUILDWHEEL=1
+export TUPLEX_BUILD_ALL=0
+export CIBW_ARCHS_LINUX=x86_64
+export CIBW_MANYLINUX_X86_64_IMAGE='registry-1.docker.io/tuplex/ci:latest'
+
+# uncomment to prefer local image when building locally
+# export CIBW_MANYLINUX_X86_64_IMAGE='tuplex/ci'
+
+# check whether lambda zip was build and stored in build-lambda
+TUPLEX_LAMBDA_ZIP=${TUPLEX_LAMBDA_ZIP:-build-lambda/tplxlam.zip}
+
+echo "work dir is: $(pwd)"
+if [[ -f "${TUPLEX_LAMBDA_ZIP}" ]]; then
+	echo "Found lambda runner ${TUPLEX_LAMBDA_ZIP}, adding to package"
+	mkdir -p tuplex/other
+	cp ${TUPLEX_LAMBDA_ZIP} tuplex/other/tplxlam.zip
+fi
+
+# add to environment, e.g. TUPLEX_BUILD_TYPE=tsan to force a tsan build. Release is the default mode
+export CIBW_ENVIRONMENT="TUPLEX_LAMBDA_ZIP='./tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
+
+# Use the following line to build only python3.7-3.9 wheel
+export CIBW_BUILD="cp3{8,9,10,11}-*"
+export CIBW_ARCHS_LINUX="x86_64"
+
+# do not build musllinux yet
+export CIBW_SKIP="*-musllinux_*"
+
+# to test the others from 3.7-3.9, use these two lines:
+#export CIBW_BUILD="cp3{7,8,9}-*"
+#export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
+
+export CIBW_BUILD_VERBOSITY=3
+export CIBW_PROJECT_REQUIRES_PYTHON=">=3.8"
+
+# uncomment to increase verbosity of cibuildwheel
+# export CIBW_BUILD_VERBOSITY=3
+
+export CIBW_TEST_REQUIRES="pytest pytest-timeout numpy nbformat jupyter"
+export CIBW_TEST_COMMAND="cd {project} && pytest tuplex/python/tests --timeout_method thread --timeout 300 -l -v"
+
+cibuildwheel --platform linux .
+
+popd > /dev/null
+
+echo "Done!"
diff --git a/scripts/build_macos_wheels.sh b/scripts/build_macos_wheels.sh
@@ -59,9 +59,6 @@ fi
 
 echo "-- Building wheels for ${CIBW_BUILD}"
 
-# if macOS is 10.x -> use this as minimum
-MINIMUM_TARGET="10.13"
-
 MACOS_VERSION=$(sw_vers -productVersion)
 echo "-- Processing on MacOS ${MACOS_VERSION}"
 function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; }
@@ -82,13 +79,13 @@ cd ..
 
 # Note: protobuf 3.20 - 3.21.2 is broken for MacOS, do not use those versions
 export CIBW_BEFORE_BUILD_MACOS="brew install protobuf coreutils zstd zlib libmagic llvm@16 aws-sdk-cpp pcre2 antlr4-cpp-runtime googletest gflags yaml-cpp celero wget boost ninja snappy"
-export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' "
+export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DCMAKE_BUILD_TYPE=Release' CMAKE_BUILD_TYPE=Release"
 
 export CIBW_BUILD="${CIBW_BUILD}"
 export CIBW_PROJECT_REQUIRES_PYTHON=">=3.8"
 
 # uncomment to increase verbosity of cibuildwheel
-# export CIBW_BUILD_VERBOSITY=3
+export CIBW_BUILD_VERBOSITY=3
 
 cibuildwheel --platform macos
 

diff --git a/scripts/build_macos_wheels_with_test.sh b/scripts/build_macos_wheels_with_test.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# (c) 2017-2023 Tuplex team
+# builds x86_64 (and arm64) wheels
+
+# add -x option for verbose output
+set -euo pipefail
+
+function fail {
+    printf '%s\n' "$1" >&2
+    exit "${2-1}"
+}
+
+function detect_instruction_set() {
+  arch="$(uname -m)"  # -i is only linux, -m is linux and apple
+  if [[ "$arch" = x86_64* ]]; then
+    if [[ "$(uname -a)" = *ARM64* ]]; then
+      echo 'arm64'
+    else
+      echo 'x86_64'
+    fi
+  elif [[ "$arch" = i*86 ]]; then
+    echo 'x86_32'
+  elif [[ "$arch" = arm* ]]; then
+    echo $arch
+  elif test "$arch" = aarch64; then
+    echo 'arm64'
+  else
+    exit 1
+	fi
+}
+
+# check from where script is invoked
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+echo " || Tuplex macOS wheel builder || "
+echo "-- Executing buildwheel script located in $CWD"
+
+# check platform is darwin
+if [ ! "$(uname -s)" = "Darwin" ]; then
+  fail "Error: Need to run script under macOS"
+fi
+
+# check which tags are supported
+arch=$(detect_instruction_set)
+echo "-- Detected arch ${arch}"
+
+# try to extract version of compiler first via command-line tools or xcode
+# either needs to be installed.
+xcode_version_str=$(pkgutil --pkg-info=com.apple.pkg.CLTools_Executables 2>/dev/null | grep version || pkgutil --pkg-info=com.apple.pkg.Xcode | grep version)
+echo "-- Detected Xcode ${xcode_version_str}"
+
+# if no param is given, use defaults to build all
+if [ "${arch}" = "arm64" ]; then
+  # build Python 3.9 - 3.11
+  CIBW_BUILD=${CIBW_BUILD-"cp3{9,10,11}-macosx_arm64"}
+else
+  # build Python 3.8 - 3.11
+  CIBW_BUILD=${CIBW_BUILD-"cp3{8,9,10,11}-macosx_x86_64"}
+fi
+
+echo "-- Building wheels for ${CIBW_BUILD}"
+
+MACOS_VERSION=$(sw_vers -productVersion)
+echo "-- Processing on MacOS ${MACOS_VERSION}"
+function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; }
+
+MACOS_VERSION_MAJOR=`echo $MACOS_VERSION | cut -d . -f1`
+
+if [ "$MACOS_VERSION_MAJOR" -ge 11 ]; then
+    echo "-- Newer MacOS detected (>=11.0), using more recent base target."
+    echo "-- Using minimum target ${MACOS_VERSION_MAJOR}.0"
+    MINIMUM_TARGET="${MACOS_VERSION_MAJOR}.0"
+else
+    # keep as is
+    echo "-- Defaulting build to use as minimum target ${MINIMUM_TARGET}"
+fi
+
+pushd $CWD > /dev/null
+cd ..
+
+# fix because of Python
+MINIMUM_TARGET=11.0
+
+# Note: 3.8 only supports tags up to 10.16
+MINIMUM_TARGET=10.13
+
+# Note: protobuf 3.20 - 3.21.2 is broken for MacOS, do not use those versions
+export CIBW_BEFORE_BUILD_MACOS="brew install protobuf coreutils zstd zlib libmagic llvm@16 aws-sdk-cpp pcre2 antlr4-cpp-runtime googletest gflags yaml-cpp celero wget boost ninja snappy libdwarf libelf"
+
+
+# Note: orc build breaks wheel right now...
+export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' TUPLEX_BUILD_TYPE=RelWithDebInfo"
+#export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' TUPLEX_BUILD_TYPE=Debug"
+
+export CIBW_BUILD="${CIBW_BUILD}"
+export CIBW_PROJECT_REQUIRES_PYTHON=">=3.8"
+
+# uncomment to increase verbosity of cibuildwheel
+export CIBW_BUILD_VERBOSITY=3
+
+# uncomment and set to specific identifier
+#export CIBW_BUILD="cp39-macosx_x86_64"
+
+export CIBW_TEST_REQUIRES="pytest pytest-timeout numpy nbformat jupyter"
+export CIBW_TEST_COMMAND="cd {project} && pytest tuplex/python/tests --timeout_method thread --timeout 300 -l -v"
+
+cibuildwheel --platform macos
+
+popd
diff --git a/scripts/macos/brew_dependencies.sh b/scripts/macos/brew_dependencies.sh
@@ -2,4 +2,9 @@
 # This script installs all required dependencies via brew
 # for instructions on how to install brew, visit https://brew.sh/
 
-brew install openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@16 pcre2 gflags yaml-cpp celero wget boost googletest
+
+# brew doesn't provide llvm@16 bottle anymore for big sur, but python3.8 only works with big sur tags. use llvm@15 instead
+brew install openjdk@11 cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf
+
+# link (when e.g. used from restoring cache)
+brew link --overwrite cmake coreutils protobuf zstd zlib libmagic llvm@15 pcre2 gflags yaml-cpp celero wget boost googletest libdwarf libelf abseil
diff --git a/scripts/macos/install_antlr4_cpp_runtime.sh b/scripts/macos/install_antlr4_cpp_runtime.sh
@@ -7,6 +7,11 @@ PREFIX=/usr/local
 # if antlr4 exists already, skip
 [ -d "antlr4" ] && exit 0
 
+if [ -d "${PREFIX}/include/antlr4-runtime" ]; then
+  echo "skip antlr4 runtime install, directory already exists"
+  exit 0
+fi
+
 # if macOS is 10.x -> use this as minimum
 MINIMUM_TARGET="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13"
 

diff --git a/scripts/macos/install_aws-sdk-cpp.sh b/scripts/macos/install_aws-sdk-cpp.sh
@@ -1,5 +1,11 @@
 #!/usr/bin/env bash
 
+# check if dir exists (i.e. restored from cache, then skip)
+if [ -d "/usr/local/include/aws" ]; then
+  echo ">> Skip aws-sdk-cpp compile from source, already exists"
+  exit 0
+fi
+
 echo ">> installing AWS SDK from source"
 CPU_CORES=$(sysctl -n hw.physicalcpu)
 

diff --git a/setup.py b/setup.py
@@ -374,6 +374,13 @@ def find_pkg_path(lines):
                 except:
                     logging.error('Could not detect macos version, defaulting to macos 10.13 as build target')
 
+            # special case: Python3.8 earlier, widely deployed versions only support suffxi 10_13 or up to 10.16 so use that as target
+            if sys.version_info.major == 3 and sys.version_info.minor == 8:
+                if macos_build_target != "10.13" or macos_build_target != "10.16":
+                    logging.warning(f"Building Tuplex with Python {sys.version_info}, however earlier versions of Python 3.8 can only comprehend tag 10_13, using therefore deployment target 10.13")
+                    macos_build_target = "10.13"
+
+            logging.info(f"Building with macOS platform tag {macos_build_target}")
             # get mac OS version
             cmake_args.append('-DCMAKE_OSX_DEPLOYMENT_TARGET={}'.format(macos_build_target))