tuplex · LeonhardFS · Nov 18, 2021 · Nov 3, 2021 · Nov 3, 2021 · Nov 3, 2021
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -42,6 +42,10 @@ jobs:
         run: bash ./scripts/ci/setup-macos.sh
         shell: bash
 
+      - name: Build Lambda runner
+        run: docker pull registry-1.docker.io/tuplex/ci:latest && bash ./scripts/create_lambda_zip.sh && mkdir -p ./tuplex/python/tuplex/other && cp /home/runner/work/tuplex/tuplex/build-lambda/tplxlam.zip ./tuplex/python/tuplex/other
+        shell: bash
+
       - name: Build wheels
         uses: pypa/cibuildwheel@v1.11.1.post1
         env:
@@ -53,8 +57,9 @@ jobs:
           # only build python 3.9 on macos
 
           # production version:
+          # no musllinux yet, no 3.10 support yet.
           CIBW_BUILD: "cp3{7,8,9}-*"
-          CIBW_SKIP: "cp3{5,6,7,8}-macosx* pp*"
+          CIBW_SKIP: "cp3{5,6,7,8}-macosx* pp* *-musllinux_*"
 
           ## for debugging purposes (only linux build)
           #CIBW_BUILD: "cp38-*"
@@ -63,6 +68,9 @@ jobs:
           CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7"
           CIBW_BEFORE_BUILD_MACOS: ./scripts/ci/setup-macos.sh
 
+          # set this environment variable to include the Lambda zip from the previous build step
+          CIBW_ENVIRONMENT: TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip'
+
       - name: reorganize files
         run: touch ./scripts/dummy.version && cp ./scripts/*.version ./wheelhouse && cp ./scripts/test_pypi.sh ./wheelhouse
 

diff --git a/.gitignore b/.gitignore
@@ -91,3 +91,4 @@ python/tuplex.egg-info/
 tuplex.egg-info/
 
 wheelhouse/
+*.zip
diff --git a/scripts/build_wheel_linux.sh b/scripts/build_wheel_linux.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# (c) 2021 Tuplex team
 # this script invokes the cibuildwheel process with necessary env variables to build the wheel for linux/docker
 
 # check from where script is invoked
@@ -17,23 +18,28 @@ rm -rf tuplex/python/tuplex/libexec/tuplex*.so
 # CIBUILDWHEEL CONFIGURATION
 export CIBUILDWHEEL=1
 export TUPLEX_BUILD_ALL=0
-export CIBW_ARCHS_LINUX=native
+export CIBW_ARCHS_LINUX=x86_64
 export CIBW_MANYLINUX_X86_64_IMAGE='registry-1.docker.io/tuplex/ci:latest'
 
+export CIBW_ENVIRONMENT="TUPLEX_LAMBDA_ZIP='./tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib"
+
 # Use the following line to build only python3.9 wheel
 export CIBW_BUILD="cp39-*"
 
-
 # For Google Colab compatible wheel, use the following:
 export CIBW_BUILD="cp37-*"
 export CIBW_ARCHS_LINUX="x86_64"
 
+# do not build musllinux yet
+export CIBW_SKIP="*-musllinux_*"
+
 # to test the others from 3.7-3.9, use these two lines:
 #export CIBW_BUILD="cp3{7,8,9}-*"
 #export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
 
 export CIBW_BUILD_VERBOSITY=3
 export CIBW_PROJECT_REQUIRES_PYTHON=">=3.7"
+
 cibuildwheel --platform linux .
 
 popd > /dev/null

diff --git a/scripts/create_lambda_zip.sh b/scripts/create_lambda_zip.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# (c) 2021 Tuplex team
+
+# this script creates a deployable AWS Lambda zip package using docker
+
+# check from where script is invoked
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+echo "Executing buildwheel script located in $CWD"
+pushd $CWD > /dev/null
+cd .. # go to root of repo
+
+# start code here...
+
+LOCAL_BUILD_FOLDER=build-lambda
+SRC_FOLDER=tuplex
+DOCKER_IMAGE=tuplex/ci
+
+# convert to absolute paths
+get_abs_filename() {
+  # $1 : relative filename
+  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
+}
+
+LOCAL_BUILD_FOLDER=$(get_abs_filename $LOCAL_BUILD_FOLDER)
+SRC_FOLDER=$(get_abs_filename $SRC_FOLDER)
+echo "Tuplex source: $SRC_FOLDER"
+echo "Building lambda in: $LOCAL_BUILD_FOLDER"
+
+mkdir -p $LOCAL_BUILD_FOLDER
+
+echo "starting docker (this might take a while...)"
+
+# start docker & volume & create awslambda target with correct settings
+# the python version to use for lambda is in /opt/lambda-python/bin/python3.8
+# In order to kick-off the build within the docker, use the following two commands:
+# export LD_LIBRARY_PATH=/opt/lambda-python/lib:$LD_LIBRARY_PATH
+# cmake -DBUILD_FOR_LAMBDA=ON -DBUILD_WITH_AWS=ON -DBOOST_ROOT=/opt/boost/python3.8/ -GNinja -DPYTHON3_EXECUTABLE=/opt/lambda-python/bin/python3.8 /code/tuplex
+# --> The preload is necessary as a shared version of python is used.
+# just use tplxlam as target, then run custom python script to package contents up.
+
+docker run --name lambda --rm -v $SRC_FOLDER:/code/tuplex -v $LOCAL_BUILD_FOLDER:/build tuplex/ci bash -c "export LD_LIBRARY_PATH=/opt/lambda-python/lib:\$LD_LIBRARY_PATH && /opt/lambda-python/bin/python3.8 -m pip install cloudpickle numpy && cd /build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_FOR_LAMBDA=ON -DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON -DPYTHON3_EXECUTABLE=/opt/lambda-python/bin/python3.8 -DBOOST_ROOT=/opt/boost/python3.8/ -GNinja /code/tuplex && cmake --build . --target tplxlam && python3.8 /code/tuplex/python/zip_cc_runtime.py --input /build/dist/bin/tplxlam --runtime /build/dist/bin/tuplex_runtime.so --python /opt/lambda-python/bin/python3.8 --output /build/tplxlam.zip"
+echo "docker command run, zipped Lambda file can be found in: ${LOCAL_BUILD_FOLDER}/tplxlam.zip"
+
+# end code here...
+popd > /dev/null
diff --git a/scripts/docker/benchmark/install_tuplex_reqs.sh b/scripts/docker/benchmark/install_tuplex_reqs.sh
@@ -87,10 +87,10 @@ git clone https://github.com/awslabs/aws-lambda-cpp.git && \
 
 # pcre2
 cd /tmp &&
-  curl -O https://ftp.pcre.org/pub/pcre/pcre2-10.34.zip &&
-  unzip pcre2-10.34.zip &&
-  rm pcre2-10.34.zip &&
-  pushd pcre2-10.34 &&
+  curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.39/pcre2-10.39.zip &&
+  unzip pcre2-10.39.zip &&
+  rm pcre2-10.39.zip &&
+  pushd pcre2-10.39 &&
   ./configure --prefix=/opt --enable-jit=auto --disable-shared CFLAGS="-O2 -fPIC" && make -j 32 && make install
 popd
 

diff --git a/scripts/docker/ci/Dockerfile b/scripts/docker/ci/Dockerfile
@@ -14,6 +14,8 @@ ADD install_llvm9.sh /opt/sbin/install_llvm9.sh
 # cmake not required to be installed, because recent image has cmake 3.20
 # it uses gcc 9.3.1
 
+# CentOS/RHEL does not use OpenSSL for the system curl, however AWSSDK must use OpenSSL backed curl.
+ADD install_curl.sh /opt/sbin/install_curl.sh
 
 # image is centos based, so use yum as package manager
 # --> install_llvm9 uses most recent 9 release.
@@ -29,6 +31,9 @@ RUN yum install -y wget
 # llvm-9 on yum repo might be broken, use manually built llvm
 RUN bash /opt/sbin/install_llvm9.sh
 
+# install curl now
+RUN bash /opt/sbin/install_curl.sh
+
 # install boost-python for 3.7, 3.8, 3.9, 3.10
 RUN bash /opt/sbin/install_boost.sh /opt/python/cp37-cp37m/bin/python3.7 /opt/boost/python3.7
 RUN bash /opt/sbin/install_boost.sh /opt/python/cp38-cp38//bin/python3.8 /opt/boost/python3.8
@@ -52,11 +57,16 @@ RUN python3.10 -m pip install cloudpickle # numpy # pandas
 # tuplex requirements
 RUN bash /opt/sbin/install_tuplex_reqs.sh
 
+# add lambda-specific Python 3.8 (full python install)
+ADD install_lambda_python.sh /opt/sbin/install_lambda_python.sh
+RUN bash /opt/sbin/install_lambda_python.sh
 
 ## MongoDB community edition for WebUI testing
 ADD mongodb-org-5.0.repo /etc/yum.repos.d/mongodb-org-5.0.repo
 RUN yum update -y && yum install -y mongodb-org
 
+# replace curl again with recent version to be 100% everything worked properly.
+RUN bash /opt/sbin/install_curl.sh
 
 # remove all the tmp stuff
 RUN rm -rf /tmp/*

diff --git a/scripts/docker/ci/install_curl.sh b/scripts/docker/ci/install_curl.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# TODO: CentOS/RHEL does not support AWS SDK. It's triggering a bug in NSS which is the SSL lib used in CentOS/RHEL.
+# cf. https://github.com/aws/aws-sdk-cpp/issues/1491
+
+# Steps to solve:
+# 1.) install recent OpenSSL
+# 2.) build Curl against it
+# 3.) Compile AWS SDK with this curl version.
+# cf. https://geekflare.com/curl-installation/ for install guide
+
+# other mentions of the NSS problem:
+# https://curl.se/mail/lib-2016-08/0119.html
+# https://bugzilla.mozilla.org/show_bug.cgi?id=1297397
+
+# select here which curl version to use
+CURL_VERSION=7.80.0
+
+# Alternative could be to also just install via cmake, i.e. from repo https://github.com/curl/curl.
+
+# Main issue is, that on CentOS an old curl compiled with NSS is preinstalled.
+# ==> remove!
+# i.e., via rm -rf /usr/lib64/libcurl*
+
+NUM_PROCS=$(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) ))
+
+cd /tmp && yum update -y && yum install wget gcc openssl-devel -y && rm -rf /usr/lib64/libcurl* && \
+wget --no-check-certificate https://curl.se/download/curl-${CURL_VERSION}.tar.gz && tar xf curl-${CURL_VERSION}.tar.gz && \
+cd curl-${CURL_VERSION} && ./configure --with-openssl --without-nss --prefix=/usr/ --libdir=/usr/lib64 && make -j ${NUM_PROCS} && make install && ldconfig
diff --git a/scripts/docker/ci/install_lambda_python.sh b/scripts/docker/ci/install_lambda_python.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# to build the lambda executor need to embed python, therefore create full version below
+
+export CFLAGS=-I/usr/include/openssl
+
+# select python version, Lambda uses 3.8.11
+PYTHON3_VERSION=3.8.11
+PYTHON3_MAJMIN=${PYTHON3_VERSION%.*}
+
+# from https://bugs.python.org/issue36044
+# change tasks, because hangs at test_faulthandler...
+export PROFILE_TASK=-m test.regrtest --pgo \
+        test_collections \
+        test_dataclasses \
+        test_difflib \
+        test_embed \
+        test_float \
+        test_functools \
+        test_generators \
+        test_int \
+        test_itertools \
+        test_json \
+        test_logging \
+        test_long \
+        test_ordered_dict \
+        test_pickle \
+        test_pprint \
+        test_re \
+        test_set \
+        test_statistics \
+        test_struct \
+        test_tabnanny \
+        test_xml_etree
+
+set -ex && cd /tmp && wget https://www.python.org/ftp/python/${PYTHON3_VERSION}/Python-${PYTHON3_VERSION}.tgz && tar xf Python-${PYTHON3_VERSION}.tgz \
+    && cd Python-${PYTHON3_VERSION} && ./configure --with-lto --prefix=/opt/lambda-python --enable-optimizations --enable-shared \
+    && make -j $(( 1 * $( egrep '^processor[[:space:]]+:' /proc/cpuinfo | wc -l ) )) \
+    && make altinstall
+
+# install cloudpickle numpy for Lambda python
+export LD_LIBRARY_PATH=/opt/lambda-python/lib:$LD_LIBRARY_PATH
+/opt/lambda-python/bin/python${PYTHON3_MAJMIN} -m pip install cloudpickle numpy tqdm
diff --git a/scripts/docker/ci/install_tuplex_reqs.sh b/scripts/docker/ci/install_tuplex_reqs.sh
@@ -4,13 +4,13 @@
 # everything will be installed to /opt
 
 # Tuplex dependencies
-# compile dependencies yum stylke
+# compile dependencies yum style
 
 yum install -y libedit-devel libzip-devel \
-  pkgconfig openssl-devel libxml2-devel libcurl-devel zlib-devel  \
+  pkgconfig openssl-devel libxml2-devel zlib-devel  \
   uuid libuuid-devel libffi-devel graphviz-devel \
   gflags-devel ncurses-devel \
-  awscli java-1.8.0-openjdk-devel libyaml-devel file-devel
+  awscli java-1.8.0-openjdk-devel libyaml-devel file-devel ninja-build zip unzip
 
 # LLVM9 is broken on Ubuntu 20.04, hence manually install...
 
@@ -63,10 +63,13 @@ popd &&
   cd - || echo "ANTLR4 runtime failed"
 
 # AWS SDK
+# tag 1.9.142?
+# => note in 1.9.134/135 there has been a renaming of cJSON symbols -> this requires linking/renaming. cf. https://github.com/aws/aws-sdk-cpp/commit/2848c4571c94b03bc558378440f091f2017ef7d3
+# note for centos7 there's an issue with SSL. Either use aws sdk with -DBUILD_DEPS=ON/-DUSE_OPENSSL=OFF. or force -DUSE_OPENSSL=ON.
 cd /tmp &&
   git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git &&
-  cd aws-sdk-cpp && git checkout tags/1.9.39 && mkdir build && pushd build &&
-  cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=OFF -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=14 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=/opt .. &&
+  cd aws-sdk-cpp && git checkout tags/1.9.133 && mkdir build && pushd build &&
+  cmake -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=ON -DENABLE_TESTING=OFF -DENABLE_UNITY_BUILD=ON -DCPP_STANDARD=14 -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY="s3;core;lambda;transfer" -DCMAKE_INSTALL_PREFIX=/opt .. &&
   make -j32 &&
   make install &&
   popd &&
@@ -84,10 +87,10 @@ git clone https://github.com/awslabs/aws-lambda-cpp.git && \
 
 # pcre2
 cd /tmp &&
-  curl -O https://ftp.pcre.org/pub/pcre/pcre2-10.34.zip &&
-  unzip pcre2-10.34.zip &&
-  rm pcre2-10.34.zip &&
-  pushd pcre2-10.34 &&
+  curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.39/pcre2-10.39.zip &&
+  unzip pcre2-10.39.zip &&
+  rm pcre2-10.39.zip &&
+  pushd pcre2-10.39 &&
   ./configure --prefix=/opt --enable-jit=auto --disable-shared CFLAGS="-O2 -fPIC" && make -j 32 && make install
 popd
 

diff --git a/scripts/ubuntu1804/install_reqs.sh b/scripts/ubuntu1804/install_reqs.sh
@@ -137,10 +137,10 @@ git clone https://github.com/awslabs/aws-lambda-cpp.git && \
 
 # pcre2
 cd /tmp &&
-  curl -O https://ftp.pcre.org/pub/pcre/pcre2-10.34.zip &&
-  unzip pcre2-10.34.zip &&
-  rm pcre2-10.34.zip &&
-  pushd pcre2-10.34 &&
+  curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.39/pcre2-10.39.zip &&
+  unzip pcre2-10.39.zip &&
+  rm pcre2-10.39.zip &&
+  pushd pcre2-10.39 &&
   ./configure --prefix=/opt --enable-jit=auto --disable-shared CFLAGS="-O2 -fPIC" && make -j 32 && make install
 popd
 

diff --git a/scripts/ubuntu2004/install_reqs.sh b/scripts/ubuntu2004/install_reqs.sh
@@ -139,10 +139,10 @@ git clone https://github.com/awslabs/aws-lambda-cpp.git && \
 
 # pcre2
 cd /tmp &&
-  curl -O https://ftp.pcre.org/pub/pcre/pcre2-10.34.zip &&
-  unzip pcre2-10.34.zip &&
-  rm pcre2-10.34.zip &&
-  pushd pcre2-10.34 &&
+  curl -LO https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.39/pcre2-10.39.zip &&
+  unzip pcre2-10.39.zip &&
+  rm pcre2-10.39.zip &&
+  pushd pcre2-10.39 &&
   ./configure --prefix=/opt --enable-jit=auto --disable-shared CFLAGS="-O2 -fPIC" && make -j 32 && make install
 popd
Original file line number	Diff line number	Diff line change
Expand Up		@@ -91,3 +91,4 @@ python/tuplex.egg-info/
		tuplex.egg-info/

		wheelhouse/
		*.zip