From aa6ab0c824de73f9e3729dda69c872dce7ba1560 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 17:05:35 -0400
Subject: [PATCH 01/24] script updates

---
 MANIFEST.in                   |  4 ++++
 scripts/build_wheel_linux.sh  | 18 ++++++++++++++++++
 setup.py                      | 28 ++++++++++++++++++----------
 tuplex/codegen/CMakeLists.txt |  5 +++++
 4 files changed, 45 insertions(+), 10 deletions(-)
 create mode 100644 MANIFEST.in
 create mode 100755 scripts/build_wheel_linux.sh

diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 000000000..87a56776d
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,4 @@
+recursive-include tuplex/libexec *
+recursive-include tuplex/tuplex/libexec *
+include *.so
+include *.dll
diff --git a/scripts/build_wheel_linux.sh b/scripts/build_wheel_linux.sh
new file mode 100755
index 000000000..578619bf5
--- /dev/null
+++ b/scripts/build_wheel_linux.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# this script invokes the cibuildwheel process with necessary env variables to build the wheel for linux/docker
+
+cd ..
+# delete dir if exists
+rm -rf wheelhouse
+# delete in tree build files
+rm -rf tuplex/python/tuplex/libexec/tuplex*.so
+
+export CIBUILDWHEEL=1
+export CIBW_ARCHS_LINUX=native
+export CIBW_MANYLINUX_X86_64_IMAGE='registry-1.docker.io/tuplex/ci:latest'
+export CIBW_BUILD="cp3{7,8,9}-*"
+export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
+export CIBW_PROJECT_REQUIRES_PYTHON=">=3.7"
+cibuildwheel --platform linux .
+
+cd scripts
diff --git a/setup.py b/setup.py
index 6de39c413..2e1a2d0b9 100644
--- a/setup.py
+++ b/setup.py
@@ -56,9 +56,9 @@ def build_extension(self, ext):
 
         cfg = "Debug" if self.debug else "Release"
 
-        # debug version
-        cfg = 'Debug'
-        cfg = 'Release'
+        # because still alpha, use RelWithDebInfo
+        cfg = "Debug" if self.debug else "RelWithDebInfo"
+
 
         # CMake lets you override the generator - we need to check this.
         # Can be set with Conda-Build, for example.
@@ -87,7 +87,7 @@ def build_extension(self, ext):
             #       -DBoost_INCLUDE_DIR=/opt/boost/python3.7/include/ \
             #       -DLLVM_ROOT=/usr/lib64/llvm9.0/ ..
             # llvm_root = '/usr/lib64/llvm9.0/' # yum based
-            llvm_root = '/opt/llvm-9.0/'  # manual install
+            llvm_root = '/opt/llvm-9.0'  # manual install
             boost_include_dir = '/opt/boost/python{}/include/'.format(py_maj_min)
             py_include_dir = pyconfig.get_paths()['include']
             py_libs_dir = pyconfig.get_paths()['stdlib']
@@ -127,6 +127,15 @@ def find_pkg_path(lines):
 
         if llvm_root is not None:
             cmake_args.append('-DLLVM_ROOT={}'.format(llvm_root))
+            if os.environ.get('CIBUILDWHEEL', '0') == '1':
+                print('setting prefix path...')
+                # ci buildwheel?
+                # /opt/llvm-9.0/lib/cmake/llvm/
+                prefix_path = "/opt/llvm-9.0/lib/cmake/llvm/" #os.path.join(llvm_root, '/lib/cmake/llvm')
+                #cmake_args.append('-DCMAKE_PREFIX_PATH={}'.format(prefix_path))
+                cmake_args.append('-DLLVM_DIR={}'.format(prefix_path))
+                cmake_args.append('-DLLVM_ROOT_DIR={}'.format(llvm_root))
+
         if py_include_dir is not None:
             cmake_args.append('-DPython3_INCLUDE_DIRS={}'.format(py_include_dir))
         if py_libs_dir is not None:
@@ -290,8 +299,7 @@ def read_readme():
 
 # The information here can also be placed in setup.cfg - better separation of
 # logic and declaration, and simpler if you include description/version in a file.
-setup(
-    name="tuplex",
+setup(name="tuplex",
     python_requires='>=3.7.0',
     version="0.3.0",
     author="Leonhard Spiegelberg",
@@ -302,6 +310,10 @@ def read_readme():
     long_description_content_type='text/markdown',
     packages=discover_packages(where="tuplex/python"),
     package_dir={"": "tuplex/python"},
+    package_data={
+      # include libs in libexec
+    'tuplex.libexec' : ['*.so', '*.dylib']
+    },
     ext_modules=[CMakeExtension("tuplex.libexec.tuplex", "tuplex"), CMakeExtension("tuplex.libexec.tuplex_runtime", "tuplex")],
     cmdclass={"build_ext": CMakeBuild},
     # deactivate for now, first fix python sources to work properly!
@@ -323,10 +335,6 @@ def read_readme():
         'jupyter',
         'nbformat'
     ],
-    package_data={
-        # include libs in libexec
-        'tuplex.libexec': ['*.so', '*.dylib']
-    },
     # metadata for upload to PyPI
     url="https://tuplex.cs.brown.edu",
     license="Apache 2.0",
diff --git a/tuplex/codegen/CMakeLists.txt b/tuplex/codegen/CMakeLists.txt
index e0b2783d6..bd2b7ce1e 100755
--- a/tuplex/codegen/CMakeLists.txt
+++ b/tuplex/codegen/CMakeLists.txt
@@ -79,6 +79,11 @@ ENDIF()
 
 # for brewed llvm, add to cmakemodulepath
 IF(LLVM_ROOT_DIR)
+    message(STATUS "Given LLVM_ROOT_DIR=${LLVM_ROOT_DIR}")
+    message(STATUS "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}")
+    message(STATUS "LLVM_ROOT=${LLVM_ROOT}")
+    EXECUTE_PROCESS(COMMAND "ls" "/opt" COMMAND_ECHO STDOUT)
+    EXECUTE_PROCESS(COMMAND "ls" "${LLVM_ROOT_DIR}" COMMAND_ECHO STDOUT)
     # make cmake find in config mode the right LLVMConfig.cmake file which is located here
     set(LLVM_DIR "${LLVM_ROOT_DIR}/lib/cmake/llvm")
 ENDIF()

From d32a1c4ca587ec1ce3de40eac9031c9bd300e586 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-0-130.ec2.internal>
Date: Mon, 5 Jul 2021 22:20:35 +0000
Subject: [PATCH 02/24] debugging work

---
 scripts/build_wheel_linux.sh         | 3 ++-
 setup.py                             | 1 +
 tuplex/python/src/PythonBindings.cc  | 4 ++--
 tuplex/python/src/PythonException.cc | 7 ++++++-
 4 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/scripts/build_wheel_linux.sh b/scripts/build_wheel_linux.sh
index 578619bf5..f5e723367 100755
--- a/scripts/build_wheel_linux.sh
+++ b/scripts/build_wheel_linux.sh
@@ -10,8 +10,9 @@ rm -rf tuplex/python/tuplex/libexec/tuplex*.so
 export CIBUILDWHEEL=1
 export CIBW_ARCHS_LINUX=native
 export CIBW_MANYLINUX_X86_64_IMAGE='registry-1.docker.io/tuplex/ci:latest'
-export CIBW_BUILD="cp3{7,8,9}-*"
+export CIBW_BUILD="cp3{9}-*"
 export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
+export CIBW_BUILD_VERBOSITY=3
 export CIBW_PROJECT_REQUIRES_PYTHON=">=3.7"
 cibuildwheel --platform linux .
 
diff --git a/setup.py b/setup.py
index 2e1a2d0b9..5d0288358 100644
--- a/setup.py
+++ b/setup.py
@@ -59,6 +59,7 @@ def build_extension(self, ext):
         # because still alpha, use RelWithDebInfo
         cfg = "Debug" if self.debug else "RelWithDebInfo"
 
+        cfg = "Debug"
 
         # CMake lets you override the generator - we need to check this.
         # Can be set with Conda-Build, for example.
diff --git a/tuplex/python/src/PythonBindings.cc b/tuplex/python/src/PythonBindings.cc
index 3642b6398..08f019ed2 100644
--- a/tuplex/python/src/PythonBindings.cc
+++ b/tuplex/python/src/PythonBindings.cc
@@ -35,7 +35,7 @@ PYMODULE {
     // std::cout<<"init done"<<std::endl;
 
 
-    register_exception_translator<tuplex::PythonException>(&tuplex::translateCCException);
+    //register_exception_translator<tuplex::PythonException>(&tuplex::translateCCException);
 
     class_<tuplex::PythonDataSet>("_DataSet")
             .def("show", &tuplex::PythonDataSet::show)
@@ -78,4 +78,4 @@ PYMODULE {
             .def("getTotalCompilationTime", &tuplex::PythonMetrics::getTotalCompilationTime)
             .def("getTotalExceptionCount", &tuplex::PythonMetrics::getTotalExceptionCount)
             .def("getJSONString", &tuplex::PythonMetrics::getJSONString);
-}
\ No newline at end of file
+}
diff --git a/tuplex/python/src/PythonException.cc b/tuplex/python/src/PythonException.cc
index 15774ec7f..70820151f 100644
--- a/tuplex/python/src/PythonException.cc
+++ b/tuplex/python/src/PythonException.cc
@@ -9,10 +9,15 @@
 //--------------------------------------------------------------------------------------------------------------------//
 
 #include <PythonException.h>
+#include <iostream>
 
 namespace tuplex {
 
     void translateCCException(const PythonException& e) {
+        
+
+        std::cerr<<"C++ exception raised: "<<e.what()<<std::endl;
+	std::cerr.flush();
 
 #ifndef NDEBUG
         printf("C++ exception raised: %s", e.what());
@@ -21,4 +26,4 @@ namespace tuplex {
 
         PyErr_SetString(PyExc_Exception, e.what());
     }
-}
\ No newline at end of file
+}

From e02efbe08edabb1db2ec569eb97082df6cbed1e8 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 19:06:35 -0400
Subject: [PATCH 03/24] fix to not crash when no AWS credentials are present

---
 tuplex/core/src/Context.cc           | 12 +++++++++++-
 tuplex/core/src/ee/aws/AWSCommon.cc  |  4 +---
 tuplex/python/src/PythonContext.cc   | 24 ++++++++++++++++++++----
 tuplex/python/src/PythonException.cc |  5 ++---
 4 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/tuplex/core/src/Context.cc b/tuplex/core/src/Context.cc
index 7f3efdd38..72b3a0e40 100644
--- a/tuplex/core/src/Context.cc
+++ b/tuplex/core/src/Context.cc
@@ -44,7 +44,7 @@ namespace tuplex {
         // init AWS SDK to get access to S3 filesystem
         auto aws_credentials = AWSCredentials::get();
         Timer timer;
-        initAWS(aws_credentials, options.AWS_REQUESTER_PAY());
+        bool aws_init_rc = initAWS(aws_credentials, options.AWS_REQUESTER_PAY());
         logger.debug("initialized AWS SDK in " + std::to_string(timer.time()) + "s");
 #endif
 
@@ -59,6 +59,16 @@ namespace tuplex {
 #ifndef BUILD_WITH_AWS
                 throw std::runtime_error("Build Tuplex with -DBUILD_WITH_AWS to enable the AWS Lambda backend");
 #else
+                // warn if credentials are not found.
+                if(!aws_init_rc) {
+                    if(aws_credentials.access_key.empty() || aws_credentials.secret_key.empty())
+                        throw std::runtime_error("To use Tuplex Lambda backend, please specify valid AWS credentials."
+                                                 " E.g., run aws configure or add two environment variables"
+                                                 " AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID");
+                    else
+                        throw std::runtime_error("Requesting Tuplex Lambda backend, but initialization failed.");
+                }
+
                 _ee = std::make_unique<AwsLambdaBackend>(AWSCredentials::get(), "tplxlam", options);
 #endif
                 break;
diff --git a/tuplex/core/src/ee/aws/AWSCommon.cc b/tuplex/core/src/ee/aws/AWSCommon.cc
index 3c7c5ed24..c1807f507 100644
--- a/tuplex/core/src/ee/aws/AWSCommon.cc
+++ b/tuplex/core/src/ee/aws/AWSCommon.cc
@@ -60,10 +60,8 @@ namespace tuplex {
     bool initAWS(const AWSCredentials& credentials, bool requesterPay) {
         initAWSSDK();
 
-#ifndef BUILD_FOR_CI
         if(credentials.secret_key.empty() || credentials.access_key.empty())
-            throw std::runtime_error("empty AWS credentials, please specify them!");
-#endif
+           return false;
 
         // add S3 file system
         VirtualFileSystem::addS3FileSystem(credentials.access_key, credentials.secret_key, "", false, requesterPay);
diff --git a/tuplex/python/src/PythonContext.cc b/tuplex/python/src/PythonContext.cc
index 0839d7191..8264801f6 100644
--- a/tuplex/python/src/PythonContext.cc
+++ b/tuplex/python/src/PythonContext.cc
@@ -1152,7 +1152,7 @@ namespace tuplex {
 
         ContextOptions co = ContextOptions::defaults();
 
-#warning "uncomment this code, because it will cause a deadlock sometimes... (Weird python output shit)"
+#warning "this code is commented, because it will cause a deadlock sometimes... Uncomment to activate output in Jupyter notebook."
         // init logging system here
         // @TODO: add as context option
         // note: this should come BEFORE any Logger::instance()... calls
@@ -1187,12 +1187,28 @@ namespace tuplex {
         // for context creation release GIL
         assert(PyGILState_Check()); // make sure this thread holds the GIL!
         python::unlockGIL();
-        _context = new Context(co);
-        if(!name.empty())
-            _context->setName(name);
+        std::string err_message = ""; // leave this as empty string!
+        try {
+            _context = new Context(co);
+            if(!name.empty())
+                _context->setName(name);
+        } catch(const std::exception& e) {
+            err_message = e.what();
+            assert(!err_message.empty());
+            Logger::instance().defaultLogger().error(err_message);
+        } catch(...) {
+            err_message = "unknown C++ exception occurred, please change type s.t. it's derived from std::exception for meaningful display.";
+            Logger::instance().defaultLogger().error(err_message);
+        }
+
         // restore GIL
         python::lockGIL();
         Logger::instance().flushAll();
+
+        // manually set python error -> do not trust boost::python exception translation, it's faulty!
+        if(!err_message.empty()) {
+            PyErr_SetString(PyExc_RuntimeError, err_message.c_str());
+        }
     }
 
 
diff --git a/tuplex/python/src/PythonException.cc b/tuplex/python/src/PythonException.cc
index 70820151f..44ccbe2fc 100644
--- a/tuplex/python/src/PythonException.cc
+++ b/tuplex/python/src/PythonException.cc
@@ -14,10 +14,9 @@
 namespace tuplex {
 
     void translateCCException(const PythonException& e) {
-        
-
+        // this is buggy... --> switch to pybind11?
         std::cerr<<"C++ exception raised: "<<e.what()<<std::endl;
-	std::cerr.flush();
+	    std::cerr.flush();
 
 #ifndef NDEBUG
         printf("C++ exception raised: %s", e.what());

From e16e92e852913a4b04573b9ad6ee077eb949ac89 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 19:09:50 -0400
Subject: [PATCH 04/24] cibuildwheel test script updated

---
 scripts/build_wheel_linux.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/build_wheel_linux.sh b/scripts/build_wheel_linux.sh
index f5e723367..694c80b27 100755
--- a/scripts/build_wheel_linux.sh
+++ b/scripts/build_wheel_linux.sh
@@ -7,11 +7,19 @@ rm -rf wheelhouse
 # delete in tree build files
 rm -rf tuplex/python/tuplex/libexec/tuplex*.so
 
+
+# CIBUILDWHEEL CONFIGURATION
 export CIBUILDWHEEL=1
 export CIBW_ARCHS_LINUX=native
 export CIBW_MANYLINUX_X86_64_IMAGE='registry-1.docker.io/tuplex/ci:latest'
-export CIBW_BUILD="cp3{9}-*"
-export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
+
+# Use the following line to build only python3.9 wheel
+export CIBW_BUILD="cp39-*"
+
+# to test the others from 3.7-3.9, use these two lines:
+#export CIBW_BUILD="cp3{7,8,9}-*"
+#export CIBW_SKIP="cp3{5,6,7,8}-macosx* pp*"
+
 export CIBW_BUILD_VERBOSITY=3
 export CIBW_PROJECT_REQUIRES_PYTHON=">=3.7"
 cibuildwheel --platform linux .

From 94e046be1b46ebdd1418d2b8ff4859a5110e8c2b Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 19:50:03 -0400
Subject: [PATCH 05/24] script updates

---
 pyproject.toml         |   1 +
 scripts/.gitignore     |   1 +
 scripts/set_version.py | 123 ++++++++++++++++++++++++++++++++---------
 3 files changed, 99 insertions(+), 26 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index fc6863776..ac5a97441 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,5 +7,6 @@ requires = [
     "numpy",
     "ninja; sys_platform != 'win32'",
     "cmake>=3.12",
+    "requests"
 ]
 build-backend = "setuptools.build_meta"
\ No newline at end of file
diff --git a/scripts/.gitignore b/scripts/.gitignore
index 7c7508eeb..2b95306a0 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -3,4 +3,5 @@
 python_backup/
 python_patched/
 *.tgz
+*.log
 
diff --git a/scripts/set_version.py b/scripts/set_version.py
index e20752640..a50e61c58 100644
--- a/scripts/set_version.py
+++ b/scripts/set_version.py
@@ -1,40 +1,111 @@
 import os
 import re
 import datetime
+import argparse
+import logging
+import sys
+import requests
+import re
+from distutils.version import StrictVersion
 
+# to create a testpypi version use X.Y.devN
 version = '0.3.0'
-major, minor, patch = version.split('.')
 
-# paths etc.
-doc_path = '../doc/source/conf.py'
-version_py_path = '../tuplex/python/tuplex/utils/version.py'
-setup_py_path = '../tuplex/python/setup.py'
-toplevel_setup_py_path = '../setup.py'
-version_hist_path = '../tuplex/historyserver/thserver/version.py'
+# https://pypi.org/simple/tuplex/
+# or https://test.pypi.org/simple/tuplex/
+def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
+    r = requests.get(url)
+
+    # parse all strings from page
+    links = re.findall(r'href=[\'"]?([^\'" >]+)', r.text)
+
+    links = list(filter(lambda s: 'tuplex' in s, map(lambda s: s[s.find('tuplex'):s.rfind('.whl')], links)))
+
+    # extract version string & sort
+    links = {link[len('tuplex-'):link.find('-cp')] for link in links}
+
+    links = sorted(list(links), key=StrictVersion)
+
+    # what's the latest version?
+    return links[-1]
+
+if __name__ == '__main__':
+    file_handler = logging.FileHandler(filename='version.log')
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    handlers = [file_handler, stdout_handler]
+
+    logging.basicConfig(level=logging.INFO,
+        format='[%(asctime)s] %(levelname)s - %(message)s',
+        handlers=handlers)
+
+    parser = argparse.ArgumentParser(description='Zillow cleaning')
+    parser.add_argument('--version', type=str, dest='version', default=version,
+                        help='version string to use (better modify this file)')
+    parser.add_argument('--dev', dest='dev', action="store_true",
+                        help="whether to create a latest dev version or not")
+    args = parser.parse_args()
+
+    if version != args.version:
+        logging.info('renaming script uses different version then specified.')
+        version = args.version
+
+    major, minor, patch = version.split('.')
+
+    # check if version is not there yet...
+    version_pypi = get_latest_pypi_version()
+    version_test = get_latest_pypi_version(url='https://test.pypi.org/simple/tuplex/')
+    logging.info('latest pypi.org version of tuplex is: {}'.format(version_pypi))
+    logging.info('latest test.pypi.org version of tuplex is: {}'.format(version_test))
+
+    if args.dev:
+        # get from testpypi (dynamic renaming basically)
+        major, minor, patch = version_test.split('.')
+
+        # check if there's dev in latest version, if not create one!
+        if 'dev' not in patch:
+            patch = 'dev0'
+        else:
+            # patch should be dev...
+            no = int(patch[len('dev'):])
+            patch = 'dev' + str(no + 1)
+        dev_version = '{}.{}.{}'.format(major, minor, patch)
+        logging.info('creating dev version {}'.format(dev_version))
+    else:
+        # skip if requested version is on test pypi
+        if StrictVersion(version) <= StrictVersion(version_pypi):
+            logging.error("newer version on pypi.org, abort")
+            sys.exit(1)
+
+    # paths etc.
+    doc_path = '../doc/source/conf.py'
+    version_py_path = '../tuplex/python/tuplex/utils/version.py'
+    setup_py_path = '../tuplex/python/setup.py'
+    toplevel_setup_py_path = '../setup.py'
+    version_hist_path = '../tuplex/historyserver/thserver/version.py'
 
 
-# modify files...
-with open(version_py_path, 'w') as fp:
-    fp.writelines('# (c) L.Spiegelberg 2017 - {}\n__version__="{}"'.format(datetime.datetime.now().year, version))
+    # modify files...
+    with open(version_py_path, 'w') as fp:
+        fp.writelines('# (c) L.Spiegelberg 2017 - {}\n__version__="{}"'.format(datetime.datetime.now().year, version))
 
-with open(version_hist_path, 'w') as fp:
-    fp.writelines('# (c) L.Spiegelberg 2017 - {}\n__version__="{}"'.format(datetime.datetime.now().year, version))
+    with open(version_hist_path, 'w') as fp:
+        fp.writelines('# (c) L.Spiegelberg 2017 - {}\n__version__="{}"'.format(datetime.datetime.now().year, version))
 
-with open(setup_py_path, 'r') as fp:
-    contents = fp.read()
-with open(setup_py_path, 'w') as fp:
-    fp.write(re.sub('version\s*=\s*[\'"].*[\'"]', 'version="{}"'.format(version), contents))
+    with open(setup_py_path, 'r') as fp:
+        contents = fp.read()
+    with open(setup_py_path, 'w') as fp:
+        fp.write(re.sub('version\s*=\s*[\'"].*[\'"]', 'version="{}"'.format(version), contents))
 
-with open(toplevel_setup_py_path, 'r') as fp:
-    contents = fp.read()
-with open(toplevel_setup_py_path, 'w') as fp:
-    fp.write(re.sub('version\s*=\s*[\'"].*[\'"]', 'version="{}"'.format(version), contents))
+    with open(toplevel_setup_py_path, 'r') as fp:
+        contents = fp.read()
+    with open(toplevel_setup_py_path, 'w') as fp:
+        fp.write(re.sub('version\s*=\s*[\'"].*[\'"]', 'version="{}"'.format(version), contents))
 
-with open(doc_path, 'r') as fp:
-    contents = fp.read()
+    with open(doc_path, 'r') as fp:
+        contents = fp.read()
 
-contents = re.sub('version\s*=\s*[\'"].*[\'"]', 'version="{}.{}"'.format(major, minor), contents)
-contents = re.sub('release\s*=\s*[\'"].*[\'"]', 'release="{}"'.format(version), contents)
+    contents = re.sub('version\s*=\s*[\'"].*[\'"]', 'version="{}.{}"'.format(major, minor), contents)
+    contents = re.sub('release\s*=\s*[\'"].*[\'"]', 'release="{}"'.format(version), contents)
 
-with open(doc_path, 'w') as fp:
-    fp.write(contents)
+    with open(doc_path, 'w') as fp:
+        fp.write(contents)

From c40e6cc6f22231398f0c1342c760cc5c4ebf4005 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 20:08:52 -0400
Subject: [PATCH 06/24] script updates

---
 .github/workflows/build_wheels.yml | 28 ++++++++++++++++++++++++++--
 scripts/set_version.py             |  6 ++++--
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 125ba8e37..d98ef0e92 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -31,6 +31,11 @@ jobs:
       #   with:
       #     platforms: all
 
+      - name: Create dev version
+        if: github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true
+        run: cd ./scripts && python3 set_version.py --dev
+        shell: bash
+
       - name: MacOS dependencies
         if: runner.os == 'macOS'
         run: bash ./scripts/ci/setup-macos.sh
@@ -67,8 +72,8 @@ jobs:
     needs: [ build_wheels ]
     runs-on: ubuntu-20.04
     # remove repository url to publish to default pypi.
-    # upload to PyPI on every tag starting with 'v'
-    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+    # upload to PyPI on every tag starting with 'v' ONLY on official tuplex repo.
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && github.repository == 'tuplex/tuplex'
     # alternatively, to publish when a GitHub Release is created, use the following rule:
     # if: github.event_name == 'release' && github.event.action == 'published'
     steps:
@@ -82,3 +87,22 @@ jobs:
           user: ${{ secrets.pypi_user }}
           password: ${{ secrets.pypi_password }}
           # repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
+
+  upload_testpypi:
+    needs: [ build_wheels ]
+    runs-on: ubuntu-20.04
+    # inverse condition, always create test release, any repo with passwords can work with this.
+    if: github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@v1.4.2
+        with:
+          user: ${{ secrets.pypi_user }}
+          password: ${{ secrets.pypi_password }}
+          repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
+
+      # TODO: add here test checkout using matrix of python versions to validate everything runs...
diff --git a/scripts/set_version.py b/scripts/set_version.py
index a50e61c58..b248bee05 100644
--- a/scripts/set_version.py
+++ b/scripts/set_version.py
@@ -43,6 +43,8 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
                         help='version string to use (better modify this file)')
     parser.add_argument('--dev', dest='dev', action="store_true",
                         help="whether to create a latest dev version or not")
+    parser.add_argument('-f', '--force', dest='force', action="store_true",
+                        help="whether to force version on files")
     args = parser.parse_args()
 
     if version != args.version:
@@ -69,10 +71,11 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
             no = int(patch[len('dev'):])
             patch = 'dev' + str(no + 1)
         dev_version = '{}.{}.{}'.format(major, minor, patch)
+        version = dev_version
         logging.info('creating dev version {}'.format(dev_version))
     else:
         # skip if requested version is on test pypi
-        if StrictVersion(version) <= StrictVersion(version_pypi):
+        if not args.force and StrictVersion(version) <= StrictVersion(version_pypi):
             logging.error("newer version on pypi.org, abort")
             sys.exit(1)
 
@@ -83,7 +86,6 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
     toplevel_setup_py_path = '../setup.py'
     version_hist_path = '../tuplex/historyserver/thserver/version.py'
 
-
     # modify files...
     with open(version_py_path, 'w') as fp:
         fp.writelines('# (c) L.Spiegelberg 2017 - {}\n__version__="{}"'.format(datetime.datetime.now().year, version))

From 59d6831f3e5a6bf67b62973cc53445b9928cbff7 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 20:11:44 -0400
Subject: [PATCH 07/24] adding test step

---
 .github/workflows/build_wheels.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index d98ef0e92..660d43a1c 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -106,3 +106,6 @@ jobs:
           repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
 
       # TODO: add here test checkout using matrix of python versions to validate everything runs...
+      - name: Pip check
+        run: python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple tuplex && python3 -c 'import tuplex; c = tuplex.Context()'
+        shell: bash
\ No newline at end of file

From c6463f6e573856c7bbea5755847b20ba3728d3be Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 20:12:53 -0400
Subject: [PATCH 08/24] doc update

---
 tuplex/awslambda/README.md | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/tuplex/awslambda/README.md b/tuplex/awslambda/README.md
index e56e81831..606e0ac35 100644
--- a/tuplex/awslambda/README.md
+++ b/tuplex/awslambda/README.md
@@ -2,7 +2,7 @@
 
 This folder contains code for the AWS C++ Lambda executor which can be deployed when using the AWS Lambda backend.
 
-Can only built on Linux.
+Is only built on Linux (for MacOS use the hosted package).
 
 # Setup Guide
 ## Initial AWS Setup
@@ -19,6 +19,7 @@ Can only built on Linux.
 - Go to `benchmark/distributed/tuplex/`
 - Run `python runtuplex.py --output-path s3://<your-bucket>/zillow.csv --scratch-dir s3://<your-bucket>/scratch` 
 - You can add `--path s3://tuplex-public/data/1000GB/*.csv` to the command above to run over 1000GB instead of the default (100GB).
+
 ### AWS Parameters
 - Some relevant parameters that can be set up in the tuplex config are:
   - `scratchDir`: an s3 location that tuplex can use for scratch files (MUST BE SET TO RUN ANY PIPELINE!)
@@ -27,8 +28,7 @@ Can only built on Linux.
   - `aws.connectTimeout`: the connection timeout (s),
   - `aws.maxConcurrency`: the maximum amount of executors to be used concurrently,
   - `aws.requesterPay`: whether to make the requester pay to access s3 files (required for `s3://tuplex-public`),
-
-
+    
 ## Debugging
  - We can view logs on Cloudwatch to debug lambda execution
    - Go to CloudWatch > CloudWatch Logs > Log Groups > /aws/lambda/tplxlam
@@ -50,15 +50,3 @@ Can only built on Linux.
 ## Debugging:
 `sudo yum install elfutils-devel` or `sudo yum install binutils-devel` on Amazon Linux.
 This will enable further debug info in cloud logs.
-
-# Old Information (Amazon Linux 1)
-On Amazon Linux 1, `gdb` is outdated (7.6.1). In order to work with CLion, update it to a more recent version via
-
-```
-# CLion 2021 supports 7.8.x - 10.1 (https://www.jetbrains.com/help/clion/configuring-debugger-options.html)
-
-wget ftp://ftp.gnu.org/gnu/gdb/gdb-10.1.tar.gz
-cd gdb-10.1
-make -j8
-sudo make install
-```

From c9b90be3820521e42fc16263ffd6c520dd860580 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 20:20:01 -0400
Subject: [PATCH 09/24] version update

---
 setup.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 5d0288358..97a521882 100644
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,7 @@
 from distutils import sysconfig
 
 import fnmatch
-
+import re
 
 def find_files(pattern, path):
     result = []
@@ -121,11 +121,15 @@ def find_pkg_path(lines):
         cmake_args = [
             # "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir),
             "-DPYTHON_EXECUTABLE={}".format(sys.executable),
-            "-DVERSION_INFO={}".format(self.distribution.get_version()),
             "-DCMAKE_BUILD_TYPE={}".format(cfg),  # not used on MSVC, but no harm
             "-DPYTHON3_VERSION={}".format(py_maj_min),
         ]
 
+        # add version info if not dev
+        version_cmake = "-DVERSION_INFO={}".format(self.distribution.get_version())
+        if re.match(r'\d+.\d+.\d+', version_cmake):
+            cmake_args.append(version_cmake)
+
         if llvm_root is not None:
             cmake_args.append('-DLLVM_ROOT={}'.format(llvm_root))
             if os.environ.get('CIBUILDWHEEL', '0') == '1':

From 6085dbe47c901f02c81bc467e46cde9638d99e4a Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 22:32:48 -0400
Subject: [PATCH 10/24] dependencies update

---
 setup.py               | 8 ++++----
 tuplex/python/setup.py | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/setup.py b/setup.py
index 97a521882..67171d9b5 100644
--- a/setup.py
+++ b/setup.py
@@ -324,11 +324,13 @@ def read_readme():
     # deactivate for now, first fix python sources to work properly!
     zip_safe=False,
     install_requires=[
+        'jupyter',
+        'nbformat',
         'attrs>=19.2.0',
         'dill>=0.2.7.1',
         'pluggy>=0.6.0, <1.0.0',
         'py>=1.5.2',
-        'Pygments>=2.3.1',
+        'pygments',
         'pytest>=5.3.2',
         'six>=1.11.0',
         'wcwidth>=0.1.7',
@@ -336,9 +338,7 @@ def read_readme():
         'prompt_toolkit>=2.0.7',
         'jedi>=0.13.2',
         'cloudpickle>=0.6.1',
-        'PyYAML>=3.13',
-        'jupyter',
-        'nbformat'
+        'PyYAML>=3.13'
     ],
     # metadata for upload to PyPI
     url="https://tuplex.cs.brown.edu",
diff --git a/tuplex/python/setup.py b/tuplex/python/setup.py
index 2ccaf5fe6..750c894f4 100644
--- a/tuplex/python/setup.py
+++ b/tuplex/python/setup.py
@@ -36,11 +36,13 @@
     license="Apache 2.0",
     keywords="ETL BigData Python LLVM UDF",
     install_requires=[
+        'jupyter',
+        'nbformat',
         'attrs>=19.2.0',
         'dill>=0.2.7.1',
         'pluggy>=0.6.0, <1.0.0',
         'py>=1.5.2',
-        'Pygments>=2.3.1',
+        'pygments',
         'pytest>=5.3.2',
         'six>=1.11.0',
         'wcwidth>=0.1.7',
@@ -48,9 +50,7 @@
         'prompt_toolkit>=2.0.7',
         'jedi>=0.13.2',
         'cloudpickle>=0.6.1',
-        'PyYAML>=3.13',
-        'jupyter',
-        'nbformat'
+        'PyYAML>=3.13'
     ],
     url="https://tuplex.cs.brown.edu"
     #,

From bf67fc74886d46ba68f354d76743fce8a19aba7f Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 22:37:13 -0400
Subject: [PATCH 11/24] version sort

---
 scripts/set_version.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/set_version.py b/scripts/set_version.py
index b248bee05..e9944601d 100644
--- a/scripts/set_version.py
+++ b/scripts/set_version.py
@@ -6,7 +6,7 @@
 import sys
 import requests
 import re
-from distutils.version import StrictVersion
+from distutils.version import LooseVersion
 
 # to create a testpypi version use X.Y.devN
 version = '0.3.0'
@@ -24,7 +24,7 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
     # extract version string & sort
     links = {link[len('tuplex-'):link.find('-cp')] for link in links}
 
-    links = sorted(list(links), key=StrictVersion)
+    links = sorted(list(links), key=LooseVersion)
 
     # what's the latest version?
     return links[-1]

From d9cb3650b342002bcd8eb7a713e73772396930c9 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 23:25:41 -0400
Subject: [PATCH 12/24] getting ready for hotfix release

---
 doc/source/conf.py                       | 2 +-
 scripts/set_version.py                   | 4 ++--
 setup.py                                 | 5 +++--
 tuplex/historyserver/thserver/version.py | 2 +-
 tuplex/python/setup.py                   | 2 +-
 tuplex/python/tuplex/utils/version.py    | 2 +-
 6 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/doc/source/conf.py b/doc/source/conf.py
index bb07ee4b9..fea23a619 100755
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -36,7 +36,7 @@
 # The short X.Y version
 version="0.3"
 # The full version, including alpha/beta/rc tags
-release="0.3.0"
+release="0.3.1"
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/scripts/set_version.py b/scripts/set_version.py
index e9944601d..9c37dcea1 100644
--- a/scripts/set_version.py
+++ b/scripts/set_version.py
@@ -9,7 +9,7 @@
 from distutils.version import LooseVersion
 
 # to create a testpypi version use X.Y.devN
-version = '0.3.0'
+version = '0.3.1'
 
 # https://pypi.org/simple/tuplex/
 # or https://test.pypi.org/simple/tuplex/
@@ -75,7 +75,7 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
         logging.info('creating dev version {}'.format(dev_version))
     else:
         # skip if requested version is on test pypi
-        if not args.force and StrictVersion(version) <= StrictVersion(version_pypi):
+        if not args.force and LooseVersion(version) <= LooseVersion(version_pypi):
             logging.error("newer version on pypi.org, abort")
             sys.exit(1)
 
diff --git a/setup.py b/setup.py
index 67171d9b5..eb9b5a482 100644
--- a/setup.py
+++ b/setup.py
@@ -59,7 +59,8 @@ def build_extension(self, ext):
         # because still alpha, use RelWithDebInfo
         cfg = "Debug" if self.debug else "RelWithDebInfo"
 
-        cfg = "Debug"
+        # force release version
+        cfg = "Release"
 
         # CMake lets you override the generator - we need to check this.
         # Can be set with Conda-Build, for example.
@@ -306,7 +307,7 @@ def read_readme():
 # logic and declaration, and simpler if you include description/version in a file.
 setup(name="tuplex",
     python_requires='>=3.7.0',
-    version="0.3.0",
+    version="0.3.1",
     author="Leonhard Spiegelberg",
     author_email="tuplex@cs.brown.edu",
     description="Tuplex is a novel big data analytics framework incorporating a Python UDF compiler based on LLVM "
diff --git a/tuplex/historyserver/thserver/version.py b/tuplex/historyserver/thserver/version.py
index a0e48681f..6fe15431e 100644
--- a/tuplex/historyserver/thserver/version.py
+++ b/tuplex/historyserver/thserver/version.py
@@ -1,2 +1,2 @@
 # (c) L.Spiegelberg 2017 - 2021
-__version__="0.3.0"
\ No newline at end of file
+__version__="0.3.1"
\ No newline at end of file
diff --git a/tuplex/python/setup.py b/tuplex/python/setup.py
index 750c894f4..dc4ad96de 100644
--- a/tuplex/python/setup.py
+++ b/tuplex/python/setup.py
@@ -23,7 +23,7 @@
 
 setup(
     name="Tuplex",
-    version="0.3.0",
+    version="0.3.1",
     packages=find_packages(),
     package_data={
       # include libs in libexec
diff --git a/tuplex/python/tuplex/utils/version.py b/tuplex/python/tuplex/utils/version.py
index a0e48681f..6fe15431e 100644
--- a/tuplex/python/tuplex/utils/version.py
+++ b/tuplex/python/tuplex/utils/version.py
@@ -1,2 +1,2 @@
 # (c) L.Spiegelberg 2017 - 2021
-__version__="0.3.0"
\ No newline at end of file
+__version__="0.3.1"
\ No newline at end of file

From 7f59d7ea6e40c2cc380fdd36d124fba77c871945 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Mon, 5 Jul 2021 23:29:17 -0400
Subject: [PATCH 13/24] fix

---
 setup.py               | 2 +-
 tuplex/python/setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index eb9b5a482..fdb635522 100644
--- a/setup.py
+++ b/setup.py
@@ -331,7 +331,7 @@ def read_readme():
         'dill>=0.2.7.1',
         'pluggy>=0.6.0, <1.0.0',
         'py>=1.5.2',
-        'pygments',
+        'pygments>=2.4.1',
         'pytest>=5.3.2',
         'six>=1.11.0',
         'wcwidth>=0.1.7',
diff --git a/tuplex/python/setup.py b/tuplex/python/setup.py
index dc4ad96de..0e7a0e775 100644
--- a/tuplex/python/setup.py
+++ b/tuplex/python/setup.py
@@ -42,7 +42,7 @@
         'dill>=0.2.7.1',
         'pluggy>=0.6.0, <1.0.0',
         'py>=1.5.2',
-        'pygments',
+        'pygments>=2.4.1',
         'pytest>=5.3.2',
         'six>=1.11.0',
         'wcwidth>=0.1.7',

From 1f46056439656ed4680d828fa4ffdc7739a6e424 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 00:22:24 -0400
Subject: [PATCH 14/24] trigger fix

---
 .github/workflows/build_wheels.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 660d43a1c..d8cf45191 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -92,7 +92,8 @@ jobs:
     needs: [ build_wheels ]
     runs-on: ubuntu-20.04
     # inverse condition, always create test release, any repo with passwords can work with this.
-    if: github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true
+    # note, pull requests are not sharing secrets...
+    if: github.event_name != 'pull_request' && (github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true)
     steps:
       - uses: actions/download-artifact@v2
         with:

From 4dc54d75a4f25b0c88f157ea211e02860438c720 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 09:51:37 -0400
Subject: [PATCH 15/24] comment

---
 tuplex/python/src/PythonBindings.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tuplex/python/src/PythonBindings.cc b/tuplex/python/src/PythonBindings.cc
index 08f019ed2..c110fb098 100644
--- a/tuplex/python/src/PythonBindings.cc
+++ b/tuplex/python/src/PythonBindings.cc
@@ -34,8 +34,8 @@ PYMODULE {
     // std::cout<<"Status of PyEval_ThreadsInitialized threads: "<<PyEval_ThreadsInitialized()<<std::endl;
     // std::cout<<"init done"<<std::endl;
 
-
-    //register_exception_translator<tuplex::PythonException>(&tuplex::translateCCException);
+    // deactivated C++ -> python exception translation, because it's buggy for some reason.
+    // register_exception_translator<tuplex::PythonException>(&tuplex::translateCCException);
 
     class_<tuplex::PythonDataSet>("_DataSet")
             .def("show", &tuplex::PythonDataSet::show)

From 4feac9f06feb64ab828a409bc335dd0d00a60ffb Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 16:22:13 -0400
Subject: [PATCH 16/24] action fix

---
 .github/workflows/build_wheels.yml | 5 +++--
 scripts/.gitignore                 | 2 +-
 scripts/set_version.py             | 9 +++++++++
 3 files changed, 13 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 scripts/set_version.py

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index d8cf45191..1f9a6ee7d 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -31,6 +31,7 @@ jobs:
       #   with:
       #     platforms: all
 
+      # this will create a dummy dev version based on the current time to avoid conflicts on test.pypi.org
       - name: Create dev version
         if: github.event_name != 'push' || startsWith(github.event.ref, 'refs/tags/v') != true
         run: cd ./scripts && python3 set_version.py --dev
@@ -106,7 +107,7 @@ jobs:
           password: ${{ secrets.pypi_password }}
           repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
 
-      # TODO: add here test checkout using matrix of python versions to validate everything runs...
+      # test checkout of the created dev version. Note: There might be a version conflict, but else this should be alright.
       - name: Pip check
-        run: python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple tuplex && python3 -c 'import tuplex; c = tuplex.Context()'
+        run: python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple "tuplex==$(cat ./scripts/dev.version)" && python3 -c 'import tuplex; c = tuplex.Context()'
         shell: bash
\ No newline at end of file
diff --git a/scripts/.gitignore b/scripts/.gitignore
index 2b95306a0..6b38be4c1 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -4,4 +4,4 @@ python_backup/
 python_patched/
 *.tgz
 *.log
-
+*.version
diff --git a/scripts/set_version.py b/scripts/set_version.py
old mode 100644
new mode 100755
index 9c37dcea1..d7034b06f
--- a/scripts/set_version.py
+++ b/scripts/set_version.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import os
 import re
 import datetime
@@ -70,9 +71,17 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
             # patch should be dev...
             no = int(patch[len('dev'):])
             patch = 'dev' + str(no + 1)
+
+            # to avoid conflicts use datetime as version!
+            patch = 'dev' + datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+
         dev_version = '{}.{}.{}'.format(major, minor, patch)
         version = dev_version
         logging.info('creating dev version {}'.format(dev_version))
+
+        # write to file
+        with open('dev.version', 'w') as fp:
+            fp.write(dev_version)
     else:
         # skip if requested version is on test pypi
         if not args.force and LooseVersion(version) <= LooseVersion(version_pypi):

From bf1c481674bed7dfc9140cf3585f9d8a7c193742 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 16:31:05 -0400
Subject: [PATCH 17/24] adding ms to version

---
 scripts/set_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/set_version.py b/scripts/set_version.py
index d7034b06f..bd6a3322f 100755
--- a/scripts/set_version.py
+++ b/scripts/set_version.py
@@ -73,7 +73,7 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
             patch = 'dev' + str(no + 1)
 
             # to avoid conflicts use datetime as version!
-            patch = 'dev' + datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+            patch = 'dev' + datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
 
         dev_version = '{}.{}.{}'.format(major, minor, patch)
         version = dev_version

From 27362ac093cfa74ff94d947a7524e9e0853b3093 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 17:26:16 -0400
Subject: [PATCH 18/24] script, bash subsitution not wokring on gh actions

---
 .github/workflows/build_wheels.yml | 2 +-
 scripts/test_pypi.sh               | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100755 scripts/test_pypi.sh

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 1f9a6ee7d..171df1ae1 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -109,5 +109,5 @@ jobs:
 
       # test checkout of the created dev version. Note: There might be a version conflict, but else this should be alright.
       - name: Pip check
-        run: python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple "tuplex==$(cat ./scripts/dev.version)" && python3 -c 'import tuplex; c = tuplex.Context()'
+        run: ./scripts/test_pypi.sh
         shell: bash
\ No newline at end of file
diff --git a/scripts/test_pypi.sh b/scripts/test_pypi.sh
new file mode 100755
index 000000000..dbe940db1
--- /dev/null
+++ b/scripts/test_pypi.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# checks out tuplex from test.pypi
+python3 --version
+cat ./scripts/dev.version
+python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple "tuplex==$(cat ./scripts/dev.version)"
+python3 -c 'import tuplex; c = tuplex.Context()'
\ No newline at end of file

From b88d97f63ac933ddc963aad8adec2ed5a44ee2a9 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 21:16:51 -0400
Subject: [PATCH 19/24] test

---
 .github/workflows/build_wheels.yml | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 171df1ae1..90fa8a188 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -65,7 +65,10 @@ jobs:
 
       - uses: actions/upload-artifact@v2
         with:
-          path: ./wheelhouse/*.whl
+          path: |
+            ./wheelhouse/*.whl
+            ./scripts/*.version
+            ./scripts/test_pypi.sh
 
   # cf. https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml
   # potentially also create a sdist.
@@ -107,7 +110,21 @@ jobs:
           password: ${{ secrets.pypi_password }}
           repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
 
+
+      # for debugging purposes
+      - name: display dir
+        run: ls -R
+
+      - name: test
+        run: ls ./scripts
+
+      - name: test2
+        runt: ls dist
+
+      - name: 'Echo download path'
+        run: echo ${{steps.download.outputs.download-path}}
+
       # test checkout of the created dev version. Note: There might be a version conflict, but else this should be alright.
       - name: Pip check
-        run: ./scripts/test_pypi.sh
+        run: ./dist/scripts/test_pypi.sh
         shell: bash
\ No newline at end of file

From f2f4f2b39be282ccfb976f54844ef516bccaae69 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 21:44:22 -0400
Subject: [PATCH 20/24] typo fix

---
 .github/workflows/build_wheels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 90fa8a188..a431c1f64 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -119,7 +119,7 @@ jobs:
         run: ls ./scripts
 
       - name: test2
-        runt: ls dist
+        run: ls dist
 
       - name: 'Echo download path'
         run: echo ${{steps.download.outputs.download-path}}

From 4cce0eca94378d474ca61210e6103bf07e9cb4c1 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 22:52:01 -0400
Subject: [PATCH 21/24] one more ci try, waiting for it for an hour...

---
 .github/workflows/build_wheels.yml | 27 ++++++++++++++++-----------
 scripts/test_pypi.sh               |  1 +
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index a431c1f64..75315be48 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -62,13 +62,14 @@ jobs:
 
           CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7"
           CIBW_BEFORE_BUILD_MACOS: ./scripts/ci/setup-macos.sh
-
+      - name: reorganize files
+        run: cp ./scripts/*.version ./wheelhouse && cp ./scripts/test_pypi.sj ./wheelhouse
       - uses: actions/upload-artifact@v2
         with:
           path: |
             ./wheelhouse/*.whl
-            ./scripts/*.version
-            ./scripts/test_pypi.sh
+            ./wheelhouse/*.version
+            ./wheelhouse/test_pypi.sh
 
   # cf. https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml
   # potentially also create a sdist.
@@ -86,11 +87,13 @@ jobs:
           name: artifact
           path: dist
 
+      - name: remove test files
+        run: rm dist/*.version && rm dist/*.sh
+
       - uses: pypa/gh-action-pypi-publish@v1.4.2
         with:
           user: ${{ secrets.pypi_user }}
           password: ${{ secrets.pypi_password }}
-          # repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
 
   upload_testpypi:
     needs: [ build_wheels ]
@@ -104,12 +107,8 @@ jobs:
           name: artifact
           path: dist
 
-      - uses: pypa/gh-action-pypi-publish@v1.4.2
-        with:
-          user: ${{ secrets.pypi_user }}
-          password: ${{ secrets.pypi_password }}
-          repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
-
+      - name: reorganize
+        run: mkdir -p scripts && mv dist/*.sh ./scripts/ && mv dist/*.version ./scripts/ && chmod +x ./scripts/test_pypi.sh
 
       # for debugging purposes
       - name: display dir
@@ -124,7 +123,13 @@ jobs:
       - name: 'Echo download path'
         run: echo ${{steps.download.outputs.download-path}}
 
+      - uses: pypa/gh-action-pypi-publish@v1.4.2
+        with:
+          user: ${{ secrets.pypi_user }}
+          password: ${{ secrets.pypi_password }}
+          repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
+
       # test checkout of the created dev version. Note: There might be a version conflict, but else this should be alright.
       - name: Pip check
-        run: ./dist/scripts/test_pypi.sh
+        run: ./scripts/test_pypi.sh
         shell: bash
\ No newline at end of file
diff --git a/scripts/test_pypi.sh b/scripts/test_pypi.sh
index dbe940db1..cfa91d7dd 100755
--- a/scripts/test_pypi.sh
+++ b/scripts/test_pypi.sh
@@ -2,5 +2,6 @@
 # checks out tuplex from test.pypi
 python3 --version
 cat ./scripts/dev.version
+echo "tuplex==$(cat ./scripts/dev.version)"
 python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple "tuplex==$(cat ./scripts/dev.version)"
 python3 -c 'import tuplex; c = tuplex.Context()'
\ No newline at end of file

From 3ee9c3e9409ae2db9326c4cea90132ec7942c08f Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Tue, 6 Jul 2021 23:59:44 -0400
Subject: [PATCH 22/24] typo

---
 .github/workflows/build_wheels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 75315be48..82e998015 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -63,7 +63,7 @@ jobs:
           CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7"
           CIBW_BEFORE_BUILD_MACOS: ./scripts/ci/setup-macos.sh
       - name: reorganize files
-        run: cp ./scripts/*.version ./wheelhouse && cp ./scripts/test_pypi.sj ./wheelhouse
+        run: cp ./scripts/*.version ./wheelhouse && cp ./scripts/test_pypi.sh ./wheelhouse
       - uses: actions/upload-artifact@v2
         with:
           path: |

From 324245cc817a3232c83d969facb93d17c4f838f5 Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Wed, 7 Jul 2021 10:39:30 -0400
Subject: [PATCH 23/24] another try

---
 scripts/test_pypi.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/test_pypi.sh b/scripts/test_pypi.sh
index cfa91d7dd..68064f277 100755
--- a/scripts/test_pypi.sh
+++ b/scripts/test_pypi.sh
@@ -3,5 +3,9 @@
 python3 --version
 cat ./scripts/dev.version
 echo "tuplex==$(cat ./scripts/dev.version)"
+
+# version propagation takes a while...
+sleep 30s
+
 python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple "tuplex==$(cat ./scripts/dev.version)"
 python3 -c 'import tuplex; c = tuplex.Context()'
\ No newline at end of file

From 4c428b2ef716480076dd18285e9899efc0aa51fa Mon Sep 17 00:00:00 2001
From: Leonhard Spiegelberg <leonhard_spiegelberg@brown.edu>
Date: Wed, 7 Jul 2021 16:42:56 -0400
Subject: [PATCH 24/24] cleanup

---
 .github/workflows/build_wheels.yml | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 82e998015..8cfba497f 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -110,26 +110,8 @@ jobs:
       - name: reorganize
         run: mkdir -p scripts && mv dist/*.sh ./scripts/ && mv dist/*.version ./scripts/ && chmod +x ./scripts/test_pypi.sh
 
-      # for debugging purposes
-      - name: display dir
-        run: ls -R
-
-      - name: test
-        run: ls ./scripts
-
-      - name: test2
-        run: ls dist
-
-      - name: 'Echo download path'
-        run: echo ${{steps.download.outputs.download-path}}
-
       - uses: pypa/gh-action-pypi-publish@v1.4.2
         with:
           user: ${{ secrets.pypi_user }}
           password: ${{ secrets.pypi_password }}
-          repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
-
-      # test checkout of the created dev version. Note: There might be a version conflict, but else this should be alright.
-      - name: Pip check
-        run: ./scripts/test_pypi.sh
-        shell: bash
\ No newline at end of file
+          repository_url: https://test.pypi.org/legacy/ # uncomment for test purposes
\ No newline at end of file