tuplex · LeonhardFS · Jul 1, 2022 · Jun 30, 2022 · Jun 30, 2022 · Jun 30, 2022
diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ requires = [
     "cloudpickle",
     "numpy",
     "ninja; sys_platform != 'win32'",
-    "cmake>=3.12,<3.22",
+    "cmake>=3.19,<3.22",
     "delocate; sys.platform == 'darwin'",
     "auditwheel; sys.platform == 'linux'",
     "requests"

diff --git a/scripts/docker/benchmark/install_cmake.sh b/scripts/docker/benchmark/install_cmake.sh
@@ -5,15 +5,16 @@ apt-get install -y curl
 
 # fetch recent cmake & install
 CMAKE_VER_MAJOR=3
-CMAKE_VER_MINOR=19
-CMAKE_VER_PATCH=7
+CMAKE_VER_MINOR=23
+CMAKE_VER_PATCH=2
 CMAKE_VER="${CMAKE_VER_MAJOR}.${CMAKE_VER_MINOR}"
 CMAKE_VERSION="${CMAKE_VER}.${CMAKE_VER_PATCH}"
+URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
 mkdir -p /tmp/build && cd /tmp/build &&
-  curl -sSL https://cmake.org/files/v${CMAKE_VER}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz >cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  tar -v -zxf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  rm -f cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  cd cmake-${CMAKE_VERSION}-Linux-x86_64 &&
+  curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  cd cmake-${CMAKE_VERSION}-linux-x86_64 &&
   cp -rp bin/* /usr/local/bin/ &&
   cp -rp share/* /usr/local/share/ &&
   cd / && rm -rf /tmp/build
diff --git a/scripts/docker/ci/install_cmake.sh b/scripts/docker/ci/install_cmake.sh
@@ -5,15 +5,16 @@ apt-get install -y curl
 
 # fetch recent cmake & install
 CMAKE_VER_MAJOR=3
-CMAKE_VER_MINOR=19
-CMAKE_VER_PATCH=7
+CMAKE_VER_MINOR=23
+CMAKE_VER_PATCH=2
 CMAKE_VER="${CMAKE_VER_MAJOR}.${CMAKE_VER_MINOR}"
 CMAKE_VERSION="${CMAKE_VER}.${CMAKE_VER_PATCH}"
+URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
 mkdir -p /tmp/build && cd /tmp/build &&
-  curl -sSL https://cmake.org/files/v${CMAKE_VER}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz >cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  tar -v -zxf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  rm -f cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  cd cmake-${CMAKE_VERSION}-Linux-x86_64 &&
+  curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  cd cmake-${CMAKE_VERSION}-linux-x86_64 &&
   cp -rp bin/* /usr/local/bin/ &&
   cp -rp share/* /usr/local/share/ &&
   cd / && rm -rf /tmp/build
diff --git a/scripts/set_version.py b/scripts/set_version.py
@@ -20,20 +20,23 @@ def LooseVersion(v):
 # https://pypi.org/simple/tuplex/
 # or https://test.pypi.org/simple/tuplex/
 def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
-    r = requests.get(url)
+    try:
+        r = requests.get(url)
+
+        # parse all strings from page
+        links = re.findall(r'href=[\'"]?([^\'" >]+)', r.text)
 
-    # parse all strings from page
-    links = re.findall(r'href=[\'"]?([^\'" >]+)', r.text)
+        links = list(filter(lambda s: 'tuplex' in s, map(lambda s: s[s.find('tuplex'):s.rfind('.whl')], links)))
 
-    links = list(filter(lambda s: 'tuplex' in s, map(lambda s: s[s.find('tuplex'):s.rfind('.whl')], links)))
+        # extract version string & sort
+        links = {link[len('tuplex-'):link.find('-cp')] for link in links}
 
-    # extract version string & sort
-    links = {link[len('tuplex-'):link.find('-cp')] for link in links}
+        links = sorted(list(links), key=LooseVersion)
 
-    links = sorted(list(links), key=LooseVersion)
-
-    # what's the latest version?
-    return links[-1]
+        # what's the latest version?
+        return links[-1]
+    except:
+        return None
 
 if __name__ == '__main__':
     file_handler = logging.FileHandler(filename='version.log')
@@ -65,6 +68,9 @@ def get_latest_pypi_version(url='https://pypi.org/simple/tuplex/'):
     logging.info('latest pypi.org version of tuplex is: {}'.format(version_pypi))
     logging.info('latest test.pypi.org version of tuplex is: {}'.format(version_test))
 
+    if version_test is None and version_pypi is not None:
+        version_test = version_pypi
+
     if args.dev:
         # get from testpypi (dynamic renaming basically)
         major, minor, patch = version_test.split('.')

diff --git a/scripts/ubuntu1804/install_reqs.sh b/scripts/ubuntu1804/install_reqs.sh
@@ -41,15 +41,16 @@ python3.7 -m pip install --upgrade pip
 
 # fetch recent cmake & install
 CMAKE_VER_MAJOR=3
-CMAKE_VER_MINOR=16
-CMAKE_VER_PATCH=4
+CMAKE_VER_MINOR=23
+CMAKE_VER_PATCH=2
 CMAKE_VER="${CMAKE_VER_MAJOR}.${CMAKE_VER_MINOR}"
 CMAKE_VERSION="${CMAKE_VER}.${CMAKE_VER_PATCH}"
+URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
 mkdir -p /tmp/build && cd /tmp/build &&
-  curl -sSL https://cmake.org/files/v${CMAKE_VER}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz >cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  tar -v -zxf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  rm -f cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  cd cmake-${CMAKE_VERSION}-Linux-x86_64 &&
+  curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  cd cmake-${CMAKE_VERSION}-linux-x86_64 &&
   cp -rp bin/* /usr/local/bin/ &&
   cp -rp share/* /usr/local/share/ &&
   cd / && rm -rf /tmp/build

diff --git a/scripts/ubuntu2004/install_reqs.sh b/scripts/ubuntu2004/install_reqs.sh
@@ -48,15 +48,16 @@ python3 -m pip install --upgrade pip
 
 # fetch recent cmake & install
 CMAKE_VER_MAJOR=3
-CMAKE_VER_MINOR=19
-CMAKE_VER_PATCH=7
+CMAKE_VER_MINOR=23
+CMAKE_VER_PATCH=2
 CMAKE_VER="${CMAKE_VER_MAJOR}.${CMAKE_VER_MINOR}"
 CMAKE_VERSION="${CMAKE_VER}.${CMAKE_VER_PATCH}"
+URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
 mkdir -p /tmp/build && cd /tmp/build &&
-  curl -sSL https://cmake.org/files/v${CMAKE_VER}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz >cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  tar -v -zxf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  rm -f cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz &&
-  cd cmake-${CMAKE_VERSION}-Linux-x86_64 &&
+  curl -sSL $URL -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  tar -v -zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  rm -f cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz &&
+  cd cmake-${CMAKE_VERSION}-linux-x86_64 &&
   cp -rp bin/* /usr/local/bin/ &&
   cp -rp share/* /usr/local/share/ &&
   cd / && rm -rf /tmp/build

diff --git a/setup.py b/setup.py
@@ -58,8 +58,9 @@ def in_google_colab():
 
 # TODO: add option to install these
 test_dependencies = [
-'jupyter',
-'nbformat',
+'nbocnvert<7.0',
+'jupyter<7.0',
+'nbformat<7.0',
 'prompt_toolkit>=2.0.7',
 'pytest>=5.3.2',
 ]
@@ -75,7 +76,8 @@ def in_google_colab():
 ]
 
 # dependencies for AWS Lambda backend...
-aws_lambda_dependencies = ['boto3']
+# boto is broken currently...
+aws_lambda_dependencies = []
 
 
 # manual fix for google colab
@@ -100,7 +102,6 @@ def in_google_colab():
         'PyYAML>=3.13',
         'psutil',
         'pymongo',
-        'boto3',
         'iso8601'
     ]
 else:

diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt
@@ -51,6 +51,7 @@ message(STATUS "Using language versions C++${CMAKE_CXX_STANDARD} and C${CMAKE_C_
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/")
 message(STATUS "additional cmake module path is ${CMAKE_MODULE_PATH}")
 include("${CMAKE_SOURCE_DIR}/cmake/ucm.cmake") #handy package to manipulate compiler flags
+include("${CMAKE_SOURCE_DIR}/cmake/CPM.cmake") # package manager from https://github.com/cpm-cmake/CPM.cmake
 # for debug mode export all symbols
 set(CMAKE_ENABLE_EXPORTS true)
 include(targetLinkLibrariesWithDynamicLookup)
@@ -474,7 +475,87 @@ if(PYTHON3_EXECUTABLE)
     message(STATUS "Detected Python3 Root dir to be: ${Python3_ROOT_DIR}")
 endif()
 
+
+# Computes the realtionship between two version strings.  A version
+# string is a number delineated by '.'s such as 1.3.2 and 0.99.9.1.
+# You can feed version strings with different number of dot versions,
+# and the shorter version number will be padded with zeros: 9.2 <
+# 9.2.1 will actually compare 9.2.0 < 9.2.1.
+#
+# Input: a_in - value, not variable
+#        b_in - value, not variable
+#        result_out - variable with value:
+#                         -1 : a_in <  b_in
+#                          0 : a_in == b_in
+#                          1 : a_in >  b_in
+#
+# Written by James Bigler.
+MACRO(COMPARE_VERSION_STRINGS a_in b_in result_out)
+  # Since SEPARATE_ARGUMENTS using ' ' as the separation token,
+  # replace '.' with ' ' to allow easy tokenization of the string.
+  STRING(REPLACE "." " " a ${a_in})
+  STRING(REPLACE "." " " b ${b_in})
+  SEPARATE_ARGUMENTS(a)
+  SEPARATE_ARGUMENTS(b)
+
+  # Check the size of each list to see if they are equal.
+  LIST(LENGTH a a_length)
+  LIST(LENGTH b b_length)
+
+  # Note that range needs to be one less than the length as the for
+  # loop is inclusive (silly CMake).
+  IF(a_length LESS b_length)
+    # a is shorter
+    MATH(EXPR range "${a_length} - 1")
+  ELSE(a_length LESS b_length)
+    # b is shorter
+    MATH(EXPR range "${b_length} - 1")
+  ENDIF(a_length LESS b_length)
+
+  SET(result 0)
+  FOREACH(index RANGE ${range})
+    IF(result EQUAL 0)
+      # Only continue to compare things as long as they are equal
+      LIST(GET a ${index} a_version)
+      LIST(GET b ${index} b_version)
+      # LESS
+      IF(a_version LESS b_version)
+        SET(result -1)
+      ENDIF(a_version LESS b_version)
+      # GREATER
+      IF(a_version GREATER b_version)
+        SET(result 1)
+      ENDIF(a_version GREATER b_version)
+    ENDIF(result EQUAL 0)
+  ENDFOREACH(index)
+
+  # Copy out the return result
+  SET(${result_out} ${result})
+ENDMACRO(COMPARE_VERSION_STRINGS)
+
+
+
 # this is a macro to find python3 depending on version etc.
+function(FindPython3Exe NAMES VERSION EXECUTABLE)
+	# first check if appropriate program can be found by general path search
+	set(NAMES_TO_SEARCH ${NAMES})
+	separate_arguments(NAMES_TO_SEARCH)
+	message(STATUS "names to search to find python: ${NAMES_TO_SEARCH}")
+	find_program(TEMP_EXE NAMES ${NAMES_TO_SEARCH})
+	message(STATUS "exe: ${TEMP_EXE}")
+	if(TEMP_EXE)
+            # check version (must match VERSION)
+	    execute_process(COMMAND "${TEMP_EXE}" -c "import platform;print(platform.python_version())" RESULT_VARIABLE _result OUTPUT_VARIABLE TEMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+            # check if version matches
+
+	    compare_version_strings(${VERSION} ${TEMP_VERSION} _result)
+	    if(result EQUAL 0)
+	        message(STATUS "Found ${TEMP_EXE} with version ${TEMP_VERSION} matching desired version ${VERSION}")
+		set(${EXECUTABLE} ${TEMP_EXE} PARENT_SCOPE) # write out
+	    endif()
+	endif()
+
+endfunction()
 
 # is a python3 version set?
 if(PYTHON3_VERSION STREQUAL "")
@@ -498,14 +579,21 @@ else()
     if(PYTHON3_VERSION MATCHES "^([0-9]+)\\.([0-9]+)(\\.([0-9]+))?$")
         string(REGEX MATCH "^([0-9]+)\\.([0-9]+)(\\.([0-9]+))?$" VERSION_STRING "${PYTHON3_VERSION}")
         message(STATUS "Forcing python3 version to ${VERSION_STRING}")
+
+       # get python3 maj.min
+       string(REPLACE "." ";" VERSION_LIST ${VERSION_STRING})
+       list(GET VERSION_LIST 0 PYTHON3_VERSION_MAJOR)
+       list(GET VERSION_LIST 1 PYTHON3_VERSION_MINOR)
+
+       #if no executable is specified, try to search for python3 executable
+       FindPython3Exe("python${PYTHON3_VERSION_MAJOR}.${PYTHON3_VERSION_MINOR} python${PYTHON3_VERSION_MAJOR}" "${PYTHON3_VERSION}" Python3_EXECUTABLE)
 	    # include/lib folders might not adhere to schema, therefore if manually given give that precedence
         # try first to find full Python3, if it fails find only Interpreter & Module
         find_package(Python3 ${VERSION_STRING} EXACT COMPONENTS Interpreter Development QUIET)
         if(Python3_FOUND)
             message(STATUS "Found full python3-dev installation")
             set(Python3_Embed_FOUND TRUE)
         else()
-
             set(Python3_FIND_STRATEGY LOCATION)
             # use more modern approach using findpython3
             set(Python3_EXECUTABLE "${PYTHON3_EXECUTABLE}")
@@ -536,6 +624,8 @@ if(Python3_FOUND)
     if(Python3_VERSION_MAJOR LESS 3 OR Python3_VERSION_MINOR LESS 5)
         message(FATAL_ERROR "Tuplex requires python3.5 at least, found incompatible python ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}")
     endif()
+
+
     if(UNIX AND NOT APPLE)
         set(Boost_USE_STATIC_LIBS ON)
     endif()
@@ -567,9 +657,20 @@ else()
     message(FATAL_ERROR "Python not found, required to compile Tuplex.")
 endif()
 
+# retrieve Root dir if empty
+if(NOT "${Python3_ROOT_DIR}" OR "${Python3_ROOT_DIR}" STREQUAL "")
+	# python3 -c 'import sys,pathlib; print(pathlib.Path(sys.executable).parent.parent)'
+       execute_process (COMMAND "${Python3_EXECUTABLE}" -c "import sys,pathlib; print(pathlib.Path(sys.executable).parent.parent)"
+                RESULT_VARIABLE _result
+		OUTPUT_VARIABLE Python3_ROOT_DIR
+                ERROR_QUIET
+                OUTPUT_STRIP_TRAILING_WHITESPACE)	
+endif()
+
 message(STATUS "Using Python3 executable ${Python3_EXECUTABLE}")
 message(STATUS "Found Python3 headers in ${Python3_INCLUDE_DIRS}")
 message(STATUS "Found Python3 libs in ${Python3_LIBRARIES}")
+message(STATUS "Python3 Root dir is ${Python3_ROOT_DIR}")
 
 find_package(pcre2 REQUIRED)
 if(pcre2_FOUND)

diff --git a/tuplex/adapters/cpython/CMakeLists.txt b/tuplex/adapters/cpython/CMakeLists.txt
@@ -1,7 +1,6 @@
 # (c) 2018 Andy Ly
 # this build file builds the adapters of the Tuplex project
 CMAKE_MINIMUM_REQUIRED(VERSION 3.12 FATAL_ERROR)
-
 INCLUDE_DIRECTORIES("include")
 
 INCLUDE_DIRECTORIES("${Python3_INCLUDE_DIRS}")
@@ -21,4 +20,4 @@ target_include_directories(libcpythonadapter PUBLIC
         )
 
 # Declare the library
-target_link_libraries(libcpythonadapter libutils)
+target_link_libraries(libcpythonadapter libutils)
diff --git a/tuplex/awslambda/CMakeLists.txt b/tuplex/awslambda/CMakeLists.txt
@@ -61,10 +61,4 @@ else()
     aws_lambda_package_target(${LAMBDA_NAME})
 endif()
 
-# Add all the python dependencies to the zip file
-set(PYTHON_RESOURCES_LOC ${CMAKE_CURRENT_SOURCE_DIR}/python38_resources)
-set(PYTHON_RESOURCES_ZIP ${PYTHON_RESOURCES_LOC}.zip)
-message("PYTHON_RESOURCES_ZIP = ${PYTHON_RESOURCES_ZIP}")
-message("PYTHON_RESOURCES_LOC = ${PYTHON_RESOURCES_LOC}")
-
 # To build Lambda runner deployment package, use ./scripts/create_lambda.zip.sh
diff --git a/tuplex/cmake/CPM.cmake b/tuplex/cmake/CPM.cmake
@@ -0,0 +1,21 @@
+set(CPM_DOWNLOAD_VERSION 0.35.1)
+
+if(CPM_SOURCE_CACHE)
+  # Expand relative path. This is important if the provided path contains a tilde (~)
+  get_filename_component(CPM_SOURCE_CACHE ${CPM_SOURCE_CACHE} ABSOLUTE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION}))
+  message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}")
+  file(DOWNLOAD
+       https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+       ${CPM_DOWNLOAD_LOCATION}
+  )
+endif()
+
+include(${CPM_DOWNLOAD_LOCATION})