From 4f4eab5376d094cb376bc1690568299b325dae30 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 14 Oct 2023 17:25:42 -0700 Subject: [PATCH 01/31] enable apple silicon builds --- .github/workflows/build_wheels.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 06c9e6029..94ff90e03 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -25,16 +25,16 @@ jobs: cibw-build: "cp311-manylinux_x86_64" - os: macos-11 python-version: "3.8" - cibw-build: "cp38-macosx_x86_64" + cibw-build: "cp38-macosx*" - os: macos-11 python-version: "3.9" - cibw-build: "cp39-macosx_x86_64" + cibw-build: "cp39-macosx*" - os: macos-11 python-version: "3.10" - cibw-build: "cp310-macosx_x86_64" + cibw-build: "cp310-macosx*" - os: macos-11 python-version: "3.11" - cibw-build: "cp311-macosx_x86_64" + cibw-build: "cp311-macosx*" steps: - uses: actions/checkout@v3 @@ -60,6 +60,9 @@ jobs: # bundle aws runner with linux wheel, remove environment variable TUPLEX_LAMBDA_ZIP to remove runner. CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib" + # enable builds for both Intel and Apple Silicon based machines + CIBW_ARCHS_MACOS: x86_64 arm64 + # requires macOS 10.13 at least to build because of C++17 features. CIBW_ENVIRONMENT_MACOS: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64}" From 549a6a76e3162be007cf301cad447cfc5949a107 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 14 Oct 2023 17:28:26 -0700 Subject: [PATCH 02/31] restrict Apple Silicon builds to Python3.9+ --- .github/workflows/build_wheels.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 94ff90e03..e165cc98b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -25,16 +25,25 @@ jobs: cibw-build: "cp311-manylinux_x86_64" - os: macos-11 python-version: "3.8" - cibw-build: "cp38-macosx*" + cibw-build: "cp38-macosx_x86_64" - os: macos-11 python-version: "3.9" - cibw-build: "cp39-macosx*" + cibw-build: "cp39-macosx_x86_64" - os: macos-11 python-version: "3.10" - cibw-build: "cp310-macosx*" + cibw-build: "cp310-macosx_x86_64" - os: macos-11 python-version: "3.11" - cibw-build: "cp311-macosx*" + cibw-build: "cp311-macosx_x86_64" + - os: macos-11 + python-version: "3.9" + cibw-build: "cp39-macosx_arm64" + - os: macos-11 + python-version: "3.10" + cibw-build: "cp310-macosx_arm64" + - os: macos-11 + python-version: "3.11" + cibw-build: "cp311-macosx_arm64" steps: - uses: actions/checkout@v3 @@ -60,9 +69,6 @@ jobs: # bundle aws runner with linux wheel, remove environment variable TUPLEX_LAMBDA_ZIP to remove runner. CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib" - # enable builds for both Intel and Apple Silicon based machines - CIBW_ARCHS_MACOS: x86_64 arm64 - # requires macOS 10.13 at least to build because of C++17 features. CIBW_ENVIRONMENT_MACOS: "CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' JAVA_HOME=${JAVA_HOME_11_X64}" From 72a907f18750b2fc9e941c6727f6892486b8392a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 14 Oct 2023 17:30:36 -0700 Subject: [PATCH 03/31] hw matrix setup --- .github/workflows/build_wheels.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e165cc98b..747a6c192 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -10,40 +10,52 @@ jobs: matrix: os: [ ubuntu-20.04, macos-11 ] python-version: ["3.8", "3.9", "3.10", "3.11"] + hw: ["x86_64", "arm64"] include: - os: ubuntu-20.04 python-version: "3.8" cibw-build: "cp38-manylinux_x86_64" + hw: "x86_64" - os: ubuntu-20.04 python-version: "3.9" cibw-build: "cp39-manylinux_x86_64" + hw: "x86_64" - os: ubuntu-20.04 python-version: "3.10" cibw-build: "cp310-manylinux_x86_64" + hw: "x86_64" - os: ubuntu-20.04 python-version: "3.11" cibw-build: "cp311-manylinux_x86_64" + hw: "x86_64" - os: macos-11 python-version: "3.8" cibw-build: "cp38-macosx_x86_64" + hw: "x86_64" - os: macos-11 python-version: "3.9" cibw-build: "cp39-macosx_x86_64" + hw: "x86_64" - os: macos-11 python-version: "3.10" cibw-build: "cp310-macosx_x86_64" + hw: "x86_64" - os: macos-11 python-version: "3.11" cibw-build: "cp311-macosx_x86_64" + hw: "x86_64" - os: macos-11 python-version: "3.9" cibw-build: "cp39-macosx_arm64" + hw: "arm64" - os: macos-11 python-version: "3.10" cibw-build: "cp310-macosx_arm64" + hw: "arm64" - os: macos-11 python-version: "3.11" cibw-build: "cp311-macosx_arm64" + hw: "arm64" steps: - uses: actions/checkout@v3 From 509406ece486bba5dcead7677ede697915da77ca Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 14 Oct 2023 17:31:11 -0700 Subject: [PATCH 04/31] print status --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 747a6c192..441ce919c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build_wheels: - name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} + name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} - ${{ matrix.hw }} runs-on: ${{ matrix.os }} strategy: matrix: From 1a096f4616c6b00d3558495879e15fa476a89e59 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 14 Oct 2023 17:34:03 -0700 Subject: [PATCH 05/31] matrix magic --- .github/workflows/build_wheels.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 441ce919c..e23748c2c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -11,6 +11,12 @@ jobs: os: [ ubuntu-20.04, macos-11 ] python-version: ["3.8", "3.9", "3.10", "3.11"] hw: ["x86_64", "arm64"] + exclude: + - os: ubuntu-20.04 + hw: "arm64" + - os: macos-11 + python-version: "3.8"" + hw: "arm64" include: - os: ubuntu-20.04 python-version: "3.8" From 7b2239b28cb6c34ba595435c71fa566a96f79102 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 14 Oct 2023 17:34:30 -0700 Subject: [PATCH 06/31] yaml fix --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e23748c2c..141af55f4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -15,7 +15,7 @@ jobs: - os: ubuntu-20.04 hw: "arm64" - os: macos-11 - python-version: "3.8"" + python-version: "3.8" hw: "arm64" include: - os: ubuntu-20.04 From c63d3db7547782e523addb7cd0e08e34c2bded42 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 15 Oct 2023 20:44:49 -0700 Subject: [PATCH 07/31] test --- .github/workflows/build_wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 141af55f4..c2443db17 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -4,13 +4,13 @@ on: [push, pull_request, workflow_dispatch] jobs: build_wheels: - name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} - ${{ matrix.hw }} + name: Build wheel for ${{ matrix.cibw-build }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-20.04, macos-11 ] python-version: ["3.8", "3.9", "3.10", "3.11"] - hw: ["x86_64", "arm64"] + hw: [arm64"] #["x86_64", "arm64"] exclude: - os: ubuntu-20.04 hw: "arm64" @@ -52,7 +52,7 @@ jobs: hw: "x86_64" - os: macos-11 python-version: "3.9" - cibw-build: "cp39-macosx_arm64" + cibw-build: "cp38-macosx_arm64" hw: "arm64" - os: macos-11 python-version: "3.10" @@ -96,4 +96,4 @@ jobs: - uses: actions/upload-artifact@v3 with: path: | - ./wheelhouse/*.whl \ No newline at end of file + ./wheelhouse/*.whl From d2861b7e7a424f7884d1586298154745c25d8e00 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 15 Oct 2023 20:48:35 -0700 Subject: [PATCH 08/31] test --- .github/workflows/build_wheels.yml | 103 +++++++++++++++-------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c2443db17..ea73a6508 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -8,60 +8,61 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04, macos-11 ] - python-version: ["3.8", "3.9", "3.10", "3.11"] - hw: [arm64"] #["x86_64", "arm64"] - exclude: - - os: ubuntu-20.04 - hw: "arm64" - - os: macos-11 - python-version: "3.8" - hw: "arm64" + # os: [ ubuntu-20.04, macos-11 ] + os: ["macos-11"] + python-version: ["3.9"] #["3.8", "3.9", "3.10", "3.11"] + hw: ["arm64"] #["x86_64", "arm64"] + # exclude: + # - os: ubuntu-20.04 + # hw: "arm64" + # - os: macos-11 + # python-version: "3.8" + # hw: "arm64" include: - - os: ubuntu-20.04 - python-version: "3.8" - cibw-build: "cp38-manylinux_x86_64" - hw: "x86_64" - - os: ubuntu-20.04 - python-version: "3.9" - cibw-build: "cp39-manylinux_x86_64" - hw: "x86_64" - - os: ubuntu-20.04 - python-version: "3.10" - cibw-build: "cp310-manylinux_x86_64" - hw: "x86_64" - - os: ubuntu-20.04 - python-version: "3.11" - cibw-build: "cp311-manylinux_x86_64" - hw: "x86_64" - - os: macos-11 - python-version: "3.8" - cibw-build: "cp38-macosx_x86_64" - hw: "x86_64" - - os: macos-11 - python-version: "3.9" - cibw-build: "cp39-macosx_x86_64" - hw: "x86_64" - - os: macos-11 - python-version: "3.10" - cibw-build: "cp310-macosx_x86_64" - hw: "x86_64" - - os: macos-11 - python-version: "3.11" - cibw-build: "cp311-macosx_x86_64" - hw: "x86_64" + # - os: ubuntu-20.04 + # python-version: "3.8" + # cibw-build: "cp38-manylinux_x86_64" + # hw: "x86_64" + # - os: ubuntu-20.04 + # python-version: "3.9" + # cibw-build: "cp39-manylinux_x86_64" + # hw: "x86_64" + # - os: ubuntu-20.04 + # python-version: "3.10" + # cibw-build: "cp310-manylinux_x86_64" + # hw: "x86_64" + # - os: ubuntu-20.04 + # python-version: "3.11" + # cibw-build: "cp311-manylinux_x86_64" + # hw: "x86_64" + # - os: macos-11 + # python-version: "3.8" + # cibw-build: "cp38-macosx_x86_64" + # hw: "x86_64" + # - os: macos-11 + # python-version: "3.9" + # cibw-build: "cp39-macosx_x86_64" + # hw: "x86_64" + # - os: macos-11 + # python-version: "3.10" + # cibw-build: "cp310-macosx_x86_64" + # hw: "x86_64" + # - os: macos-11 + # python-version: "3.11" + # cibw-build: "cp311-macosx_x86_64" + # hw: "x86_64" - os: macos-11 python-version: "3.9" - cibw-build: "cp38-macosx_arm64" - hw: "arm64" - - os: macos-11 - python-version: "3.10" - cibw-build: "cp310-macosx_arm64" - hw: "arm64" - - os: macos-11 - python-version: "3.11" - cibw-build: "cp311-macosx_arm64" + cibw-build: "cp39-macosx_arm64" hw: "arm64" + # - os: macos-11 + # python-version: "3.10" + # cibw-build: "cp310-macosx_arm64" + # hw: "arm64" + # - os: macos-11 + # python-version: "3.11" + # cibw-build: "cp311-macosx_arm64" + # hw: "arm64" steps: - uses: actions/checkout@v3 @@ -78,6 +79,8 @@ jobs: # configure cibuildwheel to build native archs ('auto'), and some # emulated ones CIBW_ARCHS_LINUX: native + CIBW_ARCHS_MACOS: ${{ matrix.hw }} + CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" CIBW_BUILD: ${{ matrix.cibw-build }} From ab572b8bfb22e3c2783555139323745f3bb70938 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 15 Oct 2023 21:33:43 -0700 Subject: [PATCH 09/31] use macos12 for delocate --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ea73a6508..fa2df8cca 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: # os: [ ubuntu-20.04, macos-11 ] - os: ["macos-11"] + os: ["macos-12"] python-version: ["3.9"] #["3.8", "3.9", "3.10", "3.11"] hw: ["arm64"] #["x86_64", "arm64"] # exclude: From 2f346236f5b0d502b5a9019b7af0871bf51d0dcf Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sun, 15 Oct 2023 21:44:08 -0700 Subject: [PATCH 10/31] fix --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index fa2df8cca..77fead2e3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -51,7 +51,7 @@ jobs: # python-version: "3.11" # cibw-build: "cp311-macosx_x86_64" # hw: "x86_64" - - os: macos-11 + - os: macos-12 python-version: "3.9" cibw-build: "cp39-macosx_arm64" hw: "arm64" From 3b07d8406ba2dc9080a9b6664f67d4189dc157b8 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Mon, 23 Oct 2023 19:32:11 -0700 Subject: [PATCH 11/31] update scripts for arm64 wheels --- scripts/build_macos_wheels.sh | 85 ++++++++++++++++++++++++++++++++--- setup.py | 7 +-- 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/scripts/build_macos_wheels.sh b/scripts/build_macos_wheels.sh index 59c0c30f8..b8bbecdfa 100755 --- a/scripts/build_macos_wheels.sh +++ b/scripts/build_macos_wheels.sh @@ -1,9 +1,82 @@ #!/usr/bin/env bash +# (c) 2017-2023 Tuplex team +# builds x86_64 (and arm64) wheels + +# uncomment for debug +#set -euxo pipefail +set -euo pipefail + +function fail { + printf '%s\n' "$1" >&2 + exit "${2-1}" +} + +function detect_instruction_set() { + arch="$(uname -m)" # -i is only linux, -m is linux and apple + if [[ "$arch" = x86_64* ]]; then + if [[ "$(uname -a)" = *ARM64* ]]; then + echo 'arm64' + else + echo 'x86_64' + fi + elif [[ "$arch" = i*86 ]]; then + echo 'x86_32' + elif [[ "$arch" = arm* ]]; then + echo $arch + elif test "$arch" = aarch64; then + echo 'arm64' + else + exit 1 + fi +} # check from where script is invoked CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +echo " || Tuplex macOS wheel builder || " +echo "-- Executing buildwheel script located in $CWD" + +# check platform is darwin +if [ ! "$(uname -s)" = "Darwin" ]; then + fail "Error: Need to run script under macOS" +fi + +# check which tags are supported +arch=$(detect_instruction_set) +echo "-- Detected arch ${arch}" + +xcode_version_str=$(pkgutil --pkg-info=com.apple.pkg.CLTools_Executables | grep version) +echo "-- Detected Xcode ${xcode_version_str}" + +# if no param is given, use defaults to build all +if [ "${arch}" = "arm64" ]; then + # build Python 3.9 - 3.11 +# cp38-macosx_arm64 + CIBW_BUILD=${CIBW_BUILD-"cp3{9,10,11}-macosx_arm64"} +else + # build Python 3.8 - 3.11 + CIBW_BUILD=${CIBW_BUILD-"cp3{8,9,10,11}-macosx_x86_64}"} +fi + +echo "-- Building wheels for ${CIBW_BUILD}" + +# if macOS is 10.x -> use this as minimum +MINIMUM_TARGET="10.13" + +MACOS_VERSION=$(sw_vers -productVersion) +echo "-- processing on MacOS ${MACOS_VERSION}" +function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } + +MACOS_VERSION_MAJOR=`echo $MACOS_VERSION | cut -d . -f1` + +if [ "$MACOS_VERSION_MAJOR" -ge 11 ]; then + echo "-- Newer MacOS detected (>=11.0), using more recent base target." + echo "-- Using minimum target ${MACOS_VERSION_MAJOR}.0" + MINIMUM_TARGET="${MACOS_VERSION_MAJOR}.0" +else + # keep as is + echo "-- defaulting build to use as minimum target ${MINIMUM_TARGET}" +fi -echo "Executing buildwheel script located in $CWD" pushd $CWD > /dev/null cd .. @@ -13,13 +86,13 @@ cd .. # brew extract --version=3.19.4 protobuf $USER/local-podman # brew install $USER/local-podman/protobuf@3.19.4 # i.e., prepend to statemtnt the following: brew tap-new $USER/local; brew extract --force --version=3.19.4 protobuf $USER/local && brew install $USER/local/protobuf@3.19.4 && -export CIBW_BEFORE_BUILD_MACOS="brew install protobuf coreutils zstd zlib libmagic llvm@9 aws-sdk-cpp pcre2 antlr4-cpp-runtime googletest gflags yaml-cpp celero wget boost" -export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=10.13 CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' " +export CIBW_BEFORE_BUILD_MACOS="brew install protobuf coreutils zstd zlib libmagic llvm@16 aws-sdk-cpp pcre2 antlr4-cpp-runtime googletest gflags yaml-cpp celero wget boost ninja snappy" +export CIBW_ENVIRONMENT_MACOS="MACOSX_DEPLOYMENT_TARGET=${MINIMUM_TARGET} CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' " -export CIBW_BUILD="cp3{7,8,9}-*" -export CIBW_SKIP="cp3{5,6}-macosx* pp* *-musllinux_*" +export CIBW_BUILD="${CIBW_BUILD}" +export CIBW_PROJECT_REQUIRES_PYTHON=">=3.8" -export CIBW_PROJECT_REQUIRES_PYTHON=">=3.7" +export CIBW_BUILD_VERBOSITY=3 cibuildwheel --platform macos diff --git a/setup.py b/setup.py index 91285f654..f619ccd81 100644 --- a/setup.py +++ b/setup.py @@ -328,12 +328,13 @@ def build_extension(self, ext): if platform.system().lower() == 'darwin': # mac os, use brewed versions! out_py = subprocess.check_output(['brew', 'info', 'python3']).decode() - print(out_py) def find_pkg_path(lines): - return list(filter(lambda x: 'usr/local' in x, lines.split('\n')))[0] + ans = list(filter(lambda x: 'usr/local' in x, lines.split('\n'))) + return None if 0 == len(ans) else ans[0] out_py = find_pkg_path(out_py) - print('Found python3 @ {}'.format(out_py)) + if out_py: + logging.info('Found python3 @ {}'.format(out_py)) # setups find everything automatically... llvm_root = None From d981cc1d74f6b8dbbd7aea8e29259af2decd8192 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Mon, 23 Oct 2023 20:39:47 -0700 Subject: [PATCH 12/31] experimental transitioning to vcpkg --- tuplex/CMakeLists.txt | 26 +- tuplex/cmake/vcpkg.cmake | 611 ++++++++++++++++++++++++++++++++++++ tuplex/utils/CMakeLists.txt | 44 +-- tuplex/vcpkg.json | 15 + 4 files changed, 649 insertions(+), 47 deletions(-) create mode 100644 tuplex/cmake/vcpkg.cmake create mode 100644 tuplex/vcpkg.json diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index f00dcc5a0..7d728ce55 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -1,6 +1,22 @@ -# (c) 2017 Leonhard Spiegelberg +# (c) 2017-2023 Leonhard Spiegelberg cmake_minimum_required(VERSION 3.16 FATAL_ERROR) +# top-level language specification +# enable c++17 +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +message(STATUS "Using language version: C++${CMAKE_CXX_STANDARD}") + +# add cmake modules from cmake folder +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/") + +# use vcpkg as manager from https://github.com/bitmeal/vcpkg-cmake-integration +#set(VCPKG_VERSION edge) # optional +include("${CMAKE_SOURCE_DIR}/cmake/vcpkg.cmake") + + + + # Tuplex build options: # ===================== @@ -73,14 +89,6 @@ endif() # uncomment to get verbose cmake output # set(CMAKE_VERBOSE_MAKEFILE ON) -# top-level language specification -# enable c++17 -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -message(STATUS "Using language version: C++${CMAKE_CXX_STANDARD}") - -# add cmake modules from cmake folder -list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/") message(STATUS "additional cmake module path is ${CMAKE_MODULE_PATH}") include("${CMAKE_SOURCE_DIR}/cmake/ucm.cmake") #handy package to manipulate compiler flags include("${CMAKE_SOURCE_DIR}/cmake/CPM.cmake") # package manager from https://github.com/cpm-cmake/CPM.cmake diff --git a/tuplex/cmake/vcpkg.cmake b/tuplex/cmake/vcpkg.cmake new file mode 100644 index 000000000..03dbfb5d1 --- /dev/null +++ b/tuplex/cmake/vcpkg.cmake @@ -0,0 +1,611 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# Copyright (C) 2022, Arne Wendt +# + +# vcpkg examples use 3.0.0, assuming this as minimum version for vcpkg cmake toolchain +cmake_minimum_required(VERSION 3.0.0) + +# config: +# - VCPKG_VERSION: +# - "latest": latest git tag (undefined or empty treated as "latest") +# - "edge": last commit on master +# - VCPKG_PARENT_DIR: where to place vcpkg +# - VCPKG_FORCE_SYSTEM_BINARIES: use system cmake, zip, unzip, tar, etc. +# may be necessary on some systems as downloaded binaries may be linked against unsupported libraries +# musl-libc based distros (ALPINE)(!) require use of system binaries, but are AUTO DETECTED! +# - VCPKG_FEATURE_FLAGS: modify feature flags; default are "manifests,versions" +# +# - VCPKG_NO_INIT: do not call vcpkg_init() automatically (for use testing) + + +# set default feature flags if not defined +if(NOT DEFINED VCPKG_FEATURE_FLAGS) + set(VCPKG_FEATURE_FLAGS "manifests,versions" CACHE INTERNAL "necessary vcpkg flags for manifest based autoinstall and versioning") +endif() + +# disable metrics by default +if(NOT DEFINED VCPKG_METRICS_FLAG) + set(VCPKG_METRICS_FLAG "-disableMetrics" CACHE INTERNAL "flag to disable telemtry by default") +endif() + +# enable rebuilding of packages if requested by changed configuration +if(NOT DEFINED VCPKG_RECURSE_REBUILD_FLAG) + set(VCPKG_RECURSE_REBUILD_FLAG "--recurse" CACHE INTERNAL "enable rebuilding of packages if requested by changed configuration by default") +endif() + + +# check_conditions and find neccessary packages +find_package(Git REQUIRED) + + + +# get VCPKG +function(vcpkg_init) + # set environment (not cached) + + # mask musl-libc if masked prior + if(VCPKG_MASK_MUSL_LIBC) + vcpkg_mask_if_musl_libc() + endif() + + # use system binaries + if(VCPKG_FORCE_SYSTEM_BINARIES) + set(ENV{VCPKG_FORCE_SYSTEM_BINARIES} "1") + endif() + + # for use in scripting mode + if(CMAKE_SCRIPT_MODE_FILE) + if(VCPKG_TARGET_TRIPLET) + set(ENV{VCPKG_DEFAULT_TRIPLET} "${VCPKG_DEFAULT_TRIPLET}") + endif() + if(VCPKG_DEFAULT_TRIPLET) + set(ENV{VCPKG_DEFAULT_TRIPLET} "${VCPKG_DEFAULT_TRIPLET}") + endif() + if(VCPKG_HOST_TRIPLET) + set(ENV{VCPKG_DEFAULT_HOST_TRIPLET} "${VCPKG_DEFAULT_HOST_TRIPLET}") + endif() + if(VCPKG_DEFAULT_HOST_TRIPLET) + set(ENV{VCPKG_DEFAULT_HOST_TRIPLET} "${VCPKG_DEFAULT_HOST_TRIPLET}") + endif() + endif() + # end set environment + + + # test for vcpkg availability + # executable path set ? assume all ok : configure + if(VCPKG_EXECUTABLE EQUAL "" OR NOT DEFINED VCPKG_EXECUTABLE) + # configure vcpkg + + # use system binaries? + # IMPORTANT: we have to use system binaries on musl-libc systems, as vcpkg fetches binaries linked against glibc! + vcpkg_set_use_system_binaries_flag() + + # mask musl-libc if no triplet is provided + if( + ( ENV{VCPKG_DEFAULT_TRIPLET} EQUAL "" OR NOT DEFINED ENV{VCPKG_DEFAULT_TRIPLET}) AND + ( ENV{VCPKG_DEFAULT_HOST_TRIPLET} EQUAL "" OR NOT DEFINED ENV{VCPKG_DEFAULT_HOST_TRIPLET}) AND + ( VCPKG_TARGET_TRIPLET EQUAL "" OR NOT DEFINED VCPKG_TARGET_TRIPLET) + ) + # mask musl-libc from vcpkg + vcpkg_mask_if_musl_libc() + else() + message(WARNING "One of VCPKG_TARGET_TRIPLET, ENV{VCPKG_DEFAULT_TRIPLET} or ENV{VCPKG_DEFAULT_HOST_TRIPLET} has been defined. NOT CHECKING FOR musl-libc MASKING!") + endif() + + + # test options + if(VCPKG_PARENT_DIR EQUAL "" OR NOT DEFINED VCPKG_PARENT_DIR) + if(CMAKE_SCRIPT_MODE_FILE) + message(FATAL_ERROR "Explicitly specify VCPKG_PARENT_DIR when running in script mode!") + else() + message(STATUS "VCPKG from: ${CMAKE_CURRENT_BINARY_DIR}") + set(VCPKG_PARENT_DIR "${CMAKE_CURRENT_BINARY_DIR}/") + endif() + endif() + string(REGEX REPLACE "[/\\]$" "" VCPKG_PARENT_DIR "${VCPKG_PARENT_DIR}") + + # test if VCPKG_PARENT_DIR has to be created in script mode + if(CMAKE_SCRIPT_MODE_FILE AND NOT EXISTS "${VCPKG_PARENT_DIR}") + message(STATUS "Creating vcpkg parent directory") + file(MAKE_DIRECTORY "${VCPKG_PARENT_DIR}") + endif() + + + # set path/location varibles to expected path; necessary to detect after a CMake cache clean + vcpkg_set_vcpkg_directory_from_parent() + vcpkg_set_vcpkg_executable() + + # executable is present ? configuring done : fetch and build + execute_process(COMMAND ${VCPKG_EXECUTABLE} version RESULT_VARIABLE VCPKG_TEST_RETVAL OUTPUT_VARIABLE VCPKG_VERSION_BANNER) + if(NOT VCPKG_TEST_RETVAL EQUAL "0") + # reset executable path to prevent malfunction/wrong assumptions in case of error + set(VCPKG_EXECUTABLE "") + + # getting vcpkg + message(STATUS "No VCPKG executable found; getting new version ready...") + + # select compile script + if(WIN32) + set(VCPKG_BUILD_CMD ".\\bootstrap-vcpkg.bat") + else() + set(VCPKG_BUILD_CMD "./bootstrap-vcpkg.sh") + endif() + + # prepare and clone git sources + # include(FetchContent) + # set(FETCHCONTENT_QUIET on) + # set(FETCHCONTENT_BASE_DIR "${VCPKG_PARENT_DIR}") + # FetchContent_Declare( + # vcpkg + + # GIT_REPOSITORY "https://github.com/microsoft/vcpkg" + # GIT_PROGRESS true + + # SOURCE_DIR "${VCPKG_PARENT_DIR}/vcpkg" + # BINARY_DIR "" + # BUILD_IN_SOURCE true + # CONFIGURE_COMMAND "" + # BUILD_COMMAND "" + # ) + # FetchContent_Populate(vcpkg) + + # check for bootstrap script ? ok : fetch repository + if(NOT EXISTS "${VCPKG_DIRECTORY}/${VCPKG_BUILD_CMD}" AND NOT EXISTS "${VCPKG_DIRECTORY}\\${VCPKG_BUILD_CMD}") + message(STATUS "VCPKG bootstrap script not found; fetching...") + # directory existent ? delete + if(EXISTS "${VCPKG_DIRECTORY}") + file(REMOVE_RECURSE "${VCPKG_DIRECTORY}") + endif() + + # fetch vcpkg repo + execute_process(COMMAND ${GIT_EXECUTABLE} clone https://github.com/microsoft/vcpkg WORKING_DIRECTORY "${VCPKG_PARENT_DIR}" RESULT_VARIABLE VCPKG_GIT_CLONE_OK) + if(NOT VCPKG_GIT_CLONE_OK EQUAL "0") + message(FATAL_ERROR "Cloning VCPKG repository from https://github.com/microsoft/vcpkg failed!") + endif() + endif() + + # compute git checkout target + vcpkg_set_version_checkout() + + # hide detached head notice + execute_process(COMMAND ${GIT_EXECUTABLE} config advice.detachedHead false WORKING_DIRECTORY "${VCPKG_DIRECTORY}" RESULT_VARIABLE VCPKG_GIT_HIDE_DETACHED_HEAD_IGNORED) + # checkout asked version + execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${VCPKG_VERSION_CHECKOUT} WORKING_DIRECTORY "${VCPKG_DIRECTORY}" RESULT_VARIABLE VCPKG_GIT_TAG_CHECKOUT_OK) + if(NOT VCPKG_GIT_TAG_CHECKOUT_OK EQUAL "0") + message(FATAL_ERROR "Checking out VCPKG version/tag ${VCPKG_VERSION} failed!") + endif() + + # wrap -disableMetrics in extra single quotes for windows + # if(WIN32 AND NOT VCPKG_METRICS_FLAG EQUAL "" AND DEFINED VCPKG_METRICS_FLAG) + # set(VCPKG_METRICS_FLAG "'${VCPKG_METRICS_FLAG}'") + # endif() + + # build vcpkg + execute_process(COMMAND ${VCPKG_BUILD_CMD} ${VCPKG_USE_SYSTEM_BINARIES_FLAG} ${VCPKG_METRICS_FLAG} WORKING_DIRECTORY "${VCPKG_DIRECTORY}" RESULT_VARIABLE VCPKG_BUILD_OK) + if(NOT VCPKG_BUILD_OK EQUAL "0") + message(FATAL_ERROR "Bootstrapping VCPKG failed!") + endif() + message(STATUS "Built VCPKG!") + + + # get vcpkg path + vcpkg_set_vcpkg_executable() + + # test vcpkg binary + execute_process(COMMAND ${VCPKG_EXECUTABLE} version RESULT_VARIABLE VCPKG_OK OUTPUT_VARIABLE VCPKG_VERSION_BANNER) + if(NOT VCPKG_OK EQUAL "0") + message(FATAL_ERROR "VCPKG executable failed test!") + endif() + + message(STATUS "VCPKG OK!") + message(STATUS "Install packages using VCPKG:") + message(STATUS " * from your CMakeLists.txt by calling vcpkg_add_package()") + message(STATUS " * by providing a 'vcpkg.json' in your project directory [https://devblogs.microsoft.com/cppblog/take-control-of-your-vcpkg-dependencies-with-versioning-support/]") + + # generate empty manifest on vcpkg installation if none is found + if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg.json") + cmake_language(DEFER DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} CALL vcpkg_manifest_generation_finalize) + message(STATUS "If you need an empty manifest for setting up your project, you will find one in your bild directory") + endif() + endif() + + # we have fetched and built, but a clean has been performed + # version banner is set while testing for availability or after build + message(STATUS "VCPKG using:") + string(REGEX REPLACE "\n.*$" "" VCPKG_VERSION_BANNER "${VCPKG_VERSION_BANNER}") + message(STATUS "${VCPKG_VERSION_BANNER}") + + # cache executable path + set(VCPKG_EXECUTABLE ${VCPKG_EXECUTABLE} CACHE STRING "vcpkg executable path" FORCE) + + # initialize manifest generation + vcpkg_manifest_generation_init() + + # install from manifest if ran in script mode + if(CMAKE_SCRIPT_MODE_FILE) + message(STATUS "Running in script mode to setup environment: trying dependency installation from manifest!") + if(EXISTS "./vcpkg.json") + message(STATUS "Found vcpkg.json; installing...") + vcpkg_install_manifest() + else() + message(STATUS "NOT found vcpkg.json; skipping installation") + endif() + endif() + + # set toolchain + set(CMAKE_TOOLCHAIN_FILE "${VCPKG_DIRECTORY}/scripts/buildsystems/vcpkg.cmake") + set(CMAKE_TOOLCHAIN_FILE ${CMAKE_TOOLCHAIN_FILE} PARENT_SCOPE) + set(CMAKE_TOOLCHAIN_FILE ${CMAKE_TOOLCHAIN_FILE} CACHE STRING "") + endif() +endfunction() + + +# # make target triplet from current compiler selection and platform +# # set VCPKG_TARGET_TRIPLET in parent scope +# function(vcpkg_make_set_triplet) +# # get platform: win/linux ONLY +# if(WIN32) +# set(PLATFORM "windows") +# else() +# set(PLATFORM "linux") +# endif() + +# # get bitness: 32/64 ONLY +# if(CMAKE_SIZEOF_VOID_P EQUAL 8) +# set(BITS 64) +# else() +# set(BITS 86) +# endif() + +# set(VCPKG_TARGET_TRIPLET "x${BITS}-${PLATFORM}" PARENT_SCOPE) +# endfunction() + +# set VCPKG_DIRECTORY to assumed path based on VCPKG_PARENT_DIR +# vcpkg_set_vcpkg_directory_from_parent([VCPKG_PARENT_DIR_EXPLICIT]) +function(vcpkg_set_vcpkg_directory_from_parent) + if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) + set(VCPKG_DIRECTORY "${VCPKG_PARENT_DIR}/vcpkg" PARENT_SCOPE) + else() + set(VCPKG_DIRECTORY "${ARGV0}/vcpkg" PARENT_SCOPE) + endif() + # set(VCPKG_DIRECTORY ${VCPKG_DIRECTORY} CACHE STRING "vcpkg tool location" FORCE) +endfunction() + + +# set VCPKG_EXECUTABLE to assumed path based on VCPKG_DIRECTORY +# vcpkg_set_vcpkg_executable([VCPKG_DIRECTORY]) +function(vcpkg_set_vcpkg_executable) + if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) + set(VCPKG_DIRECTORY_EXPLICIT ${VCPKG_DIRECTORY}) + else() + set(VCPKG_DIRECTORY_EXPLICIT ${ARGV0}) + endif() + + if(WIN32) + set(VCPKG_EXECUTABLE "${VCPKG_DIRECTORY_EXPLICIT}/vcpkg.exe" PARENT_SCOPE) + else() + set(VCPKG_EXECUTABLE "${VCPKG_DIRECTORY_EXPLICIT}/vcpkg" PARENT_SCOPE) + endif() +endfunction() + +# determine git checkout target in: VCPKG_VERSION_CHECKOUT +# vcpkg_set_version_checkout([VCPKG_VERSION_EXPLICIT] [VCPKG_DIRECTORY_EXPLICIT]) +function(vcpkg_set_version_checkout) + if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) + set(VCPKG_VERSION_EXPLICIT ${VCPKG_VERSION}) + else() + set(VCPKG_VERSION_EXPLICIT ${ARGV0}) + endif() + if(ARGV1 EQUAL "" OR NOT DEFINED ARGV1) + set(VCPKG_DIRECTORY_EXPLICIT ${VCPKG_DIRECTORY}) + else() + set(VCPKG_DIRECTORY_EXPLICIT ${ARGV1}) + endif() + + # get latest git tag + execute_process(COMMAND git for-each-ref refs/tags/ --count=1 --sort=-creatordate --format=%\(refname:short\) WORKING_DIRECTORY "${VCPKG_DIRECTORY_EXPLICIT}" OUTPUT_VARIABLE VCPKG_GIT_TAG_LATEST) + string(REGEX REPLACE "\n$" "" VCPKG_GIT_TAG_LATEST "${VCPKG_GIT_TAG_LATEST}") + + # resolve versions + if(EXISTS "./vcpkg.json") + # set hash from vcpkg.json manifest + file(READ "./vcpkg.json" VCPKG_MANIFEST_CONTENTS) + + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) + string(JSON VCPKG_BASELINE GET "${VCPKG_MANIFEST_CONTENTS}" "builtin-baseline") + else() + string(REGEX REPLACE "[\n ]" "" VCPKG_MANIFEST_CONTENTS "${VCPKG_MANIFEST_CONTENTS}") + string(REGEX MATCH "\"builtin-baseline\":\"[0-9a-f]+\"" VCPKG_BASELINE "${VCPKG_MANIFEST_CONTENTS}") + string(REPLACE "\"builtin-baseline\":" "" VCPKG_BASELINE "${VCPKG_BASELINE}") + string(REPLACE "\"" "" VCPKG_BASELINE "${VCPKG_BASELINE}") + endif() + + if(NOT "${VCPKG_BASELINE}" EQUAL "") + if(NOT "${VCPKG_VERSION}" EQUAL "" AND DEFINED VCPKG_VERSION) + message(WARNING "VCPKG_VERSION was specified, but vcpkg.json manifest is used and specifies a builtin-baseline; using builtin-baseline: ${VCPKG_BASELINE}") + endif() + set(VCPKG_VERSION_EXPLICIT "${VCPKG_BASELINE}") + message(STATUS "Using VCPKG Version: ") + endif() + endif() + + if("${VCPKG_VERSION_EXPLICIT}" STREQUAL "latest" OR "${VCPKG_VERSION_EXPLICIT}" EQUAL "" OR NOT DEFINED VCPKG_VERSION_EXPLICIT) + set(VCPKG_VERSION_CHECKOUT ${VCPKG_GIT_TAG_LATEST}) + message(STATUS "Using VCPKG Version: ${VCPKG_VERSION_EXPLICIT} (latest)") + elseif("${VCPKG_VERSION_EXPLICIT}" STREQUAL "edge" OR "${VCPKG_VERSION_EXPLICIT}" STREQUAL "master") + set(VCPKG_VERSION_CHECKOUT "master") + message(STATUS "Using VCPKG Version: edge (latest commit)") + else() + message(STATUS "Using VCPKG Version: ${VCPKG_VERSION_EXPLICIT}") + set(VCPKG_VERSION_CHECKOUT ${VCPKG_VERSION_EXPLICIT}) + endif() + + set(VCPKG_VERSION_CHECKOUT ${VCPKG_VERSION_CHECKOUT} PARENT_SCOPE) +endfunction() + +# sets VCPKG_PLATFORM_MUSL_LIBC(ON|OFF) +function(vcpkg_get_set_musl_libc) + if(WIN32 OR APPLE) + # is windows + set(VCPKG_PLATFORM_MUSL_LIBC OFF) + else() + execute_process(COMMAND getconf GNU_LIBC_VERSION RESULT_VARIABLE VCPKG_PLATFORM_GLIBC) + if(VCPKG_PLATFORM_GLIBC EQUAL "0") + # has glibc + set(VCPKG_PLATFORM_MUSL_LIBC OFF) + else() + execute_process(COMMAND ldd --version RESULT_VARIABLE VCPKG_PLATFORM_LDD_OK OUTPUT_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDOUT ERROR_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDERR) + string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" VCPKG_PLATFORM_LDD_VERSION_STDOUT) + string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" VCPKG_PLATFORM_LDD_VERSION_STDERR) + string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" "musl" VCPKG_PLATFORM_LDD_FIND_MUSL_STDOUT) + string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" "musl" VCPKG_PLATFORM_LDD_FIND_MUSL_STDERR) + if( + (VCPKG_PLATFORM_LDD_OK EQUAL "0" AND NOT VCPKG_PLATFORM_LDD_FIND_MUSL_STDOUT EQUAL "-1") OR + (NOT VCPKG_PLATFORM_LDD_OK EQUAL "0" AND NOT VCPKG_PLATFORM_LDD_FIND_MUSL_STDERR EQUAL "-1") + ) + # has musl-libc + # use system binaries + set(VCPKG_PLATFORM_MUSL_LIBC ON) + message(STATUS "VCPKG: System is using musl-libc; using system binaries! (e.g. cmake, curl, zip, tar, etc.)") + else() + # has error... + message(FATAL_ERROR "VCPKG: could detect neither glibc nor musl-libc!") + endif() + endif() + endif() + + # propagate back + set(VCPKG_PLATFORM_MUSL_LIBC ${VCPKG_PLATFORM_MUSL_LIBC} PARENT_SCOPE) +endfunction() + + +# configure environment and CMake variables to mask musl-libc from vcpkg triplet checks +function(vcpkg_mask_musl_libc) + # set target triplet without '-musl' + execute_process(COMMAND ldd --version RESULT_VARIABLE VCPKG_PLATFORM_LDD_OK OUTPUT_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDOUT ERROR_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDERR) + string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" VCPKG_PLATFORM_LDD_VERSION_STDOUT) + string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" VCPKG_PLATFORM_LDD_VERSION_STDERR) + string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" "x86_64" VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDOUT) + string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" "x86_64" VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDERR) + if( + NOT VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDOUT EQUAL "-1" OR + NOT VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDERR EQUAL "-1" + ) + set(VCPKG_TARGET_TRIPLET "x64-linux") + else() + set(VCPKG_TARGET_TRIPLET "x86-linux") + endif() + + set(ENV{VCPKG_DEFAULT_TRIPLET} "${VCPKG_TARGET_TRIPLET}") + set(ENV{VCPKG_DEFAULT_HOST_TRIPLET} "${VCPKG_TARGET_TRIPLET}") + set(VCPKG_TARGET_TRIPLET "${VCPKG_TARGET_TRIPLET}" CACHE STRING "vcpkg default target triplet (possibly dont change)") + message(STATUS "VCPKG: System is using musl-libc; fixing default target triplet as: ${VCPKG_TARGET_TRIPLET}") + + set(VCPKG_MASK_MUSL_LIBC ON CACHE INTERNAL "masked musl-libc") +endfunction() + +# automate musl-libc masking +function(vcpkg_mask_if_musl_libc) + vcpkg_get_set_musl_libc() + if(VCPKG_PLATFORM_MUSL_LIBC) + vcpkg_mask_musl_libc() + endif() +endfunction() + +# sets VCPKG_USE_SYSTEM_BINARIES_FLAG from VCPKG_PLATFORM_MUSL_LIBC and/or VCPKG_FORCE_SYSTEM_BINARIES +# vcpkg_set_use_system_binaries_flag([VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT]) +function(vcpkg_set_use_system_binaries_flag) + if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) + set(VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT ${VCPKG_FORCE_SYSTEM_BINARIES}) + else() + set(VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT ${ARGV0}) + endif() + + vcpkg_get_set_musl_libc() + + if(NOT WIN32 AND (VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT OR VCPKG_PLATFORM_MUSL_LIBC) ) + set(VCPKG_USE_SYSTEM_BINARIES_FLAG "--useSystemBinaries" PARENT_SCOPE) + # has to be propagated to all install calls + set(ENV{VCPKG_FORCE_SYSTEM_BINARIES} "1") + set(VCPKG_FORCE_SYSTEM_BINARIES ON CACHE BOOL "force vcpkg to use system binaries (possibly dont change)") + + message(STATUS "VCPKG: Requested use of system binaries! (e.g. cmake, curl, zip, tar, etc.)") + else() + set(VCPKG_USE_SYSTEM_BINARIES_FLAG "" PARENT_SCOPE) + endif() +endfunction() + + +# install package +function(vcpkg_add_package PKG_NAME) + # if(VCPKG_TARGET_TRIPLET STREQUAL "" OR NOT DEFINED VCPKG_TARGET_TRIPLET) + # vcpkg_make_set_triplet() + # endif() + set(VCPKG_TARGET_TRIPLET_FLAG "") + if(DEFINED VCPKG_TARGET_TRIPLET AND NOT VCPKG_TARGET_TRIPLET EQUAL "") + set(VCPKG_TARGET_TRIPLET_FLAG "--triplet=${VCPKG_TARGET_TRIPLET}") + endif() + + message(STATUS "VCPKG: fetching ${PKG_NAME} via vcpkg_add_package") + execute_process(COMMAND ${VCPKG_EXECUTABLE} ${VCPKG_TARGET_TRIPLET_FLAG} ${VCPKG_RECURSE_REBUILD_FLAG} --feature-flags=-manifests --disable-metrics install "${PKG_NAME}" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} RESULT_VARIABLE VCPKG_INSTALL_OK) + if(NOT VCPKG_INSTALL_OK EQUAL "0") + message(FATAL_ERROR "VCPKG: failed fetching ${PKG_NAME}! Did you call vcpkg_init(<...>)?") + else() + # add package to automatically generated manifest + vcpkg_manifest_generation_add_dependency("${PKG_NAME}") + endif() +endfunction() + + +# install packages from manifest in script mode +function(vcpkg_install_manifest) + # if(VCPKG_TARGET_TRIPLET STREQUAL "" OR NOT DEFINED VCPKG_TARGET_TRIPLET) + # vcpkg_make_set_triplet() + # endif() + # message(STATUS "VCPKG: install from manifest; using target triplet: ${VCPKG_TARGET_TRIPLET}") + # execute_process(COMMAND ${VCPKG_EXECUTABLE} --triplet=${VCPKG_TARGET_TRIPLET} --feature-flags=manifests,versions --disable-metrics install WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} RESULT_VARIABLE VCPKG_INSTALL_OK) + get_filename_component(VCPKG_EXECUTABLE_ABS ${VCPKG_EXECUTABLE} ABSOLUTE) + file(COPY "./vcpkg.json" DESTINATION "${VCPKG_PARENT_DIR}") + execute_process(COMMAND ${VCPKG_EXECUTABLE_ABS} --feature-flags=manifests,versions --disable-metrics install WORKING_DIRECTORY "${VCPKG_PARENT_DIR}" RESULT_VARIABLE VCPKG_INSTALL_OK) + if(NOT VCPKG_INSTALL_OK EQUAL "0") + message(FATAL_ERROR "VCPKG: install from manifest failed") + endif() +endfunction() + +## manifest generation requires CMake > 3.19 +function(vcpkg_manifest_generation_update_cache VCPKG_GENERATED_MANIFEST) + string(REGEX REPLACE "\n" "" VCPKG_GENERATED_MANIFEST "${VCPKG_GENERATED_MANIFEST}") + set(VCPKG_GENERATED_MANIFEST "${VCPKG_GENERATED_MANIFEST}" CACHE STRING "template for automatically generated manifest by vcpkg-cmake-integration" FORCE) + mark_as_advanced(FORCE VCPKG_GENERATED_MANIFEST) +endfunction() + + +# build empty json manifest and register deferred call to finalize and write +function(vcpkg_manifest_generation_init) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) + # init "empty" json and cache variable + set(VCPKG_GENERATED_MANIFEST "{}") + + # initialize dependencies as empty list + # first vcpkg_add_package will transform to object and install finalization handler + # transform to list in finalization step + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "[]") + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" "$schema" "\"https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json\"") + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" version "\"0.1.0-autogenerated\"") + + # write baseline commit + execute_process(COMMAND git log --pretty=format:'%H' -1 WORKING_DIRECTORY "${VCPKG_DIRECTORY}" OUTPUT_VARIABLE VCPKG_GENERATED_MANIFEST_BASELINE) + string(REPLACE "'" "" VCPKG_GENERATED_MANIFEST_BASELINE "${VCPKG_GENERATED_MANIFEST_BASELINE}") + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" builtin-baseline "\"${VCPKG_GENERATED_MANIFEST_BASELINE}\"") + + vcpkg_manifest_generation_update_cache("${VCPKG_GENERATED_MANIFEST}") + + # will be initialized from vcpkg_add_package call + # # defer call to finalize manifest + # # needs to be called later as project variables are not set when initializing + # cmake_language(DEFER CALL vcpkg_manifest_generation_finalize) + endif() +endfunction() + +# add dependency to generated manifest +function(vcpkg_manifest_generation_add_dependency PKG_NAME) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) + # extract features + string(REGEX MATCH "\\[.*\\]" PKG_FEATURES "${PKG_NAME}") + string(REPLACE "${PKG_FEATURES}" "" PKG_BASE_NAME "${PKG_NAME}") + # make comma separated list + string(REPLACE "[" "" PKG_FEATURES "${PKG_FEATURES}") + string(REPLACE "]" "" PKG_FEATURES "${PKG_FEATURES}") + string(REPLACE " " "" PKG_FEATURES "${PKG_FEATURES}") + # build cmake list by separating with ; + string(REPLACE "," ";" PKG_FEATURES "${PKG_FEATURES}") + + if(NOT PKG_FEATURES) + # set package name string only + set(PKG_DEPENDENCY_JSON "\"${PKG_BASE_NAME}\"") + else() + # build dependency object with features + set(PKG_DEPENDENCY_JSON "{}") + string(JSON PKG_DEPENDENCY_JSON SET "${PKG_DEPENDENCY_JSON}" name "\"${PKG_BASE_NAME}\"") + + set(FEATURE_LIST_JSON "[]") + foreach(FEATURE IN LISTS PKG_FEATURES) + if(FEATURE STREQUAL "core") + # set default feature option if special feature "core" is specified + string(JSON PKG_DEPENDENCY_JSON SET "${PKG_DEPENDENCY_JSON}" default-features "false") + else() + # add feature to list + string(JSON FEATURE_LIST_JSON_LEN LENGTH "${FEATURE_LIST_JSON}") + string(JSON FEATURE_LIST_JSON SET "${FEATURE_LIST_JSON}" ${FEATURE_LIST_JSON_LEN} "\"${FEATURE}\"") + endif() + endforeach() + + # build dependency object with feature list + string(JSON PKG_DEPENDENCY_JSON SET "${PKG_DEPENDENCY_JSON}" features "${FEATURE_LIST_JSON}") + endif() + + # add dependency to manifest + # reset to empty object to avoid collissions and track new packages + # defer (new) finalization call + string(JSON VCPKG_GENERATED_MANIFEST_DEPENDENCIES_TYPE TYPE "${VCPKG_GENERATED_MANIFEST}" dependencies) + if(VCPKG_GENERATED_MANIFEST_DEPENDENCIES_TYPE STREQUAL "ARRAY") + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "{}") + cmake_language(DEFER CALL vcpkg_manifest_generation_finalize) + endif() + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "${PKG_BASE_NAME}" "${PKG_DEPENDENCY_JSON}") + + vcpkg_manifest_generation_update_cache("${VCPKG_GENERATED_MANIFEST}") + endif() +endfunction() + + +# build empty json manifest and register deferred call to finalize and write +function(vcpkg_manifest_generation_finalize) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) + # populate project information + string(REGEX REPLACE "[^a-z0-9\\.-]" "" VCPKG_GENERATED_MANIFEST_NAME "${PROJECT_NAME}") + string(TOLOWER VCPKG_GENERATED_MANIFEST_NAME "${VCPKG_GENERATED_MANIFEST_NAME}") + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" name "\"${VCPKG_GENERATED_MANIFEST_NAME}\"") + if(NOT PROJECT_VERSION EQUAL "" AND DEFINED PROJECT_VERSION) + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" version "\"${PROJECT_VERSION}\"") + endif() + + vcpkg_manifest_generation_update_cache("${VCPKG_GENERATED_MANIFEST}") + + # make list from dependency dictionary + # cache dependency object + string(JSON VCPKG_GENERATED_DEPENDENCY_OBJECT GET "${VCPKG_GENERATED_MANIFEST}" dependencies) + # initialize dependencies as list + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "[]") + + string(JSON VCPKG_GENERATED_DEPENDENCY_COUNT LENGTH "${VCPKG_GENERATED_DEPENDENCY_OBJECT}") + if(VCPKG_GENERATED_DEPENDENCY_COUNT GREATER 0) + # setup range stop for iteration + math(EXPR VCPKG_GENERATED_DEPENDENCY_LOOP_STOP "${VCPKG_GENERATED_DEPENDENCY_COUNT} - 1") + + # make list + foreach(DEPENDENCY_INDEX RANGE ${VCPKG_GENERATED_DEPENDENCY_LOOP_STOP}) + string(JSON DEPENDENCY_NAME MEMBER "${VCPKG_GENERATED_DEPENDENCY_OBJECT}" ${DEPENDENCY_INDEX}) + string(JSON DEPENDENCY_JSON GET "${VCPKG_GENERATED_DEPENDENCY_OBJECT}" "${DEPENDENCY_NAME}") + string(JSON DEPENDENCY_JSON_TYPE ERROR_VARIABLE DEPENDENCY_JSON_TYPE_ERROR_IGNORE TYPE "${DEPENDENCY_JSON}") + if(DEPENDENCY_JSON_TYPE STREQUAL "OBJECT") + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies ${DEPENDENCY_INDEX} "${DEPENDENCY_JSON}") + else() + string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies ${DEPENDENCY_INDEX} "\"${DEPENDENCY_JSON}\"") + endif() + endforeach() + endif() + + message(STATUS "VCPKG auto-generated manifest (${CMAKE_CURRENT_BINARY_DIR}/vcpkg.json):\n${VCPKG_GENERATED_MANIFEST}") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.json" "${VCPKG_GENERATED_MANIFEST}") + endif() +endfunction() + + +# get vcpkg and configure toolchain +if(NOT VCPKG_NO_INIT) + vcpkg_init() +endif() \ No newline at end of file diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index 472c02e86..9f753767f 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -5,42 +5,6 @@ file(GLOB_RECURSE INCLUDES include/*.h) include_directories("include") -### include fmtlib/fmt -include(ExternalProject) -set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) -ExternalProject_Add(fmt - GIT_REPOSITORY https://github.com/fmtlib/fmt.git - GIT_TAG 6.2.0 - GIT_CONFIG advice.detachedHead=false - TIMEOUT 5 - CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - PREFIX "${EXTERNAL_INSTALL_LOCATION}" - CONFIGURE_COMMAND "" # Disable configure step - BUILD_COMMAND "" # Disable build step - INSTALL_COMMAND "" # Disable install step - UPDATE_COMMAND "" # Disable update step: clones the project only once - ) -ExternalProject_Get_Property(fmt source_dir) -set(fmt_INCLUDE_DIR ${source_dir}/include) - -# add https://github.com/gabime/spdlog -set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/third_party) -ExternalProject_Add(spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.3.1 - GIT_CONFIG advice.detachedHead=false - TIMEOUT 5 - CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} #-DSPDLOG_BUILD_BENCH=OFF - PREFIX "${EXTERNAL_INSTALL_LOCATION}" - CONFIGURE_COMMAND "" # Disable configure step - BUILD_COMMAND "" # Disable build step - INSTALL_COMMAND "" # Disable install step - UPDATE_COMMAND "" # Disable update step: clones the project only once - ) - -# Specify include dir -ExternalProject_Get_Property(spdlog source_dir) -set(spdlog_INCLUDE_DIR ${source_dir}/include) # Add cJSON ==> newer AWS SDKs ship it, so exclude it then... # AWS SDK defined cjson since v1.5 @@ -93,7 +57,7 @@ include_directories(${json_INCLUDE_DIR}) # ------ # dependencies -add_dependencies(libutils fmt spdlog json) +add_dependencies(libutils json) target_include_directories(libutils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include @@ -105,5 +69,9 @@ target_include_directories(libutils PUBLIC ${Boost_INCLUDE_DIR} ${AWSSDK_INCLUDE_DIR}) +find_package(fmt REQUIRED) +find_package(spdlog REQUIRED) + # Specify here the libraries this program depends on -target_link_libraries(libutils Boost::filesystem Boost::thread Boost::system Boost::system Boost::iostreams ${AWSSDK_LINK_LIBRARIES}) +target_link_libraries(libutils PRIVATE fmt::fmt-header-only spdlog::spdlog_header_only Boost::filesystem Boost::thread Boost::system + Boost::system Boost::iostreams ${AWSSDK_LINK_LIBRARIES} ) diff --git a/tuplex/vcpkg.json b/tuplex/vcpkg.json new file mode 100644 index 000000000..6286fc243 --- /dev/null +++ b/tuplex/vcpkg.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "dependencies": [ + "boost-system", + "fmt", + "spdlog" + ], + "overrides": [ + { + "name": "fmt", + "version": "10.1.1" + } + ], + "builtin-baseline": "3265c187c74914aa5569b75355badebfdbab7987" +} From 1b4b344efb239ef152742677ba37f7d83fd328c2 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Mon, 23 Oct 2023 20:57:53 -0700 Subject: [PATCH 13/31] llvm 17 can be added --- tuplex/vcpkg.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tuplex/vcpkg.json b/tuplex/vcpkg.json index 6286fc243..96ef52f94 100644 --- a/tuplex/vcpkg.json +++ b/tuplex/vcpkg.json @@ -3,12 +3,17 @@ "dependencies": [ "boost-system", "fmt", - "spdlog" + "spdlog", + {"name" : "aws-sdk-cpp", "features": ["s3", "lambda", "transfer"]}, + {"name": "llvm", "features": ["enable-rtti", "enable-zlib", "enable-zstd", "target-aarch64", "target-x86"]} ], "overrides": [ { "name": "fmt", "version": "10.1.1" + }, + {"name": "llvm", + "version": "17.0.2" } ], "builtin-baseline": "3265c187c74914aa5569b75355badebfdbab7987" From 3496ceeed4036df6f1d1da578f5706e4d96faabe Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 24 Oct 2023 21:26:25 -0700 Subject: [PATCH 14/31] vcpkg support step by step --- tuplex/CMakeLists.txt | 2 ++ tuplex/codegen/CMakeLists.txt | 1 + tuplex/python/include/PythonCommon.h | 3 ++- tuplex/runtime/CMakeLists.txt | 2 +- tuplex/runtime/src/Runtime.cc | 1 + tuplex/utils/CMakeLists.txt | 8 +++----- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index 7d728ce55..e56c25d39 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -993,6 +993,8 @@ endif() # ncurses/curses lib for terminal manipulation find_package(Curses REQUIRED) +find_package(fmt REQUIRED) +find_package(spdlog REQUIRED) # add subdirs here... add_subdirectory(io) # <-- make sure to call this first, because it changes parent scope with io dependencies diff --git a/tuplex/codegen/CMakeLists.txt b/tuplex/codegen/CMakeLists.txt index 1147f2fe9..8b76cfec2 100755 --- a/tuplex/codegen/CMakeLists.txt +++ b/tuplex/codegen/CMakeLists.txt @@ -147,4 +147,5 @@ target_link_libraries(libcodegen ${LLVM_LIBRARIES} ${ZLIB_LIBRARIES} ${CURSES_LIBRARIES} + fmt::fmt ) \ No newline at end of file diff --git a/tuplex/python/include/PythonCommon.h b/tuplex/python/include/PythonCommon.h index f6b34d63d..561bea83b 100644 --- a/tuplex/python/include/PythonCommon.h +++ b/tuplex/python/include/PythonCommon.h @@ -1,3 +1,4 @@ + //--------------------------------------------------------------------------------------------------------------------// // // // Tuplex: Blazing Fast Python Data Science // @@ -119,7 +120,7 @@ namespace tuplex { LogMessage msg; msg.message = std::string(spdlog_msg.payload.data()); msg.timestamp = spdlog_msg.time; - msg.logger = *spdlog_msg.logger_name; + msg.logger = std::string(spdlog_msg.logger_name.begin(), spdlog_msg.logger_name.end()); msg.level = spdlog_msg.level; _messageBuffer.push_back(msg); } diff --git a/tuplex/runtime/CMakeLists.txt b/tuplex/runtime/CMakeLists.txt index 6385e6dbc..eb70a12fd 100644 --- a/tuplex/runtime/CMakeLists.txt +++ b/tuplex/runtime/CMakeLists.txt @@ -18,7 +18,7 @@ message(STATUS "Tuplex python language runtime include dir is: ${RUNTIME_INCLUDE target_include_directories(runtime PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${PCRE2_INCLUDE_DIRS}) # keep dependencies lean... -target_link_libraries(runtime libutils ${PCRE2_LIBRARIES}) +target_link_libraries(runtime libutils ${PCRE2_LIBRARIES} fmt::fmt) # require thread_local and aligned malloc keyword (C11 or C++11) target_compile_features(runtime PRIVATE cxx_thread_local) diff --git a/tuplex/runtime/src/Runtime.cc b/tuplex/runtime/src/Runtime.cc index 044c6ff0f..90dc91af1 100644 --- a/tuplex/runtime/src/Runtime.cc +++ b/tuplex/runtime/src/Runtime.cc @@ -18,6 +18,7 @@ #include // <-- implemented in StringUtils #include #include +#include #include #include #include diff --git a/tuplex/utils/CMakeLists.txt b/tuplex/utils/CMakeLists.txt index 9f753767f..1e83fcdfd 100644 --- a/tuplex/utils/CMakeLists.txt +++ b/tuplex/utils/CMakeLists.txt @@ -67,11 +67,9 @@ target_include_directories(libutils PUBLIC ${cjson_SOURCE_DIR} ${json_INCLUDE_DIR} ${Boost_INCLUDE_DIR} - ${AWSSDK_INCLUDE_DIR}) - -find_package(fmt REQUIRED) -find_package(spdlog REQUIRED) + ${AWSSDK_INCLUDE_DIR} + spdlog::spdlog_header_only) # Specify here the libraries this program depends on -target_link_libraries(libutils PRIVATE fmt::fmt-header-only spdlog::spdlog_header_only Boost::filesystem Boost::thread Boost::system +target_link_libraries(libutils spdlog::spdlog_header_only fmt::fmt Boost::filesystem Boost::thread Boost::system Boost::system Boost::iostreams ${AWSSDK_LINK_LIBRARIES} ) From 15c26f586f51d76f2fbe953b5f8134e10b673259 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 24 Oct 2023 21:40:03 -0700 Subject: [PATCH 15/31] wip llvm17 compatibility, remove typed pointers --- tuplex/codegen/CMakeLists.txt | 43 +--------------------- tuplex/codegen/include/CodegenHelper.h | 41 --------------------- tuplex/codegen/src/FlattenedTuple.cc | 1 - tuplex/codegen/src/IteratorContextProxy.cc | 6 +-- tuplex/codegen/src/LLVMEnvironment.cc | 11 +++++- 5 files changed, 11 insertions(+), 91 deletions(-) diff --git a/tuplex/codegen/CMakeLists.txt b/tuplex/codegen/CMakeLists.txt index 8b76cfec2..6bf9dcedf 100755 --- a/tuplex/codegen/CMakeLists.txt +++ b/tuplex/codegen/CMakeLists.txt @@ -20,48 +20,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # this may make it easier but increases size of shared object tremendously set(LLVM_REQUIRED_COMPONENTS core orcjit nativecodegen native scalaropts objcarcopts passes) -IF(BREW_FOUND) - IF(APPLE) - - # there might be multiple LLVM versions installed. - # check which version there is - # if not sys.stdin.isatty(): - # data = sys.stdin.readlines() - - # could use brew prefix here, but let's leave it like this - EXECUTE_PROCESS(COMMAND bash "-c" "brew info llvm | grep Cellar | cut -d ' ' -f 1" OUTPUT_VARIABLE LLVM_ROOT_DIR RESULT_VARIABLE BREW_LLVM_NOTFOUND OUTPUT_STRIP_TRAILING_WHITESPACE) - IF(NOT BREW_LLVM_NOTFOUND EQUAL "0") - MESSAGE(WARNING "did not find llvm, you might install it via `brew install llvm@9`") - ELSE() - # check version, needs to be within 5 and 9 incl. - # i.e. execute something like /usr/local/opt/llvm/bin/llvm-config --version - EXECUTE_PROCESS(COMMAND ${LLVM_ROOT_DIR}/bin/llvm-config --version OUTPUT_VARIABLE LLVM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - - # check if empty, if it is parse again using brew info json - IF("${LLVM_VERSION}" STREQUAL "") - EXECUTE_PROCESS(COMMAND bash "-c" "brew info --json=v1 llvm | python3 -c 'import sys,json; x=json.load(sys.stdin); print(x[0][\"versions\"][\"stable\"])'" OUTPUT_VARIABLE LLVM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND bash "-c" "brew info llvm | grep Cellar | cut -d ' ' -f 1" OUTPUT_VARIABLE LLVM_ROOT_DIR RESULT_VARIABLE BREW_RET OUTPUT_STRIP_TRAILING_WHITESPACE) - ENDIF() - - message(STATUS "Found LLVM ${LLVM_VERSION}") - ENDIF() - - ELSEIF(UNIX) - # ... - ENDIF() -ENDIF() - -# for brewed llvm, add to cmakemodulepath -IF(NOT "${LLVM_ROOT_DIR}" STREQUAL "") - message(STATUS "Detected LLVM root dir: ${LLVM_ROOT_DIR}") - # make cmake find in config mode the right LLVMConfig.cmake file which is located here - set(LLVM_DIR "${LLVM_ROOT_DIR}/lib/cmake/llvm") - FIND_PACKAGE(LLVM 6.0 REQUIRED COMPONENTS ${LLVM_REQUIRED_COMPONENTS}) -ELSE() - FIND_PACKAGE(LLVM 6.0 REQUIRED COMPONENTS ${LLVM_REQUIRED_COMPONENTS}) -ENDIF() - -MESSAGE(STATUS "Found LLVM ${LLVM_VERSION_STRING}") +find_package(LLVM CONFIG REQUIRED) if(LLVM_DIR) message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") endif() diff --git a/tuplex/codegen/include/CodegenHelper.h b/tuplex/codegen/include/CodegenHelper.h index 8b15e338e..805e34a7a 100644 --- a/tuplex/codegen/include/CodegenHelper.h +++ b/tuplex/codegen/include/CodegenHelper.h @@ -405,25 +405,6 @@ namespace tuplex { return get_or_throw().CreateConstGEP2_64(Ty, Ptr, Idx0, Idx1, Name); } - inline llvm::Value *CreateConstInBoundsGEP2_64(llvm::Value *Ptr, uint64_t Idx0, - uint64_t Idx1, const std::string &Name = "") const { - using namespace llvm; - - // cf. https://github.com/llvm/llvm-project/commit/544fa425c98d60042214bd78ee90abf0a46fa2ff - assert(Ptr->getType()); - llvm::Type *Ty = nullptr; - - // print types - auto ptrType = cast(Ptr->getType()->getScalarType()); - Ty = ptrType->getPointerElementType(); - -#if LLVM_VERSION_MAJOR >= 13 - // match - assert(cast(Ptr->getType()->getScalarType())->isOpaqueOrPointeeTypeMatches(Ty)); -#endif - return CreateConstInBoundsGEP2_64(Ptr, Ty, Idx0, Idx1, Name); - } - inline llvm::Value *CreatePtrToInt(llvm::Value *V, llvm::Type *DestTy, const std::string &Name = "") { return get_or_throw().CreatePtrToInt(V, DestTy, Name); } @@ -508,19 +489,6 @@ namespace tuplex { #endif } - inline llvm::LoadInst *CreateLoad(llvm::Value *Ptr, const std::string& Name ="") const { - throw std::runtime_error("need to replace this call with typed call."); - assert(Ptr->getType()->getPointerElementType()); - return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name); - } - - inline llvm::Value *CreateGEP(llvm::Value *Ptr, llvm::ArrayRef IdxList, - const std::string &Name = "") const { - assert(Ptr->getType()->getScalarType()->getPointerElementType()); - // this is deprecated - return CreateGEP(Ptr->getType()->getScalarType()->getPointerElementType(), - Ptr, IdxList, Name); - } inline llvm::Value* CreateInBoundsGEP(llvm::Value* Ptr, llvm::Type* pointee_type, llvm::Value* Idx) { return get_or_throw().CreateInBoundsGEP(pointee_type, Ptr, {Idx}); @@ -647,15 +615,6 @@ namespace tuplex { #endif } - inline llvm::Value *CreatePtrDiff(llvm::Value *LHS, llvm::Value *RHS, - const std::string &Name = "") const { - assert(LHS->getType() == RHS->getType() && LHS->getType()->isPointerTy()); - llvm::Type *ElemTy = LHS->getType()->getPointerElementType(); - assert(ElemTy); - return CreatePtrDiff(ElemTy, LHS, RHS, Name); - } - - llvm::Value *CreateRetVoid() const { return get_or_throw().CreateRetVoid(); } diff --git a/tuplex/codegen/src/FlattenedTuple.cc b/tuplex/codegen/src/FlattenedTuple.cc index 77f266450..015dd1b15 100644 --- a/tuplex/codegen/src/FlattenedTuple.cc +++ b/tuplex/codegen/src/FlattenedTuple.cc @@ -83,7 +83,6 @@ namespace tuplex { auto field_type = _tree.fieldType(index); if(field_type.isTupleType() && field_type != python::Type::EMPTYTUPLE) { // need to assign a subtree - assert(value->getType()->isStructTy() || value->getType()->getPointerElementType()->isStructTy()); // struct or struct* auto subtree = _tree.subTree(index); auto subtree_type = subtree.tupleType(); diff --git a/tuplex/codegen/src/IteratorContextProxy.cc b/tuplex/codegen/src/IteratorContextProxy.cc index cb372a2ae..30cf6efde 100644 --- a/tuplex/codegen/src/IteratorContextProxy.cc +++ b/tuplex/codegen/src/IteratorContextProxy.cc @@ -922,8 +922,6 @@ namespace tuplex { logger.debug("ft type: " + _env.getLLVMTypeName(ft)); logger.debug("iterator type: " + _env.getLLVMTypeName(iterator->getType())); - // ok, update is something crazy fancy here: mod.getOrInsertFunction(name, FT).getCallee()->getType()->getPointerElementType()->isFunctionTy() - auto nextFunc_value = llvm::getOrInsertCallable(*_env.getModule(), funcName, ft); llvm::FunctionCallee nextFunc_callee(ft, nextFunc_value); auto exhausted = builder.CreateCall(nextFunc_callee, iterator); @@ -965,7 +963,7 @@ namespace tuplex { const std::shared_ptr &iteratorInfo) { using namespace llvm; - llvm::Type *iteratorContextType = createIteratorContextTypeFromIteratorInfo(_env, *iteratorInfo); //iterator->getType()->getPointerElementType(); + llvm::Type *iteratorContextType = createIteratorContextTypeFromIteratorInfo(_env, *iteratorInfo); std::string funcName; auto iteratorName = iteratorInfo->iteratorName; @@ -999,8 +997,6 @@ namespace tuplex { logger.debug("ft type: " + _env.getLLVMTypeName(ft)); logger.debug("iterator type: " + _env.getLLVMTypeName(iterator->getType())); - // ok, update is something crazy fancy here: mod.getOrInsertFunction(name, FT).getCallee()->getType()->getPointerElementType()->isFunctionTy() - auto nextFunc_value = llvm::getOrInsertCallable(*_env.getModule(), funcName, ft); llvm::FunctionCallee nextFunc_callee(ft, nextFunc_value); auto exhausted = builder.CreateCall(nextFunc_callee, iterator); diff --git a/tuplex/codegen/src/LLVMEnvironment.cc b/tuplex/codegen/src/LLVMEnvironment.cc index e0d9fcfe1..ac3336d0f 100644 --- a/tuplex/codegen/src/LLVMEnvironment.cc +++ b/tuplex/codegen/src/LLVMEnvironment.cc @@ -1103,8 +1103,12 @@ namespace tuplex { #if (LLVM_VERSION_MAJOR > 14) if(stype->isOpaquePointerTy()) return "ptr"; -#endif +#elif (LLVM_VERSION_MAJOR >= 17) + return "ptr" +#else stype = stype->getPointerElementType(); +#endif + pointer_stars += "*"; } @@ -1166,9 +1170,12 @@ namespace tuplex { #if (LLVM_VERSION_MAJOR > 14) if(t->isOpaquePointerTy()) return "ptr"; -#endif +#elif (LLVM_VERSION_MAJOR >= 17) + return "ptr"; +#else // recurse: return getLLVMTypeName(t->getPointerElementType()) + "*"; +#endif } if (t->isArrayTy()) { From 28a65a0d9668fc6426bdd3fbac025648e65f969e Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 24 Oct 2023 21:47:59 -0700 Subject: [PATCH 16/31] remove old functions --- tuplex/codegen/src/CodegenHelper.cc | 2 ++ tuplex/core/include/physical/CSVParseRowGenerator.h | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tuplex/codegen/src/CodegenHelper.cc b/tuplex/codegen/src/CodegenHelper.cc index 5c4679692..f4ef2b7b3 100644 --- a/tuplex/codegen/src/CodegenHelper.cc +++ b/tuplex/codegen/src/CodegenHelper.cc @@ -28,7 +28,9 @@ #include #include #include +#if LLVM_VERSION_MAJOR < 17 #include +#endif #include // to iterate over predecessors/successors easily #include #include diff --git a/tuplex/core/include/physical/CSVParseRowGenerator.h b/tuplex/core/include/physical/CSVParseRowGenerator.h index 87460a1e0..e44c530f9 100644 --- a/tuplex/core/include/physical/CSVParseRowGenerator.h +++ b/tuplex/core/include/physical/CSVParseRowGenerator.h @@ -289,13 +289,13 @@ namespace tuplex { builder.CreateICmpEQ(cellEnd, _endPtr)); - auto beforeCellBegin = clampWithStartPtr(builder, builder.CreateGEP(cellBegin, _env->i32Const(-1))); + auto beforeCellBegin = clampWithStartPtr(builder, builder.MovePtrByBytes(cellBegin, -1)); // note that cellEnd is excl. Hence at cellEnd there is the character after the cell end - auto afterCellEnd = clampWithEndPtr(builder, builder.CreateGEP(cellEnd, _env->i32Const(0))); + auto afterCellEnd = clampWithEndPtr(builder, builder.MovePtrByBytes(cellEnd, (int64_t)0)); - auto beforeIsQuote = builder.CreateICmpEQ(builder.CreateLoad(beforeCellBegin), + auto beforeIsQuote = builder.CreateICmpEQ(builder.CreateLoad(builder.getInt8Ty(), beforeCellBegin), _env->i8Const(_quotechar)); - auto afterIsQuote = builder.CreateICmpEQ(builder.CreateLoad(afterCellEnd), _env->i8Const(_quotechar)); + auto afterIsQuote = builder.CreateICmpEQ(builder.CreateLoad(builder.getInt8Ty(), afterCellEnd), _env->i8Const(_quotechar)); auto beforeAndAfterAreQuotes = builder.CreateAnd(beforeIsQuote, afterIsQuote); return builder.CreateSelect(cellAtBoundaries, _env->i1Const(false), beforeAndAfterAreQuotes); From 00d56e009e24e820d30d1775477f472377778689 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 25 Oct 2023 23:23:52 -0700 Subject: [PATCH 17/31] llvm17 fixes, remove old code --- tuplex/codegen/include/IteratorContextProxy.h | 133 +++--- tuplex/codegen/src/BlockGeneratorVisitor.cc | 110 ++--- tuplex/codegen/src/FlattenedTuple.cc | 4 +- tuplex/codegen/src/IteratorContextProxy.cc | 381 +++++++++--------- tuplex/codegen/src/LLVMEnvironment.cc | 2 +- tuplex/core/src/llvm13/JITCompiler_llvm13.cc | 10 +- 6 files changed, 332 insertions(+), 308 deletions(-) diff --git a/tuplex/codegen/include/IteratorContextProxy.h b/tuplex/codegen/include/IteratorContextProxy.h index af44102a3..a87e862e4 100644 --- a/tuplex/codegen/include/IteratorContextProxy.h +++ b/tuplex/codegen/include/IteratorContextProxy.h @@ -127,70 +127,70 @@ namespace tuplex { llvm::Value *iterator, const std::shared_ptr &iteratorInfo); - /*! - * Update index for a zip iterator in preparing for the getIteratorNextElement call by calling updateIteratorIndex on each argument. - * If any argument is exhausted, return true and stop calling updateIteratorIndex on rest of the arguments. - * Only return false if none of the argument iterators is exhausted. - * @param builder - * @param iterator - * @param iteratorInfo - * @return true if iterator is exhausted (getIteratorNextElement should not get called later), otherwise false - */ - llvm::Value *updateZipIndex(const codegen::IRBuilder& builder, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo); - - /*! - * Generate the next element of a zip iterator. - * Only to be called after calling updateIteratorIndex. - * @param builder - * @param yieldType - * @param iterator - * @param iteratorInfo - * @return tuple element of yieldType - */ - SerializableValue getZipNextElement(const codegen::IRBuilder& builder, - const python::Type &yieldType, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo); - - /*! - * Generate the next element of a enumerate iterator. - * Only to be called after calling updateIteratorIndex. - * @param builder - * @param iterator - * @param iteratorInfo - * @return true if iterator is exhausted (getIteratorNextElement should not get called later), otherwise false - */ - llvm::Value *updateEnumerateIndex(const codegen::IRBuilder& builder, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo); - - /*! - * Generate the next element of a enumerate iterator. - * Only to be called after calling updateIteratorIndex. - * @param builder - * @param yieldType - * @param iterator - * @param iteratorInfo - * @return tuple element of yieldType - */ - SerializableValue getEnumerateNextElement(const codegen::IRBuilder& builder, - const python::Type &yieldType, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo); - - /*! - * Increment index field of a list/string/tuple iterator by offset. - * Increment index field of a range iterator by step * offset. - * Decrement index field of any reverseiterator by offset. - * For zip and enumerate, will use recursive calls on their arguments until a list/string/tuple iterator or a reverseiterator is reached. - * @param builder - * @param iterator - * @param iteratorInfo - * @param offset can be negative - */ - void incrementIteratorIndex(const codegen::IRBuilder& builder, llvm::Value *iterator, const std::shared_ptr &iteratorInfo, int offset); +// /*! +// * Update index for a zip iterator in preparing for the getIteratorNextElement call by calling updateIteratorIndex on each argument. +// * If any argument is exhausted, return true and stop calling updateIteratorIndex on rest of the arguments. +// * Only return false if none of the argument iterators is exhausted. +// * @param builder +// * @param iterator +// * @param iteratorInfo +// * @return true if iterator is exhausted (getIteratorNextElement should not get called later), otherwise false +// */ +// llvm::Value *updateZipIndex(const codegen::IRBuilder& builder, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo); +// +// /*! +// * Generate the next element of a zip iterator. +// * Only to be called after calling updateIteratorIndex. +// * @param builder +// * @param yieldType +// * @param iterator +// * @param iteratorInfo +// * @return tuple element of yieldType +// */ +// SerializableValue getZipNextElement(const codegen::IRBuilder& builder, +// const python::Type &yieldType, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo); +// +// /*! +// * Generate the next element of a enumerate iterator. +// * Only to be called after calling updateIteratorIndex. +// * @param builder +// * @param iterator +// * @param iteratorInfo +// * @return true if iterator is exhausted (getIteratorNextElement should not get called later), otherwise false +// */ +// llvm::Value *updateEnumerateIndex(const codegen::IRBuilder& builder, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo); +// +// /*! +// * Generate the next element of a enumerate iterator. +// * Only to be called after calling updateIteratorIndex. +// * @param builder +// * @param yieldType +// * @param iterator +// * @param iteratorInfo +// * @return tuple element of yieldType +// */ +// SerializableValue getEnumerateNextElement(const codegen::IRBuilder& builder, +// const python::Type &yieldType, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo); +// +// /*! +// * Increment index field of a list/string/tuple iterator by offset. +// * Increment index field of a range iterator by step * offset. +// * Decrement index field of any reverseiterator by offset. +// * For zip and enumerate, will use recursive calls on their arguments until a list/string/tuple iterator or a reverseiterator is reached. +// * @param builder +// * @param iterator +// * @param iteratorInfo +// * @param offset can be negative +// */ +// void incrementIteratorIndex(const codegen::IRBuilder& builder, llvm::Value *iterator, const std::shared_ptr &iteratorInfo, int offset); }; /*! @@ -200,6 +200,11 @@ namespace tuplex { * @return corresponding llvm::Type */ extern llvm::Type* createIteratorContextTypeFromIteratorInfo(LLVMEnvironment& env, const IteratorInfo& iteratorInfo); + + extern void increment_iterator_index(LLVMEnvironment& env, const codegen::IRBuilder& builder, + llvm::Value *iterator, + const std::shared_ptr &iteratorInfo, + int32_t offset); } namespace codegen { diff --git a/tuplex/codegen/src/BlockGeneratorVisitor.cc b/tuplex/codegen/src/BlockGeneratorVisitor.cc index 9eef5cd45..25657011f 100644 --- a/tuplex/codegen/src/BlockGeneratorVisitor.cc +++ b/tuplex/codegen/src/BlockGeneratorVisitor.cc @@ -3657,56 +3657,58 @@ namespace tuplex { return ft.getLoad(builder, {idx}); } else { - // THIS HERE IS BACKUP CODE, usable if the AST tree isn't reduced completely. - _logger.warn( - "backup code used for [] operator: Make sure the AST tree is properly reduced in its literal expressions."); - - // ast tree is not completely reduced here, so generate expressions - assert(isStaticValue(index_node, true)); - - FlattenedTuple ft = FlattenedTuple::fromLLVMStructVal(_env, - builder, - value.val, - value_node->getInferredType()); - auto tupleNumElements = value_node->getInferredType().parameters().size(); - - // create temp struct type & use GEP method to retrieve the element. - // go over all the first level elements that are contained - std::vector elements; - std::vector elementTypes; - for (int i = 0; i < tupleNumElements; ++i) { - auto load = ft.getLoad(builder, {i}); - elements.push_back(load); - elementTypes.push_back(load.val->getType()); - } - - // create new struct type to get the i-th element via getelementptr - auto structType = llvm::StructType::create(_env->getContext(), elementTypes, "indextuple"); - // load the values into this struct type - auto alloc = builder.CreateAlloca(structType, 0, nullptr); - for (int i = 0; i < tupleNumElements; ++i) - builder.CreateStore(elements[i].val, builder.CreateGEP(alloc, {i32Const(0), i32Const(i)})); - - - // fetch element - auto lookupPtr = builder.CreateGEP(alloc, {i32Const(0), builder.CreateTrunc(index.val, - Type::getInt32Ty( - _env->getContext()))}); - - // also need to lookup size... - auto salloc = builder.CreateAlloca(llvm::ArrayType::get(_env->i64Type(), tupleNumElements), 0, - nullptr); - // insert elements - for (int i = 0; i < tupleNumElements; ++i) - builder.CreateStore(elements[i].size, - builder.CreateGEP(salloc, {i32Const(0), i32Const(i)})); - - auto retSize = builder.CreateLoad(builder.CreateGEP(salloc, {i32Const(0), - builder.CreateTrunc(index.val, - Type::getInt32Ty( - _env->getContext()))})); - auto retVal = builder.CreateLoad(lookupPtr); - return SerializableValue(retVal, retSize); + throw std::runtime_error("indexing via [] for non homogenous tuple not supported for LLVM17+"); +// // THIS HERE IS BACKUP CODE, usable if the AST tree isn't reduced completely. +// _logger.warn( +// "backup code used for [] operator: Make sure the AST tree is properly reduced in its literal expressions."); +// +// // ast tree is not completely reduced here, so generate expressions +// assert(isStaticValue(index_node, true)); +// +// FlattenedTuple ft = FlattenedTuple::fromLLVMStructVal(_env, +// builder, +// value.val, +// value_node->getInferredType()); +// auto tupleNumElements = value_node->getInferredType().parameters().size(); +// +// // create temp struct type & use GEP method to retrieve the element. +// // go over all the first level elements that are contained +// std::vector elements; +// std::vector elementTypes; +// for (int i = 0; i < tupleNumElements; ++i) { +// auto load = ft.getLoad(builder, {i}); +// elements.push_back(load); +// elementTypes.push_back(load.val->getType()); +// } +// +// // create new struct type to get the i-th element via getelementptr +// auto structType = llvm::StructType::create(_env->getContext(), elementTypes, "indextuple"); +// // load the values into this struct type +// auto alloc = builder.CreateAlloca(structType, 0, nullptr); +// for (int i = 0; i < tupleNumElements; ++i) +// builder.CreateStore(elements[i].val, builder.CreateStructGEP(alloc, structType, i)); +// +// +// // fetch element +// auto lookupPtr = builder.CreateGEP(structType, alloc, {i32Const(0), builder.CreateTrunc(index.val, +// Type::getInt32Ty( +// _env->getContext()))}); +// +// // also need to lookup size... +// auto llvm_array_type = llvm::ArrayType::get(_env->i64Type(), tupleNumElements); +// auto salloc = builder.CreateAlloca(llvm_array_type, 0, +// nullptr); +// // insert elements +// for (int i = 0; i < tupleNumElements; ++i) +// builder.CreateStore(elements[i].size, +// builder.CreateGEP(llvm_array_type, salloc, {i32Const(0), i32Const(i)})); +// +// auto retSize = builder.CreateLoad(builder.getInt64Ty(), builder.CreateGEP(llvm_array_type, salloc, {i32Const(0), +// builder.CreateTrunc(index.val, +// Type::getInt32Ty( +// _env->getContext()))})); +// auto retVal = builder.CreateLoad(lookupPtr); +// return SerializableValue(retVal, retSize); } } @@ -5302,10 +5304,11 @@ namespace tuplex { // empty iterator is always exhausted loopCond = _env->i1Const(false); } else { + auto iterator = exprAlloc.val; // increment iterator index by 1 and check if it is exhausted - auto iteratorExhausted = _iteratorContextProxy->updateIteratorIndex(builder, exprAlloc.val, iteratorInfo); + auto iteratorExhausted = _iteratorContextProxy->updateIteratorIndex(builder, iterator, iteratorInfo); // decrement iterator index by 1 - _iteratorContextProxy->incrementIteratorIndex(builder, exprAlloc.val, iteratorInfo, -1); + increment_iterator_index(*_env, builder, iterator, iteratorInfo, -1); // loopCond = !iteratorExhausted i.e. if iterator exhausted, ends the loop loopCond = builder.CreateICmpEQ(iteratorExhausted, _env->i1Const(false)); } @@ -5334,8 +5337,9 @@ namespace tuplex { // first iteration is guaranteed to exist, or an exception would have been raised earlier _logger.debug("first iteration of for loop unrolled to allow type-stability during loop"); if(exprType.isIteratorType()) { + auto iterator = exprAlloc.val; // increment iterator index by 1 - _iteratorContextProxy->incrementIteratorIndex(builder, exprAlloc.val, iteratorInfo, 1); + increment_iterator_index(*_env, builder, iterator, iteratorInfo, 1); } else { builder.CreateStore(builder.CreateAdd(start, step), currPtr); } diff --git a/tuplex/codegen/src/FlattenedTuple.cc b/tuplex/codegen/src/FlattenedTuple.cc index 015dd1b15..598dcc8c4 100644 --- a/tuplex/codegen/src/FlattenedTuple.cc +++ b/tuplex/codegen/src/FlattenedTuple.cc @@ -1038,10 +1038,10 @@ namespace tuplex { // here it is just a load // ==> an empty tuple can't have a bitmap! if(isEmptyTuple()) { - throw std::runtime_error("need to figure this out..."); // what needs to be stored here anyways?? + throw std::runtime_error("need to figure this out..."); // what needs to be stored here anyways? assert(1 == numElements()); // store size for packed empty tuple - builder.CreateStore(_tree.get(0).size, builder.CreateGEP(ptr, {_env->i32Const(0), _env->i32Const(numElements())})); + builder.CreateStore(_tree.get(0).size, builder.CreateStructGEP(ptr, llvmType, numElements())); return; } diff --git a/tuplex/codegen/src/IteratorContextProxy.cc b/tuplex/codegen/src/IteratorContextProxy.cc index 30cf6efde..c5321e4ed 100644 --- a/tuplex/codegen/src/IteratorContextProxy.cc +++ b/tuplex/codegen/src/IteratorContextProxy.cc @@ -465,193 +465,200 @@ namespace tuplex { return next_from_iterator(*_env, builder, yieldType, iterator, iteratorInfo); } - llvm::Value *IteratorContextProxy::updateZipIndex(const codegen::IRBuilder& builder, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo) { - using namespace llvm; - - auto argsType = iteratorInfo->argsType; - auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; - - int zipSize = argsType.parameters().size(); - if(zipSize == 0) { - return _env->i1Const(true); - } - - BasicBlock *currBB = builder.GetInsertBlock(); - BasicBlock *exhaustedBB = BasicBlock::Create(_env->getContext(), "exhaustedBB", currBB->getParent()); - BasicBlock *endBB = BasicBlock::Create(_env->getContext(), "endBB", currBB->getParent()); - - builder.SetInsertPoint(exhaustedBB); - builder.CreateBr(endBB); - - builder.SetInsertPoint(endBB); - // zipExhausted indicates whether the given zip iterator is exhausted - auto zipExhausted = builder.CreatePHI(_env->i1Type(), 2); - zipExhausted->addIncoming(_env->i1Const(true), exhaustedBB); - - std::vector zipElementEntryBB; - std::vector zipElementCondBB; - for (int i = 0; i < zipSize; ++i) { - BasicBlock *currElementEntryBB = BasicBlock::Create(_env->getContext(), "zipElementBB" + std::to_string(i), currBB->getParent()); - BasicBlock *currElementCondBB = BasicBlock::Create(_env->getContext(), "currCondBB" + std::to_string(i), currBB->getParent()); - zipElementEntryBB.push_back(currElementEntryBB); - zipElementCondBB.push_back(currElementCondBB); - } - zipExhausted->addIncoming(_env->i1Const(false), zipElementCondBB[zipSize - 1]); - - builder.SetInsertPoint(currBB); - builder.CreateBr(zipElementEntryBB[0]); - // iterate over all arg iterators - // if the current arg iterator is exhausted, jump directly to exhaustedBB and zipExhausted will be set to true - for (int i = 0; i < zipSize; ++i) { - builder.SetInsertPoint(zipElementEntryBB[i]); - auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); - auto currIterator = builder.CreateLoad(currIteratorPtr); - auto currIteratorInfo = argsIteratorInfo[i]; - assert(currIteratorInfo); - auto exhausted = updateIteratorIndex(builder, currIterator, currIteratorInfo); - builder.CreateBr(zipElementCondBB[i]); - builder.SetInsertPoint(zipElementCondBB[i]); - if(i == zipSize - 1) { - builder.CreateCondBr(exhausted, exhaustedBB, endBB); - } else { - builder.CreateCondBr(exhausted, exhaustedBB, zipElementEntryBB[i+1]); - } - } - builder.SetInsertPoint(endBB); - - return zipExhausted; - } - - SerializableValue IteratorContextProxy::getZipNextElement(const codegen::IRBuilder& builder, - const python::Type &yieldType, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo) { - using namespace llvm; - auto argsType = iteratorInfo->argsType; - auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; - - FlattenedTuple ft(_env); - ft.init(yieldType); - - // previously UpdateIteratorIndexFunction was called on each arg iterator which increments index of each arg iterator by 1 - // restore index for all arg iterators - incrementIteratorIndex(builder, iterator, iteratorInfo, -1); - for (int i = 0; i < argsType.parameters().size(); ++i) { - auto currIteratorInfo = argsIteratorInfo[i]; - auto llvm_curr_iterator_type = createIteratorContextTypeFromIteratorInfo(*_env, *currIteratorInfo.get()); - auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); - auto currIterator = builder.CreateLoad(llvm_curr_iterator_type->getPointerTo(), currIteratorPtr); - - // update current arg iterator index before fetching value - incrementIteratorIndex(builder, currIterator, currIteratorInfo, 1); - auto currIteratorNextVal = getIteratorNextElement(builder, yieldType.parameters()[i], currIterator, currIteratorInfo); - ft.setElement(builder, i, currIteratorNextVal.val, currIteratorNextVal.size, currIteratorNextVal.is_null); - } - auto retVal = ft.getLoad(builder); - auto retSize = ft.getSize(builder); - return SerializableValue(retVal, retSize); - } - - llvm::Value *IteratorContextProxy::updateEnumerateIndex(const codegen::IRBuilder& builder, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo) { - using namespace llvm; - - auto argIteratorInfo = iteratorInfo->argsIteratorInfo.front(); - auto argIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); - auto argIterator = builder.CreateLoad(argIteratorPtr); - auto enumerateExhausted = updateIteratorIndex(builder, argIterator, argIteratorInfo); - - return enumerateExhausted; - } - - SerializableValue IteratorContextProxy::getEnumerateNextElement(const codegen::IRBuilder& builder, - const python::Type &yieldType, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo) { - using namespace llvm; - - auto argIteratorInfo = iteratorInfo->argsIteratorInfo.front(); - - FlattenedTuple ft(_env); - ft.init(yieldType); - auto startValPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(0)}); - auto startVal = builder.CreateLoad(startValPtr); - auto start = SerializableValue(startVal, _env->i64Const(8)); - auto argIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); - auto argIterator = builder.CreateLoad(argIteratorPtr); - auto val = getIteratorNextElement(builder, yieldType.parameters()[1], argIterator, argIteratorInfo); - ft.setElement(builder, 0, start.val, start.size, start.is_null); - ft.setElement(builder, 1, val.val, val.size, val.is_null); - auto retVal = ft.getLoad(builder); - auto retSize = ft.getSize(builder); - // increment start index value - auto newStartVal = builder.CreateAdd(startVal, _env->i64Const(1)); - builder.CreateStore(newStartVal, startValPtr); - - return SerializableValue(retVal, retSize); - } - - void IteratorContextProxy::incrementIteratorIndex(const codegen::IRBuilder& builder, - llvm::Value *iterator, - const std::shared_ptr &iteratorInfo, - int offset) { - using namespace llvm; - - auto iteratorName = iteratorInfo->iteratorName; - auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; - - if(iteratorName == "zip") { - for (int i = 0; i < argsIteratorInfo.size(); ++i) { - auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); - - // get iterator type - auto llvm_iterator_type = createIteratorContextTypeFromIteratorInfo(*_env, *argsIteratorInfo[i]); - - auto currIterator = builder.CreateLoad(llvm_iterator_type->getPointerTo(), currIteratorPtr); - incrementIteratorIndex(builder, currIterator, argsIteratorInfo[i], offset); - } - return; - } - - if(iteratorName == "enumerate") { - auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); - auto currIterator = builder.CreateLoad(currIteratorPtr); - incrementIteratorIndex(builder, currIterator, argsIteratorInfo.front(), offset); - return; - } - - auto iterablesType = iteratorInfo->argsType; - if(iteratorName == "iter") { - if(iterablesType.isIteratorType()) { - // iter() call on an iterator, ignore the outer iter and call again - assert(argsIteratorInfo.front()); - incrementIteratorIndex(builder, iterator, argsIteratorInfo.front(), offset); - return; - } - } else if(iteratorName == "reversed") { - // for reverseiterator, need to decrement index by offset - offset = -offset; - } else { - throw std::runtime_error("unsupported iterator" + iteratorName); - } - - // change index field - auto indexPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); - auto currIndex = builder.CreateLoad(builder.getInt32Ty(), indexPtr); - if(iterablesType == python::Type::RANGE) { - // index will change by offset * step - auto rangePtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(2)}); - auto range = builder.CreateLoad(rangePtr); - auto stepPtr = builder.CreateGEP(_env->getRangeObjectType(), range, {_env->i32Const(0), _env->i32Const(2)}); - auto step = builder.CreateLoad(stepPtr); - builder.CreateStore(builder.CreateAdd(currIndex, builder.CreateMul(_env->i64Const(offset), step)), indexPtr); - } else { - builder.CreateStore(builder.CreateAdd(currIndex, _env->i32Const(offset)), indexPtr); - } - } +// llvm::Value *IteratorContextProxy::updateZipIndex(const codegen::IRBuilder& builder, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo) { +// // deprecated +// return nullptr; +//// using namespace llvm; +//// +//// auto argsType = iteratorInfo->argsType; +//// auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; +//// +//// int zipSize = argsType.parameters().size(); +//// if(zipSize == 0) { +//// return _env->i1Const(true); +//// } +//// +//// BasicBlock *currBB = builder.GetInsertBlock(); +//// BasicBlock *exhaustedBB = BasicBlock::Create(_env->getContext(), "exhaustedBB", currBB->getParent()); +//// BasicBlock *endBB = BasicBlock::Create(_env->getContext(), "endBB", currBB->getParent()); +//// +//// builder.SetInsertPoint(exhaustedBB); +//// builder.CreateBr(endBB); +//// +//// builder.SetInsertPoint(endBB); +//// // zipExhausted indicates whether the given zip iterator is exhausted +//// auto zipExhausted = builder.CreatePHI(_env->i1Type(), 2); +//// zipExhausted->addIncoming(_env->i1Const(true), exhaustedBB); +//// +//// std::vector zipElementEntryBB; +//// std::vector zipElementCondBB; +//// for (int i = 0; i < zipSize; ++i) { +//// BasicBlock *currElementEntryBB = BasicBlock::Create(_env->getContext(), "zipElementBB" + std::to_string(i), currBB->getParent()); +//// BasicBlock *currElementCondBB = BasicBlock::Create(_env->getContext(), "currCondBB" + std::to_string(i), currBB->getParent()); +//// zipElementEntryBB.push_back(currElementEntryBB); +//// zipElementCondBB.push_back(currElementCondBB); +//// } +//// zipExhausted->addIncoming(_env->i1Const(false), zipElementCondBB[zipSize - 1]); +//// +//// builder.SetInsertPoint(currBB); +//// builder.CreateBr(zipElementEntryBB[0]); +//// // iterate over all arg iterators +//// // if the current arg iterator is exhausted, jump directly to exhaustedBB and zipExhausted will be set to true +//// for (int i = 0; i < zipSize; ++i) { +//// builder.SetInsertPoint(zipElementEntryBB[i]); +//// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); +//// auto currIterator = builder.CreateLoad(currIteratorPtr); +//// auto currIteratorInfo = argsIteratorInfo[i]; +//// assert(currIteratorInfo); +//// auto exhausted = updateIteratorIndex(builder, currIterator, currIteratorInfo); +//// builder.CreateBr(zipElementCondBB[i]); +//// builder.SetInsertPoint(zipElementCondBB[i]); +//// if(i == zipSize - 1) { +//// builder.CreateCondBr(exhausted, exhaustedBB, endBB); +//// } else { +//// builder.CreateCondBr(exhausted, exhaustedBB, zipElementEntryBB[i+1]); +//// } +//// } +//// builder.SetInsertPoint(endBB); +//// +//// return zipExhausted; +// } +// +// SerializableValue IteratorContextProxy::getZipNextElement(const codegen::IRBuilder& builder, +// const python::Type &yieldType, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo) { +// +// // deprecated +// return {}; +//// using namespace llvm; +//// auto argsType = iteratorInfo->argsType; +//// auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; +//// +//// FlattenedTuple ft(_env); +//// ft.init(yieldType); +//// +//// // previously UpdateIteratorIndexFunction was called on each arg iterator which increments index of each arg iterator by 1 +//// // restore index for all arg iterators +//// incrementIteratorIndex(builder, iterator, iteratorInfo, -1); +//// for (int i = 0; i < argsType.parameters().size(); ++i) { +//// auto currIteratorInfo = argsIteratorInfo[i]; +//// auto llvm_curr_iterator_type = createIteratorContextTypeFromIteratorInfo(*_env, *currIteratorInfo.get()); +//// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); +//// auto currIterator = builder.CreateLoad(llvm_curr_iterator_type->getPointerTo(), currIteratorPtr); +//// +//// // update current arg iterator index before fetching value +//// incrementIteratorIndex(builder, currIterator, currIteratorInfo, 1); +//// auto currIteratorNextVal = getIteratorNextElement(builder, yieldType.parameters()[i], currIterator, currIteratorInfo); +//// ft.setElement(builder, i, currIteratorNextVal.val, currIteratorNextVal.size, currIteratorNextVal.is_null); +//// } +//// auto retVal = ft.getLoad(builder); +//// auto retSize = ft.getSize(builder); +//// return SerializableValue(retVal, retSize); +// } +// +// llvm::Value *IteratorContextProxy::updateEnumerateIndex(const codegen::IRBuilder& builder, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo) { +// using namespace llvm; +// +// auto argIteratorInfo = iteratorInfo->argsIteratorInfo.front(); +// auto argIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); +// auto argIterator = builder.CreateLoad(argIteratorPtr); +// auto enumerateExhausted = updateIteratorIndex(builder, argIterator, argIteratorInfo); +// +// return enumerateExhausted; +// } +// +// SerializableValue IteratorContextProxy::getEnumerateNextElement(const codegen::IRBuilder& builder, +// const python::Type &yieldType, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo) { +// // deprecated +// return nullptr; +//// using namespace llvm; +//// +//// auto argIteratorInfo = iteratorInfo->argsIteratorInfo.front(); +//// +//// FlattenedTuple ft(_env); +//// ft.init(yieldType); +//// auto startValPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(0)}); +//// auto startVal = builder.CreateLoad(startValPtr); +//// auto start = SerializableValue(startVal, _env->i64Const(8)); +//// auto argIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); +//// auto argIterator = builder.CreateLoad(argIteratorPtr); +//// auto val = getIteratorNextElement(builder, yieldType.parameters()[1], argIterator, argIteratorInfo); +//// ft.setElement(builder, 0, start.val, start.size, start.is_null); +//// ft.setElement(builder, 1, val.val, val.size, val.is_null); +//// auto retVal = ft.getLoad(builder); +//// auto retSize = ft.getSize(builder); +//// // increment start index value +//// auto newStartVal = builder.CreateAdd(startVal, _env->i64Const(1)); +//// builder.CreateStore(newStartVal, startValPtr); +//// +//// return SerializableValue(retVal, retSize); +// } + +// void IteratorContextProxy::incrementIteratorIndex(const codegen::IRBuilder& builder, +// llvm::Value *iterator, +// const std::shared_ptr &iteratorInfo, +// int offset) { +// using namespace llvm; +// +// auto iteratorName = iteratorInfo->iteratorName; +// auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; +// +// if(iteratorName == "zip") { +// for (int i = 0; i < argsIteratorInfo.size(); ++i) { +// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); +// +// // get iterator type +// auto llvm_iterator_type = createIteratorContextTypeFromIteratorInfo(*_env, *argsIteratorInfo[i]); +// +// auto currIterator = builder.CreateLoad(llvm_iterator_type->getPointerTo(), currIteratorPtr); +// incrementIteratorIndex(builder, currIterator, argsIteratorInfo[i], offset); +// } +// return; +// } +// +// if(iteratorName == "enumerate") { +// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); +// auto currIterator = builder.CreateLoad(currIteratorPtr); +// incrementIteratorIndex(builder, currIterator, argsIteratorInfo.front(), offset); +// return; +// } +// +// auto iterablesType = iteratorInfo->argsType; +// if(iteratorName == "iter") { +// if(iterablesType.isIteratorType()) { +// // iter() call on an iterator, ignore the outer iter and call again +// assert(argsIteratorInfo.front()); +// incrementIteratorIndex(builder, iterator, argsIteratorInfo.front(), offset); +// return; +// } +// } else if(iteratorName == "reversed") { +// // for reverseiterator, need to decrement index by offset +// offset = -offset; +// } else { +// throw std::runtime_error("unsupported iterator" + iteratorName); +// } +// +// // change index field +// auto indexPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); +// auto currIndex = builder.CreateLoad(builder.getInt32Ty(), indexPtr); +// if(iterablesType == python::Type::RANGE) { +// // index will change by offset * step +// auto rangePtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(2)}); +// auto range = builder.CreateLoad(rangePtr); +// auto stepPtr = builder.CreateGEP(_env->getRangeObjectType(), range, {_env->i32Const(0), _env->i32Const(2)}); +// auto step = builder.CreateLoad(stepPtr); +// builder.CreateStore(builder.CreateAdd(currIndex, builder.CreateMul(_env->i64Const(offset), step)), indexPtr); +// } else { +// builder.CreateStore(builder.CreateAdd(currIndex, _env->i32Const(offset)), indexPtr); +// } +// } // helper to retrieve iteratorcontexttype from iteratorInfo llvm::Type* createIteratorContextTypeFromIteratorInfo(LLVMEnvironment& env, const IteratorInfo& iteratorInfo) { diff --git a/tuplex/codegen/src/LLVMEnvironment.cc b/tuplex/codegen/src/LLVMEnvironment.cc index ac3336d0f..9e11b7827 100644 --- a/tuplex/codegen/src/LLVMEnvironment.cc +++ b/tuplex/codegen/src/LLVMEnvironment.cc @@ -1669,7 +1669,7 @@ namespace tuplex { auto str_size = CreateFirstBlockAlloca(builder, i64Type()); auto str = builder.CreateCall(func, {value, str_size}); - return SerializableValue(str, builder.CreateLoad(str_size)); + return SerializableValue(str, builder.CreateLoad(builder.getInt64Ty(), str_size)); } diff --git a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc index feca7dabd..f2d2d7661 100644 --- a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc +++ b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc @@ -252,8 +252,16 @@ namespace tuplex { jitlib.addGenerator(std::move(*ProcessSymbolsGenerator)); // define symbols from custom symbols for this jitlib - for(auto keyval: _customSymbols) + for(auto keyval: _customSymbols) { +#if LLVM_VERSION_MAJOR <= 16 auto rc = jitlib.define(absoluteSymbols({{Mangle(keyval.first), keyval.second}})); +#else + auto rc = jitlib.define(absoluteSymbols(SymbolMap({ + { Mangle(keyval.first), + { ExecutorAddr::fromPtr(&keyval.second), JITSymbolFlags() } } + }); +#endif + } _dylibs.push_back(&jitlib); // save reference for search auto err = _lljit->addIRModule(jitlib, std::move(tsm.get())); From 767b3110e13c6d3426ac34b91155a720cd191949 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 25 Oct 2023 23:44:23 -0700 Subject: [PATCH 18/31] fixes --- .../core/include/llvm13/JITCompiler_llvm13.h | 1 + tuplex/core/src/llvm13/JITCompiler_llvm13.cc | 32 +++++++++++-------- .../core/src/physical/CSVParserGenerator.cc | 12 +++---- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/tuplex/core/include/llvm13/JITCompiler_llvm13.h b/tuplex/core/include/llvm13/JITCompiler_llvm13.h index c02996dd1..bb6f33e7a 100644 --- a/tuplex/core/include/llvm13/JITCompiler_llvm13.h +++ b/tuplex/core/include/llvm13/JITCompiler_llvm13.h @@ -77,6 +77,7 @@ namespace tuplex { // custom symbols std::unordered_map _customSymbols; + void defineCustomSymbols(llvm::orc::JITDylib &jitlib); }; } #endif diff --git a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc index f2d2d7661..685943fe2 100644 --- a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc +++ b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc @@ -252,16 +252,7 @@ namespace tuplex { jitlib.addGenerator(std::move(*ProcessSymbolsGenerator)); // define symbols from custom symbols for this jitlib - for(auto keyval: _customSymbols) { -#if LLVM_VERSION_MAJOR <= 16 - auto rc = jitlib.define(absoluteSymbols({{Mangle(keyval.first), keyval.second}})); -#else - auto rc = jitlib.define(absoluteSymbols(SymbolMap({ - { Mangle(keyval.first), - { ExecutorAddr::fromPtr(&keyval.second), JITSymbolFlags() } } - }); -#endif - } + defineCustomSymbols(jitlib); _dylibs.push_back(&jitlib); // save reference for search auto err = _lljit->addIRModule(jitlib, std::move(tsm.get())); @@ -285,6 +276,23 @@ namespace tuplex { return true; } + void JITCompiler::defineCustomSymbols(llvm::orc::JITDylib &jitlib) { + auto& ES = _lljit->getExecutionSession(); + const auto& DL = _lljit->getDataLayout(); + llvm::orc::MangleAndInterner Mangle(ES, DL); + + for(auto keyval: _customSymbols) { +#if LLVM_VERSION_MAJOR <= 16 + auto rc = jitlib.define(absoluteSymbols({{Mangle(keyval.first), keyval.second}})); +#else + auto rc = jitlib.define(llvm::orc::absoluteSymbols(llvm::orc::SymbolMap({ + { Mangle(keyval.first), + {llvm::orc::ExecutorAddr::fromPtr(&keyval.second), llvm::JITSymbolFlags() } } + }))); +#endif + } + } + bool JITCompiler::compile(std::unique_ptr mod) { llvm::Expected tsm = llvm::orc::ThreadSafeModule(std::move(mod), std::make_unique()); if(!tsm) { @@ -328,9 +336,7 @@ namespace tuplex { jitlib.addGenerator(std::move(*ProcessSymbolsGenerator)); // define symbols from custom symbols for this jitlib - for(auto keyval: _customSymbols) - auto rc = jitlib.define(llvm::orc::absoluteSymbols({{Mangle(keyval.first), keyval.second}})); - + defineCustomSymbols(jitlib); _dylibs.push_back(&jitlib); // save reference for search assert(tsm); diff --git a/tuplex/core/src/physical/CSVParserGenerator.cc b/tuplex/core/src/physical/CSVParserGenerator.cc index b06db5710..185c28d1e 100644 --- a/tuplex/core/src/physical/CSVParserGenerator.cc +++ b/tuplex/core/src/physical/CSVParserGenerator.cc @@ -43,7 +43,7 @@ namespace tuplex { // create some preliminary things - auto endPtr = oldBuilder.CreateGEP(getInputPtrArg(), getInputSizeArg()); + auto endPtr = oldBuilder.MovePtrByBytes(getInputPtrArg(), getInputSizeArg()); oldBuilder.CreateBr(bBody); @@ -59,11 +59,11 @@ namespace tuplex { // if skipHeader is true, skip first row // !!! there is no header validation/order etc. here. if(_skipHeader) { - auto parseCode = builder.CreateCall(parseRowF, {_resStructVar, builder.CreateLoad(_currentPtrVar), _endPtr}); - auto numParsedBytes = builder.CreateLoad(builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(0)})); + auto parseCode = builder.CreateCall(parseRowF, {_resStructVar, builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), _endPtr}); + auto numParsedBytes = builder.CreateLoad(builder.getInt64Ty(), builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(0)})); // inc ptr & go to loop cond - builder.CreateStore(builder.CreateGEP(builder.CreateLoad(_currentPtrVar), numParsedBytes), _currentPtrVar); + builder.CreateStore(builder.CreateGEP(builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), numParsedBytes), _currentPtrVar); } builder.CreateBr(bLoopCond); @@ -71,7 +71,7 @@ namespace tuplex { // loop condition, i.e. p < endp builder.SetInsertPoint(bLoopCond); - auto cond = builder.CreateICmpULT(builder.CreatePtrToInt(builder.CreateLoad(_currentPtrVar), _env->i64Type()), + auto cond = builder.CreateICmpULT(builder.CreatePtrToInt(builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), _env->i64Type()), builder.CreatePtrToInt(_endPtr, _env->i64Type())); builder.CreateCondBr(cond, bLoopBody, bLoopDone); @@ -80,7 +80,7 @@ namespace tuplex { builder.SetInsertPoint(bLoopBody); //call func and advance ptr - auto parseCode = builder.CreateCall(parseRowF, {_resStructVar, builder.CreateLoad(_currentPtrVar), _endPtr}); + auto parseCode = builder.CreateCall(parseRowF, {_resStructVar, builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), _endPtr}); _env->debugPrint(builder, "parseCode is ", parseCode); auto numParsedBytes = builder.CreateLoad(builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(0)})); From 19662ab5d5a45bf8416cac247bea469305c35e3c Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 26 Oct 2023 20:29:18 -0700 Subject: [PATCH 19/31] more API fixes for llvm17 --- tuplex/adapters/cpython/src/PythonHelpers.cc | 3 --- .../core/src/physical/CSVParserGenerator.cc | 24 ++++++++++++------- .../physical/IExceptionableTaskGenerator.cc | 4 ++-- tuplex/core/src/physical/PipelineBuilder.cc | 2 +- tuplex/python/tuplex/context.py | 3 --- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/tuplex/adapters/cpython/src/PythonHelpers.cc b/tuplex/adapters/cpython/src/PythonHelpers.cc index 46b3cd0d1..7f37dd615 100644 --- a/tuplex/adapters/cpython/src/PythonHelpers.cc +++ b/tuplex/adapters/cpython/src/PythonHelpers.cc @@ -178,9 +178,6 @@ namespace python { // PyRun_SimpleString("gc.set_debug(gc.DEBUG_LEAK)"); // PyRun_SimpleString("gc.disable()"); - PyRun_SimpleString("import pickle"); - PyRun_SimpleString("import cloudpickle; print(cloudpickle.__version__)"); - // import cloudpickle for serialized functions PyObject *cloudpickleModule = PyImport_ImportModule("cloudpickle"); diff --git a/tuplex/core/src/physical/CSVParserGenerator.cc b/tuplex/core/src/physical/CSVParserGenerator.cc index 185c28d1e..2f61a3270 100644 --- a/tuplex/core/src/physical/CSVParserGenerator.cc +++ b/tuplex/core/src/physical/CSVParserGenerator.cc @@ -60,10 +60,11 @@ namespace tuplex { // !!! there is no header validation/order etc. here. if(_skipHeader) { auto parseCode = builder.CreateCall(parseRowF, {_resStructVar, builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), _endPtr}); - auto numParsedBytes = builder.CreateLoad(builder.getInt64Ty(), builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(0)})); + auto llvm_ret_type = _rowGenerator.resultType(); + auto numParsedBytes = builder.CreateLoad(builder.getInt64Ty(), builder.CreateStructGEP(_resStructVar, llvm_ret_type, 0)); // inc ptr & go to loop cond - builder.CreateStore(builder.CreateGEP(builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), numParsedBytes), _currentPtrVar); + builder.CreateStore(builder.MovePtrByBytes(builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), numParsedBytes), _currentPtrVar); } builder.CreateBr(bLoopCond); @@ -82,10 +83,11 @@ namespace tuplex { auto parseCode = builder.CreateCall(parseRowF, {_resStructVar, builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), _endPtr}); _env->debugPrint(builder, "parseCode is ", parseCode); - auto numParsedBytes = builder.CreateLoad(builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(0)})); + auto llvm_ret_type = _rowGenerator.resultType(); + auto numParsedBytes = builder.CreateLoad(builder.getInt64Ty(), builder.CreateStructGEP(_resStructVar, llvm_ret_type, 0)); // inc ptr & go to loop cond with next blocks - builder.CreateStore(builder.CreateGEP(builder.CreateLoad(_currentPtrVar), numParsedBytes), _currentPtrVar); + builder.CreateStore(builder.MovePtrByBytes(builder.CreateLoad(_env->i8ptrType(), _currentPtrVar), numParsedBytes), _currentPtrVar); // ignore empty results at end // maybe add assert that lineEnd is >= endPtr @@ -95,8 +97,8 @@ namespace tuplex { builder.SetInsertPoint(bNonEmpty); // can only stuff if bytes were parsed! - auto lineStart = builder.CreateLoad(builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(1)})); - auto lineEnd = builder.CreateLoad(builder.CreateGEP(_resStructVar, {_env->i32Const(0), _env->i32Const(2)})); + auto lineStart = builder.CreateLoad(_env->i8ptrType(), builder.CreateStructGEP(_resStructVar, llvm_ret_type, 1)); + auto lineEnd = builder.CreateLoad(_env->i8ptrType(), builder.CreateStructGEP(_resStructVar, llvm_ret_type, 2)); // check result code, if zero all ok. Else, go into exception handling BasicBlock *bNoException = BasicBlock::Create(context, "no_exception", getFunction()); @@ -160,8 +162,14 @@ namespace tuplex { #warning "this here is outdated... should not be used. Remove code" for(const auto& t : stype.parameters()) { - Value* val = builder.CreateLoad(builder.CreateGEP(resStructVal, {_env->i32Const(0), _env->i32Const(3 + 2 * pos)})); - Value* size = builder.CreateLoad(builder.CreateGEP(resStructVal, {_env->i32Const(0), _env->i32Const(3 + 2 * pos + 1)})); + auto llvm_ret_type = _rowGenerator.resultType(); + auto val_position = 3 + 2 * pos; + auto size_position = 3 + 2 * pos + 1; + auto val_ptr = builder.CreateStructGEP(resStructVal, llvm_ret_type, val_position); + auto size_ptr = builder.CreateStructGEP(resStructVal, llvm_ret_type, size_position); + Value* val = builder.CreateLoad(llvm_ret_type->getStructElementType(val_position), val_ptr); + assert(llvm_ret_type->getStructElementType(size_position) == builder.getInt64Ty()); + Value* size = builder.CreateLoad(builder.getInt64Ty(), size_ptr); // !!! zero terminated string if(python::Type::STRING == t) diff --git a/tuplex/core/src/physical/IExceptionableTaskGenerator.cc b/tuplex/core/src/physical/IExceptionableTaskGenerator.cc index 7ff24b7f2..23d94694f 100644 --- a/tuplex/core/src/physical/IExceptionableTaskGenerator.cc +++ b/tuplex/core/src/physical/IExceptionableTaskGenerator.cc @@ -121,8 +121,8 @@ namespace tuplex { // adjust inputptr (has been already updated) to previous row uwsing inputlength - auto inputptr = builder.CreateGEP(getVariable(builder, "currentInputPtr"), - builder.CreateNeg(inputlength));//builder.CreateLoad(_currentInputPtrVar); + auto inputptr = builder.MovePtrByBytes(getVariable(builder, "currentInputPtr"), + builder.CreateNeg(inputlength)); std::vector eh_parameters{_parameters["userData"], ehcode, ehopid, row, inputptr, inputlength}; builder.CreateCall(eh_func, eh_parameters); diff --git a/tuplex/core/src/physical/PipelineBuilder.cc b/tuplex/core/src/physical/PipelineBuilder.cc index c9fee174f..474c374b1 100644 --- a/tuplex/core/src/physical/PipelineBuilder.cc +++ b/tuplex/core/src/physical/PipelineBuilder.cc @@ -1418,7 +1418,7 @@ namespace tuplex { auto func = quoteForCSV_prototype(env.getContext(), env.getModule().get()); val = builder.CreateCall(func, {val, size, quotedSize, env.i8Const(','), env.i8Const('"')}); fmtString += "%s"; - fmtSize = builder.CreateAdd(fmtSize, builder.CreateLoad(quotedSize)); + fmtSize = builder.CreateAdd(fmtSize, builder.CreateLoad(env.i64Type(), quotedSize)); } else if(type.isOptionType()) { // check element type & call string conversion function with convert diff --git a/tuplex/python/tuplex/context.py b/tuplex/python/tuplex/context.py index 4267a2b1a..ceb2b6538 100644 --- a/tuplex/python/tuplex/context.py +++ b/tuplex/python/tuplex/context.py @@ -191,8 +191,6 @@ def __init__(self, conf=None, name="", **kwargs): ensure_webui(options) # last arg are the options as json string serialized b.c. of boost python problems - logging.debug('Creating C++ context object') - # because webui=False/True is convenient, pass it as well to tuplex options if 'tuplex.webui' in options.keys(): options['tuplex.webui.enable'] = options['tuplex.webui'] @@ -202,7 +200,6 @@ def __init__(self, conf=None, name="", **kwargs): del options['webui'] self._context = _Context(name, runtime_path, json.dumps(options)) - logging.debug('C++ object created.') python_metrics = self._context.getMetrics() assert python_metrics, 'internal error: metrics object should be valid' self.metrics = Metrics(python_metrics) From 13ca03a76ba716a266b59f1d86befea194988d3a Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 26 Oct 2023 20:51:35 -0700 Subject: [PATCH 20/31] more fixing --- .../physical/IExceptionableTaskGenerator.h | 2 +- .../physical/ExceptionSourceTaskBuilder.cc | 2 +- tuplex/core/src/physical/HashJoinStage.cc | 4 +- .../physical/IExceptionableTaskGenerator.cc | 17 ++-- tuplex/core/src/physical/LLVMOptimizer.cc | 89 ++++++++++--------- tuplex/core/src/physical/PipelineBuilder.cc | 6 +- .../src/physical/TuplexSourceTaskBuilder.cc | 2 +- 7 files changed, 63 insertions(+), 59 deletions(-) diff --git a/tuplex/core/include/physical/IExceptionableTaskGenerator.h b/tuplex/core/include/physical/IExceptionableTaskGenerator.h index 2a885f934..3ba318e85 100644 --- a/tuplex/core/include/physical/IExceptionableTaskGenerator.h +++ b/tuplex/core/include/physical/IExceptionableTaskGenerator.h @@ -159,7 +159,7 @@ namespace tuplex { std::map _parameters; // helper functions to use variables via alloc/store in code - std::map _variables; + std::map> _variables; void addVariable(IRBuilder& builder, const std::string name, llvm::Type* type, llvm::Value* initialValue=nullptr); llvm::Value* getVariable(IRBuilder& builder, const std::string name); llvm::Value* getPointerToVariable(IRBuilder& builder, const std::string name); diff --git a/tuplex/core/src/physical/ExceptionSourceTaskBuilder.cc b/tuplex/core/src/physical/ExceptionSourceTaskBuilder.cc index c5f8b575b..1185685cd 100644 --- a/tuplex/core/src/physical/ExceptionSourceTaskBuilder.cc +++ b/tuplex/core/src/physical/ExceptionSourceTaskBuilder.cc @@ -37,7 +37,7 @@ namespace tuplex { callProcessFuncWithHandler(builder, userData, tuple, normalRowCountVar, badRowCountVar, rowNumberVar, inputRowPtr, inputRowSize, terminateEarlyOnLimitCode, processRowFunc); } else { - Value *normalRowCount = builder.CreateLoad(normalRowCountVar, "normalRowCount"); + Value *normalRowCount = builder.CreateLoad(builder.getInt64Ty(), normalRowCountVar, "normalRowCount"); builder.CreateStore(builder.CreateAdd(normalRowCount, env().i64Const(1)), normalRowCountVar); } } diff --git a/tuplex/core/src/physical/HashJoinStage.cc b/tuplex/core/src/physical/HashJoinStage.cc index 0119fac71..8f536c238 100644 --- a/tuplex/core/src/physical/HashJoinStage.cc +++ b/tuplex/core/src/physical/HashJoinStage.cc @@ -69,10 +69,10 @@ namespace tuplex { builder.CreateStore(env->i8nullptr(), hashed_value); // read num rows - Value *numRows = builder.CreateLoad(builder.CreatePointerCast(builder.CreateLoad(curPtrVar), env->i64ptrType()), + Value *numRows = builder.CreateLoad(builder.getInt64Ty(), builder.CreatePointerCast(builder.CreateLoad(env->i8ptrType(), curPtrVar), env->i64ptrType()), "numInputRows"); // move ptr by int64_t - builder.CreateStore(builder.CreateGEP(builder.CreateLoad(curPtrVar), env->i64Const(sizeof(int64_t))), + builder.CreateStore(builder.MovePtrByBytes(builder.CreateLoad(env->i8ptrType(), curPtrVar), sizeof(int64_t)), curPtrVar); // set up diff --git a/tuplex/core/src/physical/IExceptionableTaskGenerator.cc b/tuplex/core/src/physical/IExceptionableTaskGenerator.cc index 23d94694f..bf785be61 100644 --- a/tuplex/core/src/physical/IExceptionableTaskGenerator.cc +++ b/tuplex/core/src/physical/IExceptionableTaskGenerator.cc @@ -142,26 +142,27 @@ namespace tuplex { void IExceptionableTaskGenerator::addVariable(IRBuilder &builder, const std::string name, llvm::Type *type, llvm::Value *initialValue) { - _variables[name] = builder.CreateAlloca(type, 0, nullptr, name); + _variables[name] = std::make_pair(type, builder.CreateAlloca(type, 0, nullptr, name)); if(initialValue) - builder.CreateStore(initialValue, _variables[name]); + assignToVariable(builder, name, initialValue); } llvm::Value* IExceptionableTaskGenerator::getVariable(IRBuilder &builder, const std::string name) { assert(_variables.find(name) != _variables.end()); - return builder.CreateLoad(_variables[name]); + return builder.CreateLoad(_variables[name].first, _variables[name].second); } llvm::Value* IExceptionableTaskGenerator::getPointerToVariable(IRBuilder &builder, const std::string name) { assert(_variables.find(name) != _variables.end()); - return _variables[name]; + return _variables[name].second; } void IExceptionableTaskGenerator::assignToVariable(IRBuilder &builder, const std::string name, llvm::Value *newValue) { assert(_variables.find(name) != _variables.end()); - builder.CreateStore(newValue, _variables[name]); + assert(newValue->getType() == _variables[name].first); + builder.CreateStore(newValue, _variables[name].second); } void IExceptionableTaskGenerator::linkBlocks() { @@ -235,7 +236,7 @@ namespace tuplex { // store back some variables in then block & make sure to mark as last block! // add to variable how much was serialized - auto newoutput = builder.CreateGEP(output, serializedRowSize); + auto newoutput = builder.MovePtrByBytes(output, serializedRowSize); assignToVariable(builder, "outputPtr", newoutput); auto newcapacity = builder.CreateSub(capacity, serializedRowSize); assignToVariable(builder, "outputCapacityLeft", newcapacity); @@ -244,7 +245,7 @@ namespace tuplex { // inc how many rows are written numRowsPtr = getVariable(builder, "outputBasePtr"); - auto curRows = builder.CreateLoad(numRowsPtr); + auto curRows = builder.CreateLoad(builder.getInt64Ty(), numRowsPtr); builder.CreateStore(builder.CreateAdd(curRows, _env->i64Const(1)), numRowsPtr); // inc how many writtes are written @@ -282,7 +283,7 @@ namespace tuplex { auto output_ptr = builder.CreateCall(func, parameters, "output_ptr"); // first save back to variables the memory request incl. 8 byte offset for number of rows! assignToVariable(builder, "outputBasePtr", builder.CreatePointerCast(output_ptr, _env->i64Type()->getPointerTo(0))); - assignToVariable(builder, "outputPtr", builder.CreateGEP(output_ptr, _env->i32Const(sizeof(int64_t)))); + assignToVariable(builder, "outputPtr", builder.MovePtrByBytes(output_ptr, sizeof(int64_t))); // check for null. If so (i.e. no memory returned), if so exit task function immediately // --> also if capacity returned is less than minRequested. diff --git a/tuplex/core/src/physical/LLVMOptimizer.cc b/tuplex/core/src/physical/LLVMOptimizer.cc index ee63adfa0..915d1f180 100644 --- a/tuplex/core/src/physical/LLVMOptimizer.cc +++ b/tuplex/core/src/physical/LLVMOptimizer.cc @@ -10,54 +10,57 @@ #include -#include "llvm/ADT/Triple.h" -#include "llvm/Analysis/CallGraph.h" -#include "llvm/Analysis/CallGraphSCCPass.h" -#include "llvm/Analysis/LoopPass.h" -#include "llvm/Analysis/RegionPass.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/Bitcode/BitcodeWriterPass.h" -#include "llvm/CodeGen/TargetPassConfig.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/IRPrintingPasses.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/LegacyPassManager.h" -#include "llvm/IR/LegacyPassNameParser.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Verifier.h" -#include "llvm/IRReader/IRReader.h" -#include "llvm/InitializePasses.h" -#include "llvm/LinkAllIR.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/MC/SubtargetFeature.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/PluginLoader.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/SystemUtils.h" - -#if LLVM_VERSION_MAJOR < 14 -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if (LLVM_VERSION_MAJOR < 17) +#include +#include +#include #else +#include +#include +#endif +#if (LLVM_VERSION_MAJOR < 14) +#include +#else #include - #endif -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/ToolOutputFile.h" -#include "llvm/Support/YAMLTraits.h" -#include "llvm/Target/TargetMachine.h" -// #include "llvm/Transforms/Coroutines.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" -#include "llvm/Transforms/Utils/Cloning.h" +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/tuplex/core/src/physical/PipelineBuilder.cc b/tuplex/core/src/physical/PipelineBuilder.cc index 474c374b1..c49b9933e 100644 --- a/tuplex/core/src/physical/PipelineBuilder.cc +++ b/tuplex/core/src/physical/PipelineBuilder.cc @@ -1510,7 +1510,7 @@ namespace tuplex { auto snprintf_func = snprintf_prototype(env.getContext(), env.getModule().get()); //{csvRow, fmtSize, env().strConst(builder, fmtString), ...} - args[0] = builder.CreateLoad(bufVar); args[1] = fmtSize; args[2] = env.strConst(builder, fmtString); + args[0] = builder.CreateLoad(env.i8ptrType(), bufVar); args[1] = fmtSize; args[2] = env.strConst(builder, fmtString); auto charsRequired = builder.CreateCall(snprintf_func, args); auto sizeWritten = builder.CreateAdd(builder.CreateZExt(charsRequired, env.i64Type()), env.i64Const(1)); @@ -1525,7 +1525,7 @@ namespace tuplex { // realloc with sizeWritten // store new malloc in bufVar builder.CreateStore(env.malloc(builder, sizeWritten), bufVar); - args[0] = builder.CreateLoad(bufVar); + args[0] = builder.CreateLoad(env.i8ptrType(), bufVar); args[1] = sizeWritten; builder.CreateCall(snprintf_func, args); @@ -1539,7 +1539,7 @@ namespace tuplex { // then, call writeRow - auto buf = builder.CreateLoad(bufVar); + auto buf = builder.CreateLoad(env.i8ptrType(), bufVar); // use string length instead of size, because else writer will copy '\0' too! auto length = builder.CreateSub(sizeWritten, env.i64Const(1)); diff --git a/tuplex/core/src/physical/TuplexSourceTaskBuilder.cc b/tuplex/core/src/physical/TuplexSourceTaskBuilder.cc index bde539d30..b84e7773e 100644 --- a/tuplex/core/src/physical/TuplexSourceTaskBuilder.cc +++ b/tuplex/core/src/physical/TuplexSourceTaskBuilder.cc @@ -37,7 +37,7 @@ namespace tuplex { callProcessFuncWithHandler(builder, userData, tuple, normalRowCountVar, rowNumberVar, inputRowPtr, inputRowSize, terminateEarlyOnLimitCode, processRowFunc); } else { - Value *normalRowCount = builder.CreateLoad(normalRowCountVar, "normalRowCount"); + Value *normalRowCount = builder.CreateLoad(builder.getInt64Ty(), normalRowCountVar, "normalRowCount"); builder.CreateStore(builder.CreateAdd(normalRowCount, env().i64Const(1)), normalRowCountVar); } } From 273b1aebc980670a062f78a62bc13cec8616602f Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 26 Oct 2023 21:11:04 -0700 Subject: [PATCH 21/31] compile now works, need to link with llvm libs --- tuplex/core/src/physical/HashJoinStage.cc | 30 +++++++++++------------ tuplex/test/core/UseCaseFunctionsTest.cc | 2 ++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tuplex/core/src/physical/HashJoinStage.cc b/tuplex/core/src/physical/HashJoinStage.cc index 8f536c238..eb5e61ec9 100644 --- a/tuplex/core/src/physical/HashJoinStage.cc +++ b/tuplex/core/src/physical/HashJoinStage.cc @@ -84,7 +84,7 @@ namespace tuplex { // loop cond counter < numRows builder.SetInsertPoint(bbLoopCondition); - auto cond = builder.CreateICmpSLT(builder.CreateLoad(rowCounterVar), numRows); + auto cond = builder.CreateICmpSLT(builder.CreateLoad(builder.getInt64Ty(), rowCounterVar), numRows); builder.CreateCondBr(cond, bbLoopBody, bbLoopExit); @@ -94,9 +94,9 @@ namespace tuplex { generateProbingCode(env, builder, argMap["userData"], argMap["hmap"], curPtrVar, hashed_value, rightType(), rightKeyIndex(), leftType(), leftKeyIndex(), _joinType); - auto row_number = builder.CreateLoad(rowCounterVar); + auto row_number = builder.CreateLoad(builder.getInt64Ty(), rowCounterVar); //env->debugPrint(builder, "row number: ", row_number); - builder.CreateStore(builder.CreateAdd(env->i64Const(1), builder.CreateLoad(rowCounterVar)), rowCounterVar); + builder.CreateStore(builder.CreateAdd(env->i64Const(1), builder.CreateLoad(builder.getInt64Ty(), rowCounterVar)), rowCounterVar); builder.CreateBr(bbLoopCondition); // loop body done @@ -136,7 +136,7 @@ namespace tuplex { // deserialize tuple codegen::FlattenedTuple ftIn(env.get()); ftIn.init(probeType); - auto curPtr = builder.CreateLoad(ptrVar); + auto curPtr = builder.CreateLoad(env->i8ptrType(), ptrVar); ftIn.deserializationCode(builder, curPtr); @@ -197,7 +197,7 @@ namespace tuplex { builder.SetInsertPoint(bbMatchFound); // call join code - writeJoinResult(env, builder, userData, builder.CreateLoad(hashedValueVar), buildType, buildKeyIndex, ftIn, + writeJoinResult(env, builder, userData, builder.CreateLoad(env->i8ptrType(), hashedValueVar), buildType, buildKeyIndex, ftIn, probeKeyIndex); builder.CreateBr(bbNext); @@ -208,7 +208,7 @@ namespace tuplex { auto serializedSize = ftIn.getSize(builder); // should be 341 for the first row! //env->debugPrint(builder, "serialized size:", serializedSize); - builder.CreateStore(builder.CreateGEP(curPtr, serializedSize), ptrVar); + builder.CreateStore(builder.MovePtrByBytes(curPtr, serializedSize), ptrVar); } llvm::Value *HashJoinStage::makeKey(std::shared_ptr &env, codegen::IRBuilder &builder, @@ -244,7 +244,7 @@ namespace tuplex { builder.SetInsertPoint(bbNotNull); builder.CreateStore(env->i8Const('_'), skey_ptr); - builder.CreateMemCpy(builder.CreateGEP(skey_ptr, env->i64Const(1)), 0, key.val, 0, key.size); + builder.CreateMemCpy(builder.MovePtrByBytes(skey_ptr, 1), 0, key.val, 0, key.size); builder.CreateBr(bbNext); builder.SetInsertPoint(bbNext); // update builder var! @@ -267,7 +267,7 @@ namespace tuplex { auto func = builder.GetInsertBlock()->getParent(); //env->debugPrint(builder, "joining records with all from bucket :P"); - auto numRows = builder.CreateLoad(builder.CreatePointerCast(bucketPtr, env->i64ptrType())); + auto numRows = builder.CreateLoad(builder.getInt64Ty(), builder.CreatePointerCast(bucketPtr, env->i64ptrType())); // env->debugPrint(builder, "bucket contains #rows: ", numRows); @@ -275,7 +275,7 @@ namespace tuplex { // uint8_t* row_data = rightPtr + sizeof(int64_t); // rightPtr += sizeof(int64_t) + row_length; - bucketPtr = builder.CreateGEP(bucketPtr, env->i64Const(sizeof(int64_t))); + bucketPtr = builder.MovePtrByBytes(bucketPtr, sizeof(int64_t)); // TODO: put bucketPtr Var in constructor auto bucketPtrVar = env->CreateFirstBlockAlloca(builder, @@ -295,14 +295,14 @@ namespace tuplex { builder.CreateBr(bbLoopCond); builder.SetInsertPoint(bbLoopCond); - auto cond = builder.CreateICmpSLT(builder.CreateLoad(loopVar), numRows); + auto cond = builder.CreateICmpSLT(builder.CreateLoad(builder.getInt64Ty(), loopVar), numRows); builder.CreateCondBr(cond, bbLoopBody, bbLoopDone); builder.SetInsertPoint(bbLoopBody); - bucketPtr = builder.CreateLoad(bucketPtrVar); - auto rowLength = builder.CreateLoad(builder.CreatePointerCast(bucketPtr, env->i64ptrType())); - bucketPtr = builder.CreateGEP(bucketPtr, env->i64Const(sizeof(int64_t))); + bucketPtr = builder.CreateLoad(env->i8ptrType(), bucketPtrVar); + auto rowLength = builder.CreateLoad(builder.getInt64Ty(), builder.CreatePointerCast(bucketPtr, env->i64ptrType())); + bucketPtr = builder.MovePtrByBytes(bucketPtr, sizeof(int64_t)); // actual data is now in bucketPtr // ==> deserialize! @@ -371,12 +371,12 @@ namespace tuplex { // logic here // move bucketPtr - builder.CreateStore(builder.CreateGEP(builder.CreateLoad(bucketPtrVar), + builder.CreateStore(builder.MovePtrByBytes(builder.CreateLoad(env->i8ptrType(), bucketPtrVar), builder.CreateAdd(env->i64Const(sizeof(int64_t)), rowLength)), bucketPtrVar); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(loopVar), env->i64Const(1)), loopVar); + builder.CreateStore(builder.CreateAdd(builder.CreateLoad(builder.getInt64Ty(), loopVar), env->i64Const(1)), loopVar); builder.CreateBr(bbLoopCond); builder.SetInsertPoint(bbLoopDone); diff --git a/tuplex/test/core/UseCaseFunctionsTest.cc b/tuplex/test/core/UseCaseFunctionsTest.cc index 18201718a..f1a816d29 100644 --- a/tuplex/test/core/UseCaseFunctionsTest.cc +++ b/tuplex/test/core/UseCaseFunctionsTest.cc @@ -15,7 +15,9 @@ #include #include #include +#if LLVM_VERSION_MAJOR < 17 #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif class UseCaseFunctionsTest : public PyTest { protected: From 728346be27c9184f60d195d2470a6043410595e3 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 26 Oct 2023 21:26:08 -0700 Subject: [PATCH 22/31] llvm and cmake tune --- tuplex/CMakeLists.txt | 18 +++++++++++++++--- tuplex/codegen/CMakeLists.txt | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index e56c25d39..d5689d44e 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -52,6 +52,16 @@ if(CMAKE_GENERATOR STREQUAL "Ninja") message(STATUS "Using ninja generator, if fails use -w dupbuild=err") endif() +# The -fvisibility=hidden option only works for static builds. +if (NOT BUILD_SHARED_LIBS) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) +else() + if (CMAKE_CXX_VISIBILITY_PRESET STREQUAL "hidden") + message(FATAL_ERROR "CMAKE_CXX_VISIBILITY_PRESET=hidden is incompatible \ + with BUILD_SHARED_LIBS.") + endif() +endif() + # detect MacOS Version because at least 10.13 is required when building with AWS SDK if(APPLE) execute_process(COMMAND bash -c "sw_vers | grep -Eo '([0-9]{1,}\\.)+[0-9]{1,}' | head -1" OUTPUT_VARIABLE MACOSX_VERSION_STRING OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -999,18 +1009,20 @@ find_package(spdlog REQUIRED) # add subdirs here... add_subdirectory(io) # <-- make sure to call this first, because it changes parent scope with io dependencies add_subdirectory(utils) -add_subdirectory(test) add_subdirectory(codegen) add_subdirectory(core) add_subdirectory(python) add_subdirectory(runtime) add_subdirectory(adapters) - # can only build aws lambda on linux platform if(LINUX AND BUILD_WITH_AWS) # removed AWS lambda implementation, can be found on separate branch - add_subdirectory(awslambda) + add_subdirectory(awslambda) endif() +# call test dir last to get vars from before +add_subdirectory(test) + + ########################################################################### # (7) Additional flags diff --git a/tuplex/codegen/CMakeLists.txt b/tuplex/codegen/CMakeLists.txt index 6bf9dcedf..478ae715f 100755 --- a/tuplex/codegen/CMakeLists.txt +++ b/tuplex/codegen/CMakeLists.txt @@ -27,7 +27,8 @@ endif() MESSAGE(STATUS "Found LLVM include dirs at: " ${LLVM_INCLUDE_DIRS}) MESSAGE(STATUS "LLVM library dir: ${LLVM_LIBRARY_DIRS}") set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} ${LLVM_LIBRARY_DIRS}) - +llvm_map_components_to_libnames(llvm_libs ${LLVM_REQUIRED_COMPONENTS}) +set(LLVM_LIBRARIES "${llvm_libs}") include_directories(${LLVM_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) From a3b958613b92a0a8f7aa20d64a0111fb0d9419b7 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 26 Oct 2023 21:45:28 -0700 Subject: [PATCH 23/31] linking and warnings --- tuplex/codegen/include/CodegenHelper.h | 12 ++++++------ tuplex/test/codegen/CMakeLists.txt | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tuplex/codegen/include/CodegenHelper.h b/tuplex/codegen/include/CodegenHelper.h index 805e34a7a..650cff0af 100644 --- a/tuplex/codegen/include/CodegenHelper.h +++ b/tuplex/codegen/include/CodegenHelper.h @@ -764,13 +764,13 @@ namespace tuplex { ctorBuilder.SetInsertPoint(&inst); } // disable here clang/gcc warning just for this - it's a limitation of how ctorbuilder is architected. -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreturn-local-addr" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreturn-local-addr" +// #pragma clang diagnostic push +// #pragma clang diagnostic ignored "-Wreturn-local-addr" +// #pragma GCC diagnostic push +// #pragma GCC diagnostic ignored "-Wreturn-local-addr" return std::move(ctorBuilder); -#pragma GCC diagnostic pop -#pragma clang diagnostic pop +// #pragma GCC diagnostic pop +// #pragma clang diagnostic pop } // in order to serialize/deserialize data properly and deal with diff --git a/tuplex/test/codegen/CMakeLists.txt b/tuplex/test/codegen/CMakeLists.txt index 764e38f6a..ff6f92de2 100755 --- a/tuplex/test/codegen/CMakeLists.txt +++ b/tuplex/test/codegen/CMakeLists.txt @@ -13,6 +13,7 @@ ASSERT_VAR(CURSES_LIBRARIES) TARGET_LINK_LIBRARIES(testcodegen libcodegen + libutils ${GTest_LIBRARIES} ${ZSTD_LIBRARIES} ${ZLIB_LIBRARIES} From 5af63707ec450192dc4d936035e2c761cb299899 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 26 Oct 2023 21:54:28 -0700 Subject: [PATCH 24/31] everything builds now, but tests fail --- tuplex/test/runtime/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuplex/test/runtime/CMakeLists.txt b/tuplex/test/runtime/CMakeLists.txt index 73d28c517..cff9af37d 100755 --- a/tuplex/test/runtime/CMakeLists.txt +++ b/tuplex/test/runtime/CMakeLists.txt @@ -9,7 +9,7 @@ file(GLOB SRCS *.cc) include(GoogleTest) -ADD_EXECUTABLE(testruntime ${SRCS}) +ADD_EXECUTABLE(testruntime ${SRCS} ../../runtime/src/Runtime.cc ../../runtime/src/StringFunctions.cc) TARGET_LINK_LIBRARIES(testruntime libio From 22fd6d990d71227c707216401cea0d8a246da73c Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Wed, 1 Nov 2023 17:36:36 -0700 Subject: [PATCH 25/31] fixes and export symbols in runtime shared object --- tuplex/core/src/ContextOptions.cc | 2 +- tuplex/core/src/RuntimeInterface.cc | 37 ++++++-- tuplex/core/src/llvm13/JITCompiler_llvm13.cc | 1 + tuplex/runtime/include/Runtime.h | 98 ++++++++++---------- tuplex/runtime/src/Runtime.cc | 55 ++++++++++- tuplex/test/runtime/RuntimeTest.cc | 56 +++++------ 6 files changed, 162 insertions(+), 87 deletions(-) diff --git a/tuplex/core/src/ContextOptions.cc b/tuplex/core/src/ContextOptions.cc index 5823a4bd2..c04c5f0c1 100644 --- a/tuplex/core/src/ContextOptions.cc +++ b/tuplex/core/src/ContextOptions.cc @@ -577,7 +577,7 @@ namespace tuplex { // check first with pathParent, then PATH std::vector failedPaths; for(auto c : candidates) { - URI p = URI(pathParent + "/" + c); + URI p = !pathParent.empty() ? URI(pathParent + "/" + c) : URI(c); if(p.exists() && p.isFile()) return p; else diff --git a/tuplex/core/src/RuntimeInterface.cc b/tuplex/core/src/RuntimeInterface.cc index 4c6feea1e..b74ddd269 100644 --- a/tuplex/core/src/RuntimeInterface.cc +++ b/tuplex/core/src/RuntimeInterface.cc @@ -9,8 +9,14 @@ //--------------------------------------------------------------------------------------------------------------------// #include -#include "llvm/Support/DynamicLibrary.h" +#include +#include #include +#include +#include +#include + +#include static bool _loaded = false; static std::string _libPath = ""; @@ -32,6 +38,19 @@ namespace tuplex { bool loaded() { return _loaded; } + + static void* findAddrOfSymbol(const char* name) { + auto addr_ptr = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(name); + + if(!addr_ptr) { + // try mangled version by prepending "_" + auto mangled_name = std::string("_") + name; + addr_ptr = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(mangled_name.c_str()); + } + + return addr_ptr; + } + bool init(const std::string& path) { if(path.length() == 0) @@ -68,13 +87,13 @@ namespace tuplex { rtfree_all = nullptr; rtmalloc=nullptr; rtfree=nullptr; - setRunTimeMemory = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("setRunTimeMemory")); - freeRunTimeMemory = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("freeRunTimeMemory")); - releaseRunTimeMemory = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("releaseRunTimeMemory")); - rtfree_all = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("rtfree_all")); - rtmalloc = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("rtmalloc")); - rtfree = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("rtfree")); - runTimeMemorySize = reinterpret_cast(llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("getRunTimeMemorySize")); + setRunTimeMemory = reinterpret_cast(findAddrOfSymbol("setRunTimeMemory")); + freeRunTimeMemory = reinterpret_cast(findAddrOfSymbol("freeRunTimeMemory")); + releaseRunTimeMemory = reinterpret_cast(findAddrOfSymbol("releaseRunTimeMemory")); + rtfree_all = reinterpret_cast(findAddrOfSymbol("rtfree_all")); + rtmalloc = reinterpret_cast(findAddrOfSymbol("rtmalloc")); + rtfree = reinterpret_cast(findAddrOfSymbol("rtfree")); + runTimeMemorySize = reinterpret_cast(findAddrOfSymbol("getRunTimeMemorySize")); cJSON_Hooks tmp = {rtmalloc, rtfree}; cJSON_InitHooks(&tmp); @@ -82,7 +101,7 @@ namespace tuplex { srand(time(0)); if(!setRunTimeMemory || !freeRunTimeMemory) { - Logger::instance().defaultLogger().error("Could not find required runtime symbols in shared library."); + Logger::instance().defaultLogger().error("Could not find required runtime symbols setRunTimeMemory or freeRunTimeMemory in shared library " + path + "."); return false; } diff --git a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc index 685943fe2..61c86ef02 100644 --- a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc +++ b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc @@ -109,6 +109,7 @@ namespace tuplex { tmb.setCPU(CPUStr); tmb.setRelocationModel(Reloc::Model::PIC_); tmb.addFeatures(getFeatureList()); + //tmb.addFeatures(codegen::getLLVMFeatureStr()); //<-- should add here probably SSE4.2.?? // build on top of this: diff --git a/tuplex/runtime/include/Runtime.h b/tuplex/runtime/include/Runtime.h index b23f0060c..0f5ce8ac1 100644 --- a/tuplex/runtime/include/Runtime.h +++ b/tuplex/runtime/include/Runtime.h @@ -14,6 +14,8 @@ // this file defines external C functions accesible from within the Python/UDF Compiler. Functions should be prefixed // with rt (no namespaces in C :/ ) +#define EXPORT_SYMBOL __attribute__((visibility("default"))) + #ifdef __cplusplus extern "C" { #endif @@ -28,46 +30,46 @@ extern "C" { * controls how much memory the compiled codepath should use for malloc/free * @param size if 0, dynamic autogrowth is assumed */ -extern void setRunTimeMemory(const size_t size, size_t blockSize) noexcept; -extern size_t getRunTimeMemorySize() noexcept; +EXPORT_SYMBOL extern void setRunTimeMemory(const size_t size, size_t blockSize) noexcept; +EXPORT_SYMBOL extern size_t getRunTimeMemorySize() noexcept; /*! * needs to be called in order to free all memory as used by UDFs. */ -extern void freeRunTimeMemory() noexcept; +EXPORT_SYMBOL extern void freeRunTimeMemory() noexcept; /*! * delete heap. */ -extern void releaseRunTimeMemory() noexcept; +EXPORT_SYMBOL extern void releaseRunTimeMemory() noexcept; /*! * returns address for memory block with given size * @param size * @return */ -extern void* rtmalloc(const size_t size) noexcept; // !!! do not change name without changing LLVMEnvironment.h malloc +EXPORT_SYMBOL extern void* rtmalloc(const size_t size) noexcept; // !!! do not change name without changing LLVMEnvironment.h malloc /*! * frees memory block * @param ptr */ -extern void rtfree(void* ptr) noexcept; +EXPORT_SYMBOL extern void rtfree(void* ptr) noexcept; /*! * frees all memory allocated by malloc at this point, i.e. garbage collection. * However, the C memory management is not invoked. (this is faster than always calling malloc/free) */ -extern void rtfree_all() noexcept; // !!! do not change without changing LLVMEnvironment.h freeAll +EXPORT_SYMBOL extern void rtfree_all() noexcept; // !!! do not change without changing LLVMEnvironment.h freeAll /*********** * fast conversion functions * @Todo: Maybe later add llvm versions of them, i.e. by linking the module to further optimize the code */ -extern int32_t fast_atoi64(const char *start, const char *end, int64_t* out); -extern int32_t fast_atod(const char *start, const char *end, double* out); -extern int32_t fast_atob(const char *start, const char *end, unsigned char *out); -extern int32_t fast_dequote(const char *start, const char *end, char **out, int64_t* size); +EXPORT_SYMBOL extern int32_t fast_atoi64(const char *start, const char *end, int64_t* out); +EXPORT_SYMBOL extern int32_t fast_atod(const char *start, const char *end, double* out); +EXPORT_SYMBOL extern int32_t fast_atob(const char *start, const char *end, unsigned char *out); +EXPORT_SYMBOL extern int32_t fast_dequote(const char *start, const char *end, char **out, int64_t* size); /*! * if necessary, return runtime allocated CSV quoted string, if not return string itself @@ -75,41 +77,41 @@ extern int32_t fast_dequote(const char *start, const char *end, char **out, int6 * @param size * @return */ -extern char* quoteForCSV(const char *str, int64_t size, int64_t* new_size, char separator, char quotechar); +EXPORT_SYMBOL extern char* quoteForCSV(const char *str, int64_t size, int64_t* new_size, char separator, char quotechar); -extern char* csvNormalize(const char quotechar, const char* start, const char* end, int64_t* ret_size); +EXPORT_SYMBOL extern char* csvNormalize(const char quotechar, const char* start, const char* end, int64_t* ret_size); // python3 compatible float to str function // i.e. 0.0 is outputted to 0.0 instead of 0 // --> bug or feature in python3?? -extern char* floatToStr(const double d, int64_t* res_size); +EXPORT_SYMBOL extern char* floatToStr(const double d, int64_t* res_size); /****** * String functions */ -extern char* strCenter(const char* s, int64_t s_size, int64_t width, int64_t* res_size, const char fillchar); -extern char* strLower(const char* s, int64_t size); -extern const char* strLowerSIMD(const char *s, int64_t size); -extern char* strUpper(const char* s, int64_t size); -extern char* strSwapcase(const char* s, int64_t size); -extern char* strFormat(const char* fmt, int64_t* res_size, const char* argtypes, ...); -extern int64_t strRfind(const char* s, const char* needle); -extern char* strReplace(const char* str, const char* from, const char* to, int64_t* res_size); - -extern char* strRStrip(const char* str, const char* chars, int64_t* res_size); -extern char* strLStrip(const char* str, const char* chars, int64_t* res_size); -extern char* strStrip(const char* str, const char* chars, int64_t* res_size); -extern int64_t strCount(const char* str, const char* sub, int64_t strSize, int64_t subSize); -extern int8_t strIsDecimal(const char* str); -extern int8_t strIsDigit(const char* str); -extern int8_t strIsAlpha(const char* str); -extern int8_t strIsAlNum(const char* str); - -extern char* strJoin(const char *base_str, int64_t base_str_size, int64_t num_words, const char** str_array, const int64_t* len_array, int64_t* res_size); -extern int64_t strSplit(const char *base_str, int64_t base_str_length, const char *delim, int64_t delim_length, char*** res_str_array, int64_t** res_len_array, int64_t *res_list_size); +EXPORT_SYMBOL extern char* strCenter(const char* s, int64_t s_size, int64_t width, int64_t* res_size, const char fillchar); +EXPORT_SYMBOL extern char* strLower(const char* s, int64_t size); +EXPORT_SYMBOL extern const char* strLowerSIMD(const char *s, int64_t size); +EXPORT_SYMBOL extern char* strUpper(const char* s, int64_t size); +EXPORT_SYMBOL extern char* strSwapcase(const char* s, int64_t size); +EXPORT_SYMBOL extern char* strFormat(const char* fmt, int64_t* res_size, const char* argtypes, ...); +EXPORT_SYMBOL extern int64_t strRfind(const char* s, const char* needle); +EXPORT_SYMBOL extern char* strReplace(const char* str, const char* from, const char* to, int64_t* res_size); + +EXPORT_SYMBOL extern char* strRStrip(const char* str, const char* chars, int64_t* res_size); +EXPORT_SYMBOL extern char* strLStrip(const char* str, const char* chars, int64_t* res_size); +EXPORT_SYMBOL extern char* strStrip(const char* str, const char* chars, int64_t* res_size); +EXPORT_SYMBOL extern int64_t strCount(const char* str, const char* sub, int64_t strSize, int64_t subSize); +EXPORT_SYMBOL extern int8_t strIsDecimal(const char* str); +EXPORT_SYMBOL extern int8_t strIsDigit(const char* str); +EXPORT_SYMBOL extern int8_t strIsAlpha(const char* str); +EXPORT_SYMBOL extern int8_t strIsAlNum(const char* str); + +EXPORT_SYMBOL extern char* strJoin(const char *base_str, int64_t base_str_size, int64_t num_words, const char** str_array, const int64_t* len_array, int64_t* res_size); +EXPORT_SYMBOL extern int64_t strSplit(const char *base_str, int64_t base_str_length, const char *delim, int64_t delim_length, char*** res_str_array, int64_t** res_len_array, int64_t *res_list_size); // string.capwords -extern char* stringCapwords(const char* str, int64_t size, int64_t *res_size); +EXPORT_SYMBOL extern char* stringCapwords(const char* str, int64_t size, int64_t *res_size); // @TODO: str.title @@ -125,31 +127,31 @@ struct matchObject { char *subject; size_t subject_len; }; -extern matchObject* wrapPCRE2MatchObject(pcre2_match_data *match_data, char* subject, size_t subject_len); +EXPORT_SYMBOL extern matchObject* wrapPCRE2MatchObject(pcre2_match_data *match_data, char* subject, size_t subject_len); // expose functions -extern pcre2_general_context* pcre2GetLocalGeneralContext(); -extern void* pcre2GetGlobalGeneralContext(); -extern void* pcre2GetGlobalMatchContext(); -extern void* pcre2GetGlobalCompileContext(); +EXPORT_SYMBOL extern pcre2_general_context* pcre2GetLocalGeneralContext(); +EXPORT_SYMBOL extern void* pcre2GetGlobalGeneralContext(); +EXPORT_SYMBOL extern void* pcre2GetGlobalMatchContext(); +EXPORT_SYMBOL extern void* pcre2GetGlobalCompileContext(); // could get rid of these functions, it's a direct free call... -extern void pcre2ReleaseGlobalGeneralContext(void* gcontext); -extern void pcre2ReleaseGlobalMatchContext(void* mcontext); -extern void pcre2ReleaseGlobalCompileContext(void* ccontext); +EXPORT_SYMBOL extern void pcre2ReleaseGlobalGeneralContext(void* gcontext); +EXPORT_SYMBOL extern void pcre2ReleaseGlobalMatchContext(void* mcontext); +EXPORT_SYMBOL extern void pcre2ReleaseGlobalCompileContext(void* ccontext); // return a uniformly random integer on [start, end) -extern int64_t uniform_int(int64_t start, int64_t end); +EXPORT_SYMBOL extern int64_t uniform_int(int64_t start, int64_t end); // what about overflow? -extern int64_t pow_i64(int64_t base, int64_t exp); -extern double pow_f64(double base, int64_t exp); +EXPORT_SYMBOL extern int64_t pow_i64(int64_t base, int64_t exp); +EXPORT_SYMBOL extern double pow_f64(double base, int64_t exp); // python compatible python func for float -extern double rt_py_pow(double base, double exponent, int64_t* ecCode); +EXPORT_SYMBOL extern double rt_py_pow(double base, double exponent, int64_t* ecCode); // spanner function for CSV parsing -int fallback_spanner(const char* ptr, const char c1, const char c2, const char c3, const char c4); +EXPORT_SYMBOL int fallback_spanner(const char* ptr, const char c1, const char c2, const char c3, const char c4); #ifdef __cplusplus } diff --git a/tuplex/runtime/src/Runtime.cc b/tuplex/runtime/src/Runtime.cc index 90dc91af1..b8eb4ddd4 100644 --- a/tuplex/runtime/src/Runtime.cc +++ b/tuplex/runtime/src/Runtime.cc @@ -533,6 +533,47 @@ extern "C" char* strReplace(const char* str, const char* from, const char* to, i return ret; } +// helper function to replace undefined floating point formats with correct ones +std::string replace_with_float_default_format(const std::string& fmt, const std::string& argtypes) { + + auto default_float_fmt = "{:#g}"; + + unsigned pos = 0; + std::string new_fmt; + unsigned argpos = 0; + unsigned startpos = 0; + while(pos < fmt.size()) { + auto curchar = fmt[pos]; + auto nextchar = pos + 1 < fmt.size() ? fmt[pos + 1] : 0; + + if(curchar == '{' && nextchar == '{') { + new_fmt += "{{"; + pos += 2; + } else if(curchar == '}' && nextchar == '}') { + new_fmt += "}}"; + pos += 2; + } else if(curchar == '{') { + startpos = pos; + + // special case: {} and arg is float + if(argpos < argtypes.size() && 'f' == argtypes[argpos] && nextchar == '}') { + new_fmt += default_float_fmt; + pos += 2; + } else { + new_fmt.push_back(curchar); + pos++; + } + } else if(curchar == '}') { + argpos++; + new_fmt.push_back(curchar); + pos++; + } else { + new_fmt.push_back(curchar); + pos++; + } + } + return new_fmt; +} /*! * strFormat function with variable number of arguments. Supports formatting for bool, int, float, str. @@ -563,6 +604,8 @@ extern "C" char* strFormat(const char *str, int64_t *res_size, const char* argty // retrieve the arguments va_list argp; va_start(argp, argtypes); + bool found_float = false; + auto original_argtypes = argtypes; int num_args = (int)strlen(argtypes); for(int i=0; i Date: Wed, 1 Nov 2023 20:55:41 -0700 Subject: [PATCH 26/31] compile fixes, change lllvm version to 15 to have typed pointers --- tuplex/codegen/include/CodegenHelper.h | 8 ++++---- tuplex/codegen/include/LLVMEnvironment.h | 2 +- tuplex/core/src/RuntimeInterface.cc | 4 ++-- tuplex/core/src/llvm13/JITCompiler_llvm13.cc | 12 +++++++++++- tuplex/vcpkg.json | 2 +- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tuplex/codegen/include/CodegenHelper.h b/tuplex/codegen/include/CodegenHelper.h index 650cff0af..4fa2de817 100644 --- a/tuplex/codegen/include/CodegenHelper.h +++ b/tuplex/codegen/include/CodegenHelper.h @@ -414,7 +414,7 @@ namespace tuplex { inline llvm::CallInst *CreateCall(llvm::FunctionType *FTy, llvm::Value *Callee, -#if (LLVM_VERSION_MAJOR >= 10) +#if (LLVM_VERSION_MAJOR >= 16) llvm::ArrayRef Args = std::nullopt, #else llvm::ArrayRef Args = {}, @@ -426,7 +426,7 @@ namespace tuplex { } inline llvm::CallInst* CreateCall(llvm::Value* func_value, -#if (LLVM_VERSION_MAJOR >= 10) +#if (LLVM_VERSION_MAJOR >= 16) llvm::ArrayRef Args = std::nullopt, #else llvm::ArrayRef Args = {}, @@ -440,7 +440,7 @@ namespace tuplex { } inline llvm::CallInst* CreateCall(llvm::Function* func, -#if (LLVM_VERSION_MAJOR >= 10) +#if (LLVM_VERSION_MAJOR >= 16) llvm::ArrayRef Args = std::nullopt, #else llvm::ArrayRef Args = {}, @@ -451,7 +451,7 @@ namespace tuplex { } inline llvm::CallInst *CreateCall(llvm::FunctionCallee Callee, -#if (LLVM_VERSION_MAJOR >= 10) +#if (LLVM_VERSION_MAJOR >= 16) llvm::ArrayRef Args = std::nullopt, #else llvm::ArrayRef Args = {}, diff --git a/tuplex/codegen/include/LLVMEnvironment.h b/tuplex/codegen/include/LLVMEnvironment.h index ac671a2e9..dec5598a2 100644 --- a/tuplex/codegen/include/LLVMEnvironment.h +++ b/tuplex/codegen/include/LLVMEnvironment.h @@ -76,7 +76,7 @@ namespace llvm { CallInst *CI = CallInst::Create(Callee, Ops, Name); if (FMFSource) CI->copyFastMathFlags(FMFSource); -#if (LLVM_VERSION_MAJOR <= 14) +#if (LLVM_VERSION_MAJOR <= 15) builder.GetInsertBlock()->getInstList().insert(builder.GetInsertPoint(), CI); #else CI->insertInto(builder.GetInsertBlock(), builder.GetInsertBlock()->begin()); diff --git a/tuplex/core/src/RuntimeInterface.cc b/tuplex/core/src/RuntimeInterface.cc index b74ddd269..45f0c32a5 100644 --- a/tuplex/core/src/RuntimeInterface.cc +++ b/tuplex/core/src/RuntimeInterface.cc @@ -14,9 +14,9 @@ #include #include #include +#if LLVM_VERSION_MAJOR >= 16 #include - -#include +#endif static bool _loaded = false; static std::string _libPath = ""; diff --git a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc index 61c86ef02..0d21b4ce8 100644 --- a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc +++ b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc @@ -284,7 +284,7 @@ namespace tuplex { for(auto keyval: _customSymbols) { #if LLVM_VERSION_MAJOR <= 16 - auto rc = jitlib.define(absoluteSymbols({{Mangle(keyval.first), keyval.second}})); + auto rc = jitlib.define(llvm::orc::absoluteSymbols({{Mangle(keyval.first), keyval.second}})); #else auto rc = jitlib.define(llvm::orc::absoluteSymbols(llvm::orc::SymbolMap({ { Mangle(keyval.first), @@ -318,6 +318,16 @@ namespace tuplex { // create for this module own jitlib auto& ES = _lljit->getExecutionSession(); + + // if lib with name already exists, remove + llvm::orc::JITDylib *jitlib_ptr = nullptr; + if((jitlib_ptr = ES.getJITDylibByName(module_name.str()))) { + auto err = ES.removeJITDylib(*jitlib_ptr); + if(err) + throw std::runtime_error("failed to remove JITDylib " + module_name.str() + " from execution session."); + jitlib_ptr = nullptr; + } + auto& jitlib = ES.createJITDylib(module_name.str()).get(); const auto& DL = _lljit->getDataLayout(); llvm::orc::MangleAndInterner Mangle(ES, DL); diff --git a/tuplex/vcpkg.json b/tuplex/vcpkg.json index 96ef52f94..59b705a60 100644 --- a/tuplex/vcpkg.json +++ b/tuplex/vcpkg.json @@ -13,7 +13,7 @@ "version": "10.1.1" }, {"name": "llvm", - "version": "17.0.2" + "version": "15.0.7" } ], "builtin-baseline": "3265c187c74914aa5569b75355badebfdbab7987" From 1dd7da84554d7e6f52ee68daa64cdbdb91480f40 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 2 Nov 2023 20:38:16 -0700 Subject: [PATCH 27/31] another fix --- tuplex/core/src/llvm13/JITCompiler_llvm13.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc index 0d21b4ce8..f72a9141f 100644 --- a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc +++ b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc @@ -238,6 +238,15 @@ namespace tuplex { // create for this module own jitlib auto& ES = _lljit->getExecutionSession(); + + // if lib with name already exists, remove + llvm::orc::JITDylib *jitlib_ptr = nullptr; + if((jitlib_ptr = ES.getJITDylibByName(module_name))) { + auto err = ES.removeJITDylib(*jitlib_ptr); + if(err) + throw std::runtime_error("failed to remove JITDylib " + module_name + " from execution session."); + jitlib_ptr = nullptr; + } auto& jitlib = ES.createJITDylib(module_name).get(); const auto& DL = _lljit->getDataLayout(); MangleAndInterner Mangle(ES, DL); From f73d3264d116b3f8f14b4ff4949e4487710bc445 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Thu, 2 Nov 2023 21:24:30 -0700 Subject: [PATCH 28/31] update address resolution for llvm17 --- tuplex/core/src/llvm13/JITCompiler_llvm13.cc | 9 ++++++--- tuplex/vcpkg.json | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc index f72a9141f..07c8ce4bd 100644 --- a/tuplex/core/src/llvm13/JITCompiler_llvm13.cc +++ b/tuplex/core/src/llvm13/JITCompiler_llvm13.cc @@ -295,10 +295,13 @@ namespace tuplex { #if LLVM_VERSION_MAJOR <= 16 auto rc = jitlib.define(llvm::orc::absoluteSymbols({{Mangle(keyval.first), keyval.second}})); #else + // LLVM17 introduces new llvm::orc::ExecutorSymbolDef class + // convert JITEvaluatedSymbol from map to this new class. auto rc = jitlib.define(llvm::orc::absoluteSymbols(llvm::orc::SymbolMap({ - { Mangle(keyval.first), - {llvm::orc::ExecutorAddr::fromPtr(&keyval.second), llvm::JITSymbolFlags() } } - }))); + { Mangle(keyval.first), + { llvm::orc::ExecutorAddr(keyval.second.getAddress()), + keyval.second.getFlags()} } + }))); #endif } } diff --git a/tuplex/vcpkg.json b/tuplex/vcpkg.json index 59b705a60..96ef52f94 100644 --- a/tuplex/vcpkg.json +++ b/tuplex/vcpkg.json @@ -13,7 +13,7 @@ "version": "10.1.1" }, {"name": "llvm", - "version": "15.0.7" + "version": "17.0.2" } ], "builtin-baseline": "3265c187c74914aa5569b75355badebfdbab7987" From 9e4d108f3f1b9a0954634c59a1f3c47b27b29309 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 4 Nov 2023 15:50:33 -0700 Subject: [PATCH 29/31] update github pipelines to rely on vcpkg for this build --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 77fead2e3..567eb9618 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -85,7 +85,7 @@ jobs: CIBW_BUILD: ${{ matrix.cibw-build }} # macOS dependencies separate, for linux use docker tuplex/ci:3.x images. - CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile + #CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile # bundle aws runner with linux wheel, remove environment variable TUPLEX_LAMBDA_ZIP to remove runner. CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib" From 6475ccb1208065cb7fcd12c2154f63fe37812c45 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Sat, 18 Nov 2023 11:35:35 -0800 Subject: [PATCH 30/31] protobuf change --- tuplex/core/CMakeLists.txt | 5 +---- tuplex/vcpkg.json | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tuplex/core/CMakeLists.txt b/tuplex/core/CMakeLists.txt index e5f323112..2652c5c2d 100755 --- a/tuplex/core/CMakeLists.txt +++ b/tuplex/core/CMakeLists.txt @@ -18,10 +18,7 @@ if(BUILD_WITH_AWS) # is installed # set(Protobuf_USE_STATIC_LIBS ON) # https://github.com/protocolbuffers/protobuf/issues/12637 - find_package(Protobuf CONFIG) - if(NOT Protobuf_FOUND) - find_package(Protobuf REQUIRED) - endif() + find_package(Protobuf REQUIRED) include_directories(Protobuf_INCLUDE_DIRS) add_library(proto-objects OBJECT "${CMAKE_CURRENT_LIST_DIR}/proto/Lambda.proto") diff --git a/tuplex/vcpkg.json b/tuplex/vcpkg.json index 96ef52f94..5ab426a41 100644 --- a/tuplex/vcpkg.json +++ b/tuplex/vcpkg.json @@ -5,7 +5,8 @@ "fmt", "spdlog", {"name" : "aws-sdk-cpp", "features": ["s3", "lambda", "transfer"]}, - {"name": "llvm", "features": ["enable-rtti", "enable-zlib", "enable-zstd", "target-aarch64", "target-x86"]} + {"name": "llvm", "features": ["enable-rtti", "enable-zlib", "enable-zstd", "target-aarch64", "target-x86"]}, + {"name": "protobuf", "features": ["zlib"]} ], "overrides": [ { From db3606879659db9450f68681d26bdbd51209cc23 Mon Sep 17 00:00:00 2001 From: Leonhard Spiegelberg Date: Tue, 21 Nov 2023 23:13:35 -0800 Subject: [PATCH 31/31] llvm17 fixes/updates --- .github/workflows/build_wheels.yml | 86 +-- tuplex/CMakeLists.txt | 9 - tuplex/cmake/vcpkg.cmake | 611 ------------------ tuplex/codegen/CMakeLists.txt | 44 +- tuplex/codegen/include/IteratorContextProxy.h | 64 -- tuplex/codegen/src/IteratorContextProxy.cc | 195 ------ tuplex/core/CMakeLists.txt | 5 +- tuplex/runtime/CMakeLists.txt | 2 +- tuplex/vcpkg.json | 21 - 9 files changed, 75 insertions(+), 962 deletions(-) delete mode 100644 tuplex/cmake/vcpkg.cmake delete mode 100644 tuplex/vcpkg.json diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 567eb9618..06c9e6029 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -4,65 +4,37 @@ on: [push, pull_request, workflow_dispatch] jobs: build_wheels: - name: Build wheel for ${{ matrix.cibw-build }} + name: Build wheel on ${{ matrix.os }} - py ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: matrix: - # os: [ ubuntu-20.04, macos-11 ] - os: ["macos-12"] - python-version: ["3.9"] #["3.8", "3.9", "3.10", "3.11"] - hw: ["arm64"] #["x86_64", "arm64"] - # exclude: - # - os: ubuntu-20.04 - # hw: "arm64" - # - os: macos-11 - # python-version: "3.8" - # hw: "arm64" + os: [ ubuntu-20.04, macos-11 ] + python-version: ["3.8", "3.9", "3.10", "3.11"] include: - # - os: ubuntu-20.04 - # python-version: "3.8" - # cibw-build: "cp38-manylinux_x86_64" - # hw: "x86_64" - # - os: ubuntu-20.04 - # python-version: "3.9" - # cibw-build: "cp39-manylinux_x86_64" - # hw: "x86_64" - # - os: ubuntu-20.04 - # python-version: "3.10" - # cibw-build: "cp310-manylinux_x86_64" - # hw: "x86_64" - # - os: ubuntu-20.04 - # python-version: "3.11" - # cibw-build: "cp311-manylinux_x86_64" - # hw: "x86_64" - # - os: macos-11 - # python-version: "3.8" - # cibw-build: "cp38-macosx_x86_64" - # hw: "x86_64" - # - os: macos-11 - # python-version: "3.9" - # cibw-build: "cp39-macosx_x86_64" - # hw: "x86_64" - # - os: macos-11 - # python-version: "3.10" - # cibw-build: "cp310-macosx_x86_64" - # hw: "x86_64" - # - os: macos-11 - # python-version: "3.11" - # cibw-build: "cp311-macosx_x86_64" - # hw: "x86_64" - - os: macos-12 + - os: ubuntu-20.04 + python-version: "3.8" + cibw-build: "cp38-manylinux_x86_64" + - os: ubuntu-20.04 python-version: "3.9" - cibw-build: "cp39-macosx_arm64" - hw: "arm64" - # - os: macos-11 - # python-version: "3.10" - # cibw-build: "cp310-macosx_arm64" - # hw: "arm64" - # - os: macos-11 - # python-version: "3.11" - # cibw-build: "cp311-macosx_arm64" - # hw: "arm64" + cibw-build: "cp39-manylinux_x86_64" + - os: ubuntu-20.04 + python-version: "3.10" + cibw-build: "cp310-manylinux_x86_64" + - os: ubuntu-20.04 + python-version: "3.11" + cibw-build: "cp311-manylinux_x86_64" + - os: macos-11 + python-version: "3.8" + cibw-build: "cp38-macosx_x86_64" + - os: macos-11 + python-version: "3.9" + cibw-build: "cp39-macosx_x86_64" + - os: macos-11 + python-version: "3.10" + cibw-build: "cp310-macosx_x86_64" + - os: macos-11 + python-version: "3.11" + cibw-build: "cp311-macosx_x86_64" steps: - uses: actions/checkout@v3 @@ -79,13 +51,11 @@ jobs: # configure cibuildwheel to build native archs ('auto'), and some # emulated ones CIBW_ARCHS_LINUX: native - CIBW_ARCHS_MACOS: ${{ matrix.hw }} - CIBW_MANYLINUX_X86_64_IMAGE: "registry-1.docker.io/tuplex/ci:${{ matrix.python-version }}" CIBW_BUILD: ${{ matrix.cibw-build }} # macOS dependencies separate, for linux use docker tuplex/ci:3.x images. - #CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile + CIBW_BEFORE_ALL_MACOS: bash ./scripts/macos/install_antlr4_cpp_runtime.sh && bash ./scripts/macos/brew_dependencies.sh && bash ./scripts/macos/install_aws-sdk-cpp.sh && echo 'export PATH="/usr/local/opt/openjdk@11/bin:$PATH"' >> /Users/runner/.bash_profile # bundle aws runner with linux wheel, remove environment variable TUPLEX_LAMBDA_ZIP to remove runner. CIBW_ENVIRONMENT_LINUX: "TUPLEX_LAMBDA_ZIP='./tuplex/python/tuplex/other/tplxlam.zip' CMAKE_ARGS='-DBUILD_WITH_AWS=ON -DBUILD_WITH_ORC=ON' LD_LIBRARY_PATH=/usr/local/lib:/opt/lib" @@ -99,4 +69,4 @@ jobs: - uses: actions/upload-artifact@v3 with: path: | - ./wheelhouse/*.whl + ./wheelhouse/*.whl \ No newline at end of file diff --git a/tuplex/CMakeLists.txt b/tuplex/CMakeLists.txt index d5689d44e..e544d8f92 100755 --- a/tuplex/CMakeLists.txt +++ b/tuplex/CMakeLists.txt @@ -10,13 +10,6 @@ message(STATUS "Using language version: C++${CMAKE_CXX_STANDARD}") # add cmake modules from cmake folder list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/") -# use vcpkg as manager from https://github.com/bitmeal/vcpkg-cmake-integration -#set(VCPKG_VERSION edge) # optional -include("${CMAKE_SOURCE_DIR}/cmake/vcpkg.cmake") - - - - # Tuplex build options: # ===================== @@ -1003,8 +996,6 @@ endif() # ncurses/curses lib for terminal manipulation find_package(Curses REQUIRED) -find_package(fmt REQUIRED) -find_package(spdlog REQUIRED) # add subdirs here... add_subdirectory(io) # <-- make sure to call this first, because it changes parent scope with io dependencies diff --git a/tuplex/cmake/vcpkg.cmake b/tuplex/cmake/vcpkg.cmake deleted file mode 100644 index 03dbfb5d1..000000000 --- a/tuplex/cmake/vcpkg.cmake +++ /dev/null @@ -1,611 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. -# -# Copyright (C) 2022, Arne Wendt -# - -# vcpkg examples use 3.0.0, assuming this as minimum version for vcpkg cmake toolchain -cmake_minimum_required(VERSION 3.0.0) - -# config: -# - VCPKG_VERSION: -# - "latest": latest git tag (undefined or empty treated as "latest") -# - "edge": last commit on master -# - VCPKG_PARENT_DIR: where to place vcpkg -# - VCPKG_FORCE_SYSTEM_BINARIES: use system cmake, zip, unzip, tar, etc. -# may be necessary on some systems as downloaded binaries may be linked against unsupported libraries -# musl-libc based distros (ALPINE)(!) require use of system binaries, but are AUTO DETECTED! -# - VCPKG_FEATURE_FLAGS: modify feature flags; default are "manifests,versions" -# -# - VCPKG_NO_INIT: do not call vcpkg_init() automatically (for use testing) - - -# set default feature flags if not defined -if(NOT DEFINED VCPKG_FEATURE_FLAGS) - set(VCPKG_FEATURE_FLAGS "manifests,versions" CACHE INTERNAL "necessary vcpkg flags for manifest based autoinstall and versioning") -endif() - -# disable metrics by default -if(NOT DEFINED VCPKG_METRICS_FLAG) - set(VCPKG_METRICS_FLAG "-disableMetrics" CACHE INTERNAL "flag to disable telemtry by default") -endif() - -# enable rebuilding of packages if requested by changed configuration -if(NOT DEFINED VCPKG_RECURSE_REBUILD_FLAG) - set(VCPKG_RECURSE_REBUILD_FLAG "--recurse" CACHE INTERNAL "enable rebuilding of packages if requested by changed configuration by default") -endif() - - -# check_conditions and find neccessary packages -find_package(Git REQUIRED) - - - -# get VCPKG -function(vcpkg_init) - # set environment (not cached) - - # mask musl-libc if masked prior - if(VCPKG_MASK_MUSL_LIBC) - vcpkg_mask_if_musl_libc() - endif() - - # use system binaries - if(VCPKG_FORCE_SYSTEM_BINARIES) - set(ENV{VCPKG_FORCE_SYSTEM_BINARIES} "1") - endif() - - # for use in scripting mode - if(CMAKE_SCRIPT_MODE_FILE) - if(VCPKG_TARGET_TRIPLET) - set(ENV{VCPKG_DEFAULT_TRIPLET} "${VCPKG_DEFAULT_TRIPLET}") - endif() - if(VCPKG_DEFAULT_TRIPLET) - set(ENV{VCPKG_DEFAULT_TRIPLET} "${VCPKG_DEFAULT_TRIPLET}") - endif() - if(VCPKG_HOST_TRIPLET) - set(ENV{VCPKG_DEFAULT_HOST_TRIPLET} "${VCPKG_DEFAULT_HOST_TRIPLET}") - endif() - if(VCPKG_DEFAULT_HOST_TRIPLET) - set(ENV{VCPKG_DEFAULT_HOST_TRIPLET} "${VCPKG_DEFAULT_HOST_TRIPLET}") - endif() - endif() - # end set environment - - - # test for vcpkg availability - # executable path set ? assume all ok : configure - if(VCPKG_EXECUTABLE EQUAL "" OR NOT DEFINED VCPKG_EXECUTABLE) - # configure vcpkg - - # use system binaries? - # IMPORTANT: we have to use system binaries on musl-libc systems, as vcpkg fetches binaries linked against glibc! - vcpkg_set_use_system_binaries_flag() - - # mask musl-libc if no triplet is provided - if( - ( ENV{VCPKG_DEFAULT_TRIPLET} EQUAL "" OR NOT DEFINED ENV{VCPKG_DEFAULT_TRIPLET}) AND - ( ENV{VCPKG_DEFAULT_HOST_TRIPLET} EQUAL "" OR NOT DEFINED ENV{VCPKG_DEFAULT_HOST_TRIPLET}) AND - ( VCPKG_TARGET_TRIPLET EQUAL "" OR NOT DEFINED VCPKG_TARGET_TRIPLET) - ) - # mask musl-libc from vcpkg - vcpkg_mask_if_musl_libc() - else() - message(WARNING "One of VCPKG_TARGET_TRIPLET, ENV{VCPKG_DEFAULT_TRIPLET} or ENV{VCPKG_DEFAULT_HOST_TRIPLET} has been defined. NOT CHECKING FOR musl-libc MASKING!") - endif() - - - # test options - if(VCPKG_PARENT_DIR EQUAL "" OR NOT DEFINED VCPKG_PARENT_DIR) - if(CMAKE_SCRIPT_MODE_FILE) - message(FATAL_ERROR "Explicitly specify VCPKG_PARENT_DIR when running in script mode!") - else() - message(STATUS "VCPKG from: ${CMAKE_CURRENT_BINARY_DIR}") - set(VCPKG_PARENT_DIR "${CMAKE_CURRENT_BINARY_DIR}/") - endif() - endif() - string(REGEX REPLACE "[/\\]$" "" VCPKG_PARENT_DIR "${VCPKG_PARENT_DIR}") - - # test if VCPKG_PARENT_DIR has to be created in script mode - if(CMAKE_SCRIPT_MODE_FILE AND NOT EXISTS "${VCPKG_PARENT_DIR}") - message(STATUS "Creating vcpkg parent directory") - file(MAKE_DIRECTORY "${VCPKG_PARENT_DIR}") - endif() - - - # set path/location varibles to expected path; necessary to detect after a CMake cache clean - vcpkg_set_vcpkg_directory_from_parent() - vcpkg_set_vcpkg_executable() - - # executable is present ? configuring done : fetch and build - execute_process(COMMAND ${VCPKG_EXECUTABLE} version RESULT_VARIABLE VCPKG_TEST_RETVAL OUTPUT_VARIABLE VCPKG_VERSION_BANNER) - if(NOT VCPKG_TEST_RETVAL EQUAL "0") - # reset executable path to prevent malfunction/wrong assumptions in case of error - set(VCPKG_EXECUTABLE "") - - # getting vcpkg - message(STATUS "No VCPKG executable found; getting new version ready...") - - # select compile script - if(WIN32) - set(VCPKG_BUILD_CMD ".\\bootstrap-vcpkg.bat") - else() - set(VCPKG_BUILD_CMD "./bootstrap-vcpkg.sh") - endif() - - # prepare and clone git sources - # include(FetchContent) - # set(FETCHCONTENT_QUIET on) - # set(FETCHCONTENT_BASE_DIR "${VCPKG_PARENT_DIR}") - # FetchContent_Declare( - # vcpkg - - # GIT_REPOSITORY "https://github.com/microsoft/vcpkg" - # GIT_PROGRESS true - - # SOURCE_DIR "${VCPKG_PARENT_DIR}/vcpkg" - # BINARY_DIR "" - # BUILD_IN_SOURCE true - # CONFIGURE_COMMAND "" - # BUILD_COMMAND "" - # ) - # FetchContent_Populate(vcpkg) - - # check for bootstrap script ? ok : fetch repository - if(NOT EXISTS "${VCPKG_DIRECTORY}/${VCPKG_BUILD_CMD}" AND NOT EXISTS "${VCPKG_DIRECTORY}\\${VCPKG_BUILD_CMD}") - message(STATUS "VCPKG bootstrap script not found; fetching...") - # directory existent ? delete - if(EXISTS "${VCPKG_DIRECTORY}") - file(REMOVE_RECURSE "${VCPKG_DIRECTORY}") - endif() - - # fetch vcpkg repo - execute_process(COMMAND ${GIT_EXECUTABLE} clone https://github.com/microsoft/vcpkg WORKING_DIRECTORY "${VCPKG_PARENT_DIR}" RESULT_VARIABLE VCPKG_GIT_CLONE_OK) - if(NOT VCPKG_GIT_CLONE_OK EQUAL "0") - message(FATAL_ERROR "Cloning VCPKG repository from https://github.com/microsoft/vcpkg failed!") - endif() - endif() - - # compute git checkout target - vcpkg_set_version_checkout() - - # hide detached head notice - execute_process(COMMAND ${GIT_EXECUTABLE} config advice.detachedHead false WORKING_DIRECTORY "${VCPKG_DIRECTORY}" RESULT_VARIABLE VCPKG_GIT_HIDE_DETACHED_HEAD_IGNORED) - # checkout asked version - execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${VCPKG_VERSION_CHECKOUT} WORKING_DIRECTORY "${VCPKG_DIRECTORY}" RESULT_VARIABLE VCPKG_GIT_TAG_CHECKOUT_OK) - if(NOT VCPKG_GIT_TAG_CHECKOUT_OK EQUAL "0") - message(FATAL_ERROR "Checking out VCPKG version/tag ${VCPKG_VERSION} failed!") - endif() - - # wrap -disableMetrics in extra single quotes for windows - # if(WIN32 AND NOT VCPKG_METRICS_FLAG EQUAL "" AND DEFINED VCPKG_METRICS_FLAG) - # set(VCPKG_METRICS_FLAG "'${VCPKG_METRICS_FLAG}'") - # endif() - - # build vcpkg - execute_process(COMMAND ${VCPKG_BUILD_CMD} ${VCPKG_USE_SYSTEM_BINARIES_FLAG} ${VCPKG_METRICS_FLAG} WORKING_DIRECTORY "${VCPKG_DIRECTORY}" RESULT_VARIABLE VCPKG_BUILD_OK) - if(NOT VCPKG_BUILD_OK EQUAL "0") - message(FATAL_ERROR "Bootstrapping VCPKG failed!") - endif() - message(STATUS "Built VCPKG!") - - - # get vcpkg path - vcpkg_set_vcpkg_executable() - - # test vcpkg binary - execute_process(COMMAND ${VCPKG_EXECUTABLE} version RESULT_VARIABLE VCPKG_OK OUTPUT_VARIABLE VCPKG_VERSION_BANNER) - if(NOT VCPKG_OK EQUAL "0") - message(FATAL_ERROR "VCPKG executable failed test!") - endif() - - message(STATUS "VCPKG OK!") - message(STATUS "Install packages using VCPKG:") - message(STATUS " * from your CMakeLists.txt by calling vcpkg_add_package()") - message(STATUS " * by providing a 'vcpkg.json' in your project directory [https://devblogs.microsoft.com/cppblog/take-control-of-your-vcpkg-dependencies-with-versioning-support/]") - - # generate empty manifest on vcpkg installation if none is found - if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg.json") - cmake_language(DEFER DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} CALL vcpkg_manifest_generation_finalize) - message(STATUS "If you need an empty manifest for setting up your project, you will find one in your bild directory") - endif() - endif() - - # we have fetched and built, but a clean has been performed - # version banner is set while testing for availability or after build - message(STATUS "VCPKG using:") - string(REGEX REPLACE "\n.*$" "" VCPKG_VERSION_BANNER "${VCPKG_VERSION_BANNER}") - message(STATUS "${VCPKG_VERSION_BANNER}") - - # cache executable path - set(VCPKG_EXECUTABLE ${VCPKG_EXECUTABLE} CACHE STRING "vcpkg executable path" FORCE) - - # initialize manifest generation - vcpkg_manifest_generation_init() - - # install from manifest if ran in script mode - if(CMAKE_SCRIPT_MODE_FILE) - message(STATUS "Running in script mode to setup environment: trying dependency installation from manifest!") - if(EXISTS "./vcpkg.json") - message(STATUS "Found vcpkg.json; installing...") - vcpkg_install_manifest() - else() - message(STATUS "NOT found vcpkg.json; skipping installation") - endif() - endif() - - # set toolchain - set(CMAKE_TOOLCHAIN_FILE "${VCPKG_DIRECTORY}/scripts/buildsystems/vcpkg.cmake") - set(CMAKE_TOOLCHAIN_FILE ${CMAKE_TOOLCHAIN_FILE} PARENT_SCOPE) - set(CMAKE_TOOLCHAIN_FILE ${CMAKE_TOOLCHAIN_FILE} CACHE STRING "") - endif() -endfunction() - - -# # make target triplet from current compiler selection and platform -# # set VCPKG_TARGET_TRIPLET in parent scope -# function(vcpkg_make_set_triplet) -# # get platform: win/linux ONLY -# if(WIN32) -# set(PLATFORM "windows") -# else() -# set(PLATFORM "linux") -# endif() - -# # get bitness: 32/64 ONLY -# if(CMAKE_SIZEOF_VOID_P EQUAL 8) -# set(BITS 64) -# else() -# set(BITS 86) -# endif() - -# set(VCPKG_TARGET_TRIPLET "x${BITS}-${PLATFORM}" PARENT_SCOPE) -# endfunction() - -# set VCPKG_DIRECTORY to assumed path based on VCPKG_PARENT_DIR -# vcpkg_set_vcpkg_directory_from_parent([VCPKG_PARENT_DIR_EXPLICIT]) -function(vcpkg_set_vcpkg_directory_from_parent) - if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) - set(VCPKG_DIRECTORY "${VCPKG_PARENT_DIR}/vcpkg" PARENT_SCOPE) - else() - set(VCPKG_DIRECTORY "${ARGV0}/vcpkg" PARENT_SCOPE) - endif() - # set(VCPKG_DIRECTORY ${VCPKG_DIRECTORY} CACHE STRING "vcpkg tool location" FORCE) -endfunction() - - -# set VCPKG_EXECUTABLE to assumed path based on VCPKG_DIRECTORY -# vcpkg_set_vcpkg_executable([VCPKG_DIRECTORY]) -function(vcpkg_set_vcpkg_executable) - if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) - set(VCPKG_DIRECTORY_EXPLICIT ${VCPKG_DIRECTORY}) - else() - set(VCPKG_DIRECTORY_EXPLICIT ${ARGV0}) - endif() - - if(WIN32) - set(VCPKG_EXECUTABLE "${VCPKG_DIRECTORY_EXPLICIT}/vcpkg.exe" PARENT_SCOPE) - else() - set(VCPKG_EXECUTABLE "${VCPKG_DIRECTORY_EXPLICIT}/vcpkg" PARENT_SCOPE) - endif() -endfunction() - -# determine git checkout target in: VCPKG_VERSION_CHECKOUT -# vcpkg_set_version_checkout([VCPKG_VERSION_EXPLICIT] [VCPKG_DIRECTORY_EXPLICIT]) -function(vcpkg_set_version_checkout) - if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) - set(VCPKG_VERSION_EXPLICIT ${VCPKG_VERSION}) - else() - set(VCPKG_VERSION_EXPLICIT ${ARGV0}) - endif() - if(ARGV1 EQUAL "" OR NOT DEFINED ARGV1) - set(VCPKG_DIRECTORY_EXPLICIT ${VCPKG_DIRECTORY}) - else() - set(VCPKG_DIRECTORY_EXPLICIT ${ARGV1}) - endif() - - # get latest git tag - execute_process(COMMAND git for-each-ref refs/tags/ --count=1 --sort=-creatordate --format=%\(refname:short\) WORKING_DIRECTORY "${VCPKG_DIRECTORY_EXPLICIT}" OUTPUT_VARIABLE VCPKG_GIT_TAG_LATEST) - string(REGEX REPLACE "\n$" "" VCPKG_GIT_TAG_LATEST "${VCPKG_GIT_TAG_LATEST}") - - # resolve versions - if(EXISTS "./vcpkg.json") - # set hash from vcpkg.json manifest - file(READ "./vcpkg.json" VCPKG_MANIFEST_CONTENTS) - - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) - string(JSON VCPKG_BASELINE GET "${VCPKG_MANIFEST_CONTENTS}" "builtin-baseline") - else() - string(REGEX REPLACE "[\n ]" "" VCPKG_MANIFEST_CONTENTS "${VCPKG_MANIFEST_CONTENTS}") - string(REGEX MATCH "\"builtin-baseline\":\"[0-9a-f]+\"" VCPKG_BASELINE "${VCPKG_MANIFEST_CONTENTS}") - string(REPLACE "\"builtin-baseline\":" "" VCPKG_BASELINE "${VCPKG_BASELINE}") - string(REPLACE "\"" "" VCPKG_BASELINE "${VCPKG_BASELINE}") - endif() - - if(NOT "${VCPKG_BASELINE}" EQUAL "") - if(NOT "${VCPKG_VERSION}" EQUAL "" AND DEFINED VCPKG_VERSION) - message(WARNING "VCPKG_VERSION was specified, but vcpkg.json manifest is used and specifies a builtin-baseline; using builtin-baseline: ${VCPKG_BASELINE}") - endif() - set(VCPKG_VERSION_EXPLICIT "${VCPKG_BASELINE}") - message(STATUS "Using VCPKG Version: ") - endif() - endif() - - if("${VCPKG_VERSION_EXPLICIT}" STREQUAL "latest" OR "${VCPKG_VERSION_EXPLICIT}" EQUAL "" OR NOT DEFINED VCPKG_VERSION_EXPLICIT) - set(VCPKG_VERSION_CHECKOUT ${VCPKG_GIT_TAG_LATEST}) - message(STATUS "Using VCPKG Version: ${VCPKG_VERSION_EXPLICIT} (latest)") - elseif("${VCPKG_VERSION_EXPLICIT}" STREQUAL "edge" OR "${VCPKG_VERSION_EXPLICIT}" STREQUAL "master") - set(VCPKG_VERSION_CHECKOUT "master") - message(STATUS "Using VCPKG Version: edge (latest commit)") - else() - message(STATUS "Using VCPKG Version: ${VCPKG_VERSION_EXPLICIT}") - set(VCPKG_VERSION_CHECKOUT ${VCPKG_VERSION_EXPLICIT}) - endif() - - set(VCPKG_VERSION_CHECKOUT ${VCPKG_VERSION_CHECKOUT} PARENT_SCOPE) -endfunction() - -# sets VCPKG_PLATFORM_MUSL_LIBC(ON|OFF) -function(vcpkg_get_set_musl_libc) - if(WIN32 OR APPLE) - # is windows - set(VCPKG_PLATFORM_MUSL_LIBC OFF) - else() - execute_process(COMMAND getconf GNU_LIBC_VERSION RESULT_VARIABLE VCPKG_PLATFORM_GLIBC) - if(VCPKG_PLATFORM_GLIBC EQUAL "0") - # has glibc - set(VCPKG_PLATFORM_MUSL_LIBC OFF) - else() - execute_process(COMMAND ldd --version RESULT_VARIABLE VCPKG_PLATFORM_LDD_OK OUTPUT_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDOUT ERROR_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDERR) - string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" VCPKG_PLATFORM_LDD_VERSION_STDOUT) - string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" VCPKG_PLATFORM_LDD_VERSION_STDERR) - string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" "musl" VCPKG_PLATFORM_LDD_FIND_MUSL_STDOUT) - string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" "musl" VCPKG_PLATFORM_LDD_FIND_MUSL_STDERR) - if( - (VCPKG_PLATFORM_LDD_OK EQUAL "0" AND NOT VCPKG_PLATFORM_LDD_FIND_MUSL_STDOUT EQUAL "-1") OR - (NOT VCPKG_PLATFORM_LDD_OK EQUAL "0" AND NOT VCPKG_PLATFORM_LDD_FIND_MUSL_STDERR EQUAL "-1") - ) - # has musl-libc - # use system binaries - set(VCPKG_PLATFORM_MUSL_LIBC ON) - message(STATUS "VCPKG: System is using musl-libc; using system binaries! (e.g. cmake, curl, zip, tar, etc.)") - else() - # has error... - message(FATAL_ERROR "VCPKG: could detect neither glibc nor musl-libc!") - endif() - endif() - endif() - - # propagate back - set(VCPKG_PLATFORM_MUSL_LIBC ${VCPKG_PLATFORM_MUSL_LIBC} PARENT_SCOPE) -endfunction() - - -# configure environment and CMake variables to mask musl-libc from vcpkg triplet checks -function(vcpkg_mask_musl_libc) - # set target triplet without '-musl' - execute_process(COMMAND ldd --version RESULT_VARIABLE VCPKG_PLATFORM_LDD_OK OUTPUT_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDOUT ERROR_VARIABLE VCPKG_PLATFORM_LDD_VERSION_STDERR) - string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" VCPKG_PLATFORM_LDD_VERSION_STDOUT) - string(TOLOWER "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" VCPKG_PLATFORM_LDD_VERSION_STDERR) - string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDOUT}" "x86_64" VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDOUT) - string(FIND "${VCPKG_PLATFORM_LDD_VERSION_STDERR}" "x86_64" VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDERR) - if( - NOT VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDOUT EQUAL "-1" OR - NOT VCPKG_PLATFORM_LDD_FIND_MUSL_BITS_STDERR EQUAL "-1" - ) - set(VCPKG_TARGET_TRIPLET "x64-linux") - else() - set(VCPKG_TARGET_TRIPLET "x86-linux") - endif() - - set(ENV{VCPKG_DEFAULT_TRIPLET} "${VCPKG_TARGET_TRIPLET}") - set(ENV{VCPKG_DEFAULT_HOST_TRIPLET} "${VCPKG_TARGET_TRIPLET}") - set(VCPKG_TARGET_TRIPLET "${VCPKG_TARGET_TRIPLET}" CACHE STRING "vcpkg default target triplet (possibly dont change)") - message(STATUS "VCPKG: System is using musl-libc; fixing default target triplet as: ${VCPKG_TARGET_TRIPLET}") - - set(VCPKG_MASK_MUSL_LIBC ON CACHE INTERNAL "masked musl-libc") -endfunction() - -# automate musl-libc masking -function(vcpkg_mask_if_musl_libc) - vcpkg_get_set_musl_libc() - if(VCPKG_PLATFORM_MUSL_LIBC) - vcpkg_mask_musl_libc() - endif() -endfunction() - -# sets VCPKG_USE_SYSTEM_BINARIES_FLAG from VCPKG_PLATFORM_MUSL_LIBC and/or VCPKG_FORCE_SYSTEM_BINARIES -# vcpkg_set_use_system_binaries_flag([VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT]) -function(vcpkg_set_use_system_binaries_flag) - if(ARGV0 EQUAL "" OR NOT DEFINED ARGV0) - set(VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT ${VCPKG_FORCE_SYSTEM_BINARIES}) - else() - set(VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT ${ARGV0}) - endif() - - vcpkg_get_set_musl_libc() - - if(NOT WIN32 AND (VCPKG_FORCE_SYSTEM_BINARIES_EXPLICIT OR VCPKG_PLATFORM_MUSL_LIBC) ) - set(VCPKG_USE_SYSTEM_BINARIES_FLAG "--useSystemBinaries" PARENT_SCOPE) - # has to be propagated to all install calls - set(ENV{VCPKG_FORCE_SYSTEM_BINARIES} "1") - set(VCPKG_FORCE_SYSTEM_BINARIES ON CACHE BOOL "force vcpkg to use system binaries (possibly dont change)") - - message(STATUS "VCPKG: Requested use of system binaries! (e.g. cmake, curl, zip, tar, etc.)") - else() - set(VCPKG_USE_SYSTEM_BINARIES_FLAG "" PARENT_SCOPE) - endif() -endfunction() - - -# install package -function(vcpkg_add_package PKG_NAME) - # if(VCPKG_TARGET_TRIPLET STREQUAL "" OR NOT DEFINED VCPKG_TARGET_TRIPLET) - # vcpkg_make_set_triplet() - # endif() - set(VCPKG_TARGET_TRIPLET_FLAG "") - if(DEFINED VCPKG_TARGET_TRIPLET AND NOT VCPKG_TARGET_TRIPLET EQUAL "") - set(VCPKG_TARGET_TRIPLET_FLAG "--triplet=${VCPKG_TARGET_TRIPLET}") - endif() - - message(STATUS "VCPKG: fetching ${PKG_NAME} via vcpkg_add_package") - execute_process(COMMAND ${VCPKG_EXECUTABLE} ${VCPKG_TARGET_TRIPLET_FLAG} ${VCPKG_RECURSE_REBUILD_FLAG} --feature-flags=-manifests --disable-metrics install "${PKG_NAME}" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} RESULT_VARIABLE VCPKG_INSTALL_OK) - if(NOT VCPKG_INSTALL_OK EQUAL "0") - message(FATAL_ERROR "VCPKG: failed fetching ${PKG_NAME}! Did you call vcpkg_init(<...>)?") - else() - # add package to automatically generated manifest - vcpkg_manifest_generation_add_dependency("${PKG_NAME}") - endif() -endfunction() - - -# install packages from manifest in script mode -function(vcpkg_install_manifest) - # if(VCPKG_TARGET_TRIPLET STREQUAL "" OR NOT DEFINED VCPKG_TARGET_TRIPLET) - # vcpkg_make_set_triplet() - # endif() - # message(STATUS "VCPKG: install from manifest; using target triplet: ${VCPKG_TARGET_TRIPLET}") - # execute_process(COMMAND ${VCPKG_EXECUTABLE} --triplet=${VCPKG_TARGET_TRIPLET} --feature-flags=manifests,versions --disable-metrics install WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} RESULT_VARIABLE VCPKG_INSTALL_OK) - get_filename_component(VCPKG_EXECUTABLE_ABS ${VCPKG_EXECUTABLE} ABSOLUTE) - file(COPY "./vcpkg.json" DESTINATION "${VCPKG_PARENT_DIR}") - execute_process(COMMAND ${VCPKG_EXECUTABLE_ABS} --feature-flags=manifests,versions --disable-metrics install WORKING_DIRECTORY "${VCPKG_PARENT_DIR}" RESULT_VARIABLE VCPKG_INSTALL_OK) - if(NOT VCPKG_INSTALL_OK EQUAL "0") - message(FATAL_ERROR "VCPKG: install from manifest failed") - endif() -endfunction() - -## manifest generation requires CMake > 3.19 -function(vcpkg_manifest_generation_update_cache VCPKG_GENERATED_MANIFEST) - string(REGEX REPLACE "\n" "" VCPKG_GENERATED_MANIFEST "${VCPKG_GENERATED_MANIFEST}") - set(VCPKG_GENERATED_MANIFEST "${VCPKG_GENERATED_MANIFEST}" CACHE STRING "template for automatically generated manifest by vcpkg-cmake-integration" FORCE) - mark_as_advanced(FORCE VCPKG_GENERATED_MANIFEST) -endfunction() - - -# build empty json manifest and register deferred call to finalize and write -function(vcpkg_manifest_generation_init) - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) - # init "empty" json and cache variable - set(VCPKG_GENERATED_MANIFEST "{}") - - # initialize dependencies as empty list - # first vcpkg_add_package will transform to object and install finalization handler - # transform to list in finalization step - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "[]") - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" "$schema" "\"https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json\"") - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" version "\"0.1.0-autogenerated\"") - - # write baseline commit - execute_process(COMMAND git log --pretty=format:'%H' -1 WORKING_DIRECTORY "${VCPKG_DIRECTORY}" OUTPUT_VARIABLE VCPKG_GENERATED_MANIFEST_BASELINE) - string(REPLACE "'" "" VCPKG_GENERATED_MANIFEST_BASELINE "${VCPKG_GENERATED_MANIFEST_BASELINE}") - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" builtin-baseline "\"${VCPKG_GENERATED_MANIFEST_BASELINE}\"") - - vcpkg_manifest_generation_update_cache("${VCPKG_GENERATED_MANIFEST}") - - # will be initialized from vcpkg_add_package call - # # defer call to finalize manifest - # # needs to be called later as project variables are not set when initializing - # cmake_language(DEFER CALL vcpkg_manifest_generation_finalize) - endif() -endfunction() - -# add dependency to generated manifest -function(vcpkg_manifest_generation_add_dependency PKG_NAME) - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) - # extract features - string(REGEX MATCH "\\[.*\\]" PKG_FEATURES "${PKG_NAME}") - string(REPLACE "${PKG_FEATURES}" "" PKG_BASE_NAME "${PKG_NAME}") - # make comma separated list - string(REPLACE "[" "" PKG_FEATURES "${PKG_FEATURES}") - string(REPLACE "]" "" PKG_FEATURES "${PKG_FEATURES}") - string(REPLACE " " "" PKG_FEATURES "${PKG_FEATURES}") - # build cmake list by separating with ; - string(REPLACE "," ";" PKG_FEATURES "${PKG_FEATURES}") - - if(NOT PKG_FEATURES) - # set package name string only - set(PKG_DEPENDENCY_JSON "\"${PKG_BASE_NAME}\"") - else() - # build dependency object with features - set(PKG_DEPENDENCY_JSON "{}") - string(JSON PKG_DEPENDENCY_JSON SET "${PKG_DEPENDENCY_JSON}" name "\"${PKG_BASE_NAME}\"") - - set(FEATURE_LIST_JSON "[]") - foreach(FEATURE IN LISTS PKG_FEATURES) - if(FEATURE STREQUAL "core") - # set default feature option if special feature "core" is specified - string(JSON PKG_DEPENDENCY_JSON SET "${PKG_DEPENDENCY_JSON}" default-features "false") - else() - # add feature to list - string(JSON FEATURE_LIST_JSON_LEN LENGTH "${FEATURE_LIST_JSON}") - string(JSON FEATURE_LIST_JSON SET "${FEATURE_LIST_JSON}" ${FEATURE_LIST_JSON_LEN} "\"${FEATURE}\"") - endif() - endforeach() - - # build dependency object with feature list - string(JSON PKG_DEPENDENCY_JSON SET "${PKG_DEPENDENCY_JSON}" features "${FEATURE_LIST_JSON}") - endif() - - # add dependency to manifest - # reset to empty object to avoid collissions and track new packages - # defer (new) finalization call - string(JSON VCPKG_GENERATED_MANIFEST_DEPENDENCIES_TYPE TYPE "${VCPKG_GENERATED_MANIFEST}" dependencies) - if(VCPKG_GENERATED_MANIFEST_DEPENDENCIES_TYPE STREQUAL "ARRAY") - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "{}") - cmake_language(DEFER CALL vcpkg_manifest_generation_finalize) - endif() - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "${PKG_BASE_NAME}" "${PKG_DEPENDENCY_JSON}") - - vcpkg_manifest_generation_update_cache("${VCPKG_GENERATED_MANIFEST}") - endif() -endfunction() - - -# build empty json manifest and register deferred call to finalize and write -function(vcpkg_manifest_generation_finalize) - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) - # populate project information - string(REGEX REPLACE "[^a-z0-9\\.-]" "" VCPKG_GENERATED_MANIFEST_NAME "${PROJECT_NAME}") - string(TOLOWER VCPKG_GENERATED_MANIFEST_NAME "${VCPKG_GENERATED_MANIFEST_NAME}") - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" name "\"${VCPKG_GENERATED_MANIFEST_NAME}\"") - if(NOT PROJECT_VERSION EQUAL "" AND DEFINED PROJECT_VERSION) - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" version "\"${PROJECT_VERSION}\"") - endif() - - vcpkg_manifest_generation_update_cache("${VCPKG_GENERATED_MANIFEST}") - - # make list from dependency dictionary - # cache dependency object - string(JSON VCPKG_GENERATED_DEPENDENCY_OBJECT GET "${VCPKG_GENERATED_MANIFEST}" dependencies) - # initialize dependencies as list - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies "[]") - - string(JSON VCPKG_GENERATED_DEPENDENCY_COUNT LENGTH "${VCPKG_GENERATED_DEPENDENCY_OBJECT}") - if(VCPKG_GENERATED_DEPENDENCY_COUNT GREATER 0) - # setup range stop for iteration - math(EXPR VCPKG_GENERATED_DEPENDENCY_LOOP_STOP "${VCPKG_GENERATED_DEPENDENCY_COUNT} - 1") - - # make list - foreach(DEPENDENCY_INDEX RANGE ${VCPKG_GENERATED_DEPENDENCY_LOOP_STOP}) - string(JSON DEPENDENCY_NAME MEMBER "${VCPKG_GENERATED_DEPENDENCY_OBJECT}" ${DEPENDENCY_INDEX}) - string(JSON DEPENDENCY_JSON GET "${VCPKG_GENERATED_DEPENDENCY_OBJECT}" "${DEPENDENCY_NAME}") - string(JSON DEPENDENCY_JSON_TYPE ERROR_VARIABLE DEPENDENCY_JSON_TYPE_ERROR_IGNORE TYPE "${DEPENDENCY_JSON}") - if(DEPENDENCY_JSON_TYPE STREQUAL "OBJECT") - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies ${DEPENDENCY_INDEX} "${DEPENDENCY_JSON}") - else() - string(JSON VCPKG_GENERATED_MANIFEST SET "${VCPKG_GENERATED_MANIFEST}" dependencies ${DEPENDENCY_INDEX} "\"${DEPENDENCY_JSON}\"") - endif() - endforeach() - endif() - - message(STATUS "VCPKG auto-generated manifest (${CMAKE_CURRENT_BINARY_DIR}/vcpkg.json):\n${VCPKG_GENERATED_MANIFEST}") - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.json" "${VCPKG_GENERATED_MANIFEST}") - endif() -endfunction() - - -# get vcpkg and configure toolchain -if(NOT VCPKG_NO_INIT) - vcpkg_init() -endif() \ No newline at end of file diff --git a/tuplex/codegen/CMakeLists.txt b/tuplex/codegen/CMakeLists.txt index 478ae715f..b543955b7 100755 --- a/tuplex/codegen/CMakeLists.txt +++ b/tuplex/codegen/CMakeLists.txt @@ -20,7 +20,48 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # this may make it easier but increases size of shared object tremendously set(LLVM_REQUIRED_COMPONENTS core orcjit nativecodegen native scalaropts objcarcopts passes) -find_package(LLVM CONFIG REQUIRED) +IF(BREW_FOUND) + IF(APPLE) + + # there might be multiple LLVM versions installed. + # check which version there is + # if not sys.stdin.isatty(): + # data = sys.stdin.readlines() + + # could use brew prefix here, but let's leave it like this + EXECUTE_PROCESS(COMMAND bash "-c" "brew info llvm | grep Cellar | cut -d ' ' -f 1" OUTPUT_VARIABLE LLVM_ROOT_DIR RESULT_VARIABLE BREW_LLVM_NOTFOUND OUTPUT_STRIP_TRAILING_WHITESPACE) + IF(NOT BREW_LLVM_NOTFOUND EQUAL "0") + MESSAGE(WARNING "did not find llvm, you might install it via `brew install llvm@9`") + ELSE() + # check version, needs to be within 5 and 9 incl. + # i.e. execute something like /usr/local/opt/llvm/bin/llvm-config --version + EXECUTE_PROCESS(COMMAND ${LLVM_ROOT_DIR}/bin/llvm-config --version OUTPUT_VARIABLE LLVM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + + # check if empty, if it is parse again using brew info json + IF("${LLVM_VERSION}" STREQUAL "") + EXECUTE_PROCESS(COMMAND bash "-c" "brew info --json=v1 llvm | python3 -c 'import sys,json; x=json.load(sys.stdin); print(x[0][\"versions\"][\"stable\"])'" OUTPUT_VARIABLE LLVM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND bash "-c" "brew info llvm | grep Cellar | cut -d ' ' -f 1" OUTPUT_VARIABLE LLVM_ROOT_DIR RESULT_VARIABLE BREW_RET OUTPUT_STRIP_TRAILING_WHITESPACE) + ENDIF() + + message(STATUS "Found LLVM ${LLVM_VERSION}") + ENDIF() + + ELSEIF(UNIX) + # ... + ENDIF() +ENDIF() + +# for brewed llvm, add to cmakemodulepath +IF(NOT "${LLVM_ROOT_DIR}" STREQUAL "") + message(STATUS "Detected LLVM root dir: ${LLVM_ROOT_DIR}") + # make cmake find in config mode the right LLVMConfig.cmake file which is located here + set(LLVM_DIR "${LLVM_ROOT_DIR}/lib/cmake/llvm") + FIND_PACKAGE(LLVM 6.0 REQUIRED COMPONENTS ${LLVM_REQUIRED_COMPONENTS}) +ELSE() + FIND_PACKAGE(LLVM 6.0 REQUIRED COMPONENTS ${LLVM_REQUIRED_COMPONENTS}) +ENDIF() + +MESSAGE(STATUS "Found LLVM ${LLVM_VERSION_STRING}") if(LLVM_DIR) message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") endif() @@ -107,5 +148,4 @@ target_link_libraries(libcodegen ${LLVM_LIBRARIES} ${ZLIB_LIBRARIES} ${CURSES_LIBRARIES} - fmt::fmt ) \ No newline at end of file diff --git a/tuplex/codegen/include/IteratorContextProxy.h b/tuplex/codegen/include/IteratorContextProxy.h index a87e862e4..685a0695c 100644 --- a/tuplex/codegen/include/IteratorContextProxy.h +++ b/tuplex/codegen/include/IteratorContextProxy.h @@ -127,70 +127,6 @@ namespace tuplex { llvm::Value *iterator, const std::shared_ptr &iteratorInfo); -// /*! -// * Update index for a zip iterator in preparing for the getIteratorNextElement call by calling updateIteratorIndex on each argument. -// * If any argument is exhausted, return true and stop calling updateIteratorIndex on rest of the arguments. -// * Only return false if none of the argument iterators is exhausted. -// * @param builder -// * @param iterator -// * @param iteratorInfo -// * @return true if iterator is exhausted (getIteratorNextElement should not get called later), otherwise false -// */ -// llvm::Value *updateZipIndex(const codegen::IRBuilder& builder, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo); -// -// /*! -// * Generate the next element of a zip iterator. -// * Only to be called after calling updateIteratorIndex. -// * @param builder -// * @param yieldType -// * @param iterator -// * @param iteratorInfo -// * @return tuple element of yieldType -// */ -// SerializableValue getZipNextElement(const codegen::IRBuilder& builder, -// const python::Type &yieldType, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo); -// -// /*! -// * Generate the next element of a enumerate iterator. -// * Only to be called after calling updateIteratorIndex. -// * @param builder -// * @param iterator -// * @param iteratorInfo -// * @return true if iterator is exhausted (getIteratorNextElement should not get called later), otherwise false -// */ -// llvm::Value *updateEnumerateIndex(const codegen::IRBuilder& builder, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo); -// -// /*! -// * Generate the next element of a enumerate iterator. -// * Only to be called after calling updateIteratorIndex. -// * @param builder -// * @param yieldType -// * @param iterator -// * @param iteratorInfo -// * @return tuple element of yieldType -// */ -// SerializableValue getEnumerateNextElement(const codegen::IRBuilder& builder, -// const python::Type &yieldType, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo); -// -// /*! -// * Increment index field of a list/string/tuple iterator by offset. -// * Increment index field of a range iterator by step * offset. -// * Decrement index field of any reverseiterator by offset. -// * For zip and enumerate, will use recursive calls on their arguments until a list/string/tuple iterator or a reverseiterator is reached. -// * @param builder -// * @param iterator -// * @param iteratorInfo -// * @param offset can be negative -// */ -// void incrementIteratorIndex(const codegen::IRBuilder& builder, llvm::Value *iterator, const std::shared_ptr &iteratorInfo, int offset); }; /*! diff --git a/tuplex/codegen/src/IteratorContextProxy.cc b/tuplex/codegen/src/IteratorContextProxy.cc index c5321e4ed..e09401a6d 100644 --- a/tuplex/codegen/src/IteratorContextProxy.cc +++ b/tuplex/codegen/src/IteratorContextProxy.cc @@ -465,201 +465,6 @@ namespace tuplex { return next_from_iterator(*_env, builder, yieldType, iterator, iteratorInfo); } -// llvm::Value *IteratorContextProxy::updateZipIndex(const codegen::IRBuilder& builder, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo) { -// // deprecated -// return nullptr; -//// using namespace llvm; -//// -//// auto argsType = iteratorInfo->argsType; -//// auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; -//// -//// int zipSize = argsType.parameters().size(); -//// if(zipSize == 0) { -//// return _env->i1Const(true); -//// } -//// -//// BasicBlock *currBB = builder.GetInsertBlock(); -//// BasicBlock *exhaustedBB = BasicBlock::Create(_env->getContext(), "exhaustedBB", currBB->getParent()); -//// BasicBlock *endBB = BasicBlock::Create(_env->getContext(), "endBB", currBB->getParent()); -//// -//// builder.SetInsertPoint(exhaustedBB); -//// builder.CreateBr(endBB); -//// -//// builder.SetInsertPoint(endBB); -//// // zipExhausted indicates whether the given zip iterator is exhausted -//// auto zipExhausted = builder.CreatePHI(_env->i1Type(), 2); -//// zipExhausted->addIncoming(_env->i1Const(true), exhaustedBB); -//// -//// std::vector zipElementEntryBB; -//// std::vector zipElementCondBB; -//// for (int i = 0; i < zipSize; ++i) { -//// BasicBlock *currElementEntryBB = BasicBlock::Create(_env->getContext(), "zipElementBB" + std::to_string(i), currBB->getParent()); -//// BasicBlock *currElementCondBB = BasicBlock::Create(_env->getContext(), "currCondBB" + std::to_string(i), currBB->getParent()); -//// zipElementEntryBB.push_back(currElementEntryBB); -//// zipElementCondBB.push_back(currElementCondBB); -//// } -//// zipExhausted->addIncoming(_env->i1Const(false), zipElementCondBB[zipSize - 1]); -//// -//// builder.SetInsertPoint(currBB); -//// builder.CreateBr(zipElementEntryBB[0]); -//// // iterate over all arg iterators -//// // if the current arg iterator is exhausted, jump directly to exhaustedBB and zipExhausted will be set to true -//// for (int i = 0; i < zipSize; ++i) { -//// builder.SetInsertPoint(zipElementEntryBB[i]); -//// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); -//// auto currIterator = builder.CreateLoad(currIteratorPtr); -//// auto currIteratorInfo = argsIteratorInfo[i]; -//// assert(currIteratorInfo); -//// auto exhausted = updateIteratorIndex(builder, currIterator, currIteratorInfo); -//// builder.CreateBr(zipElementCondBB[i]); -//// builder.SetInsertPoint(zipElementCondBB[i]); -//// if(i == zipSize - 1) { -//// builder.CreateCondBr(exhausted, exhaustedBB, endBB); -//// } else { -//// builder.CreateCondBr(exhausted, exhaustedBB, zipElementEntryBB[i+1]); -//// } -//// } -//// builder.SetInsertPoint(endBB); -//// -//// return zipExhausted; -// } -// -// SerializableValue IteratorContextProxy::getZipNextElement(const codegen::IRBuilder& builder, -// const python::Type &yieldType, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo) { -// -// // deprecated -// return {}; -//// using namespace llvm; -//// auto argsType = iteratorInfo->argsType; -//// auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; -//// -//// FlattenedTuple ft(_env); -//// ft.init(yieldType); -//// -//// // previously UpdateIteratorIndexFunction was called on each arg iterator which increments index of each arg iterator by 1 -//// // restore index for all arg iterators -//// incrementIteratorIndex(builder, iterator, iteratorInfo, -1); -//// for (int i = 0; i < argsType.parameters().size(); ++i) { -//// auto currIteratorInfo = argsIteratorInfo[i]; -//// auto llvm_curr_iterator_type = createIteratorContextTypeFromIteratorInfo(*_env, *currIteratorInfo.get()); -//// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); -//// auto currIterator = builder.CreateLoad(llvm_curr_iterator_type->getPointerTo(), currIteratorPtr); -//// -//// // update current arg iterator index before fetching value -//// incrementIteratorIndex(builder, currIterator, currIteratorInfo, 1); -//// auto currIteratorNextVal = getIteratorNextElement(builder, yieldType.parameters()[i], currIterator, currIteratorInfo); -//// ft.setElement(builder, i, currIteratorNextVal.val, currIteratorNextVal.size, currIteratorNextVal.is_null); -//// } -//// auto retVal = ft.getLoad(builder); -//// auto retSize = ft.getSize(builder); -//// return SerializableValue(retVal, retSize); -// } -// -// llvm::Value *IteratorContextProxy::updateEnumerateIndex(const codegen::IRBuilder& builder, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo) { -// using namespace llvm; -// -// auto argIteratorInfo = iteratorInfo->argsIteratorInfo.front(); -// auto argIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); -// auto argIterator = builder.CreateLoad(argIteratorPtr); -// auto enumerateExhausted = updateIteratorIndex(builder, argIterator, argIteratorInfo); -// -// return enumerateExhausted; -// } -// -// SerializableValue IteratorContextProxy::getEnumerateNextElement(const codegen::IRBuilder& builder, -// const python::Type &yieldType, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo) { -// // deprecated -// return nullptr; -//// using namespace llvm; -//// -//// auto argIteratorInfo = iteratorInfo->argsIteratorInfo.front(); -//// -//// FlattenedTuple ft(_env); -//// ft.init(yieldType); -//// auto startValPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(0)}); -//// auto startVal = builder.CreateLoad(startValPtr); -//// auto start = SerializableValue(startVal, _env->i64Const(8)); -//// auto argIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); -//// auto argIterator = builder.CreateLoad(argIteratorPtr); -//// auto val = getIteratorNextElement(builder, yieldType.parameters()[1], argIterator, argIteratorInfo); -//// ft.setElement(builder, 0, start.val, start.size, start.is_null); -//// ft.setElement(builder, 1, val.val, val.size, val.is_null); -//// auto retVal = ft.getLoad(builder); -//// auto retSize = ft.getSize(builder); -//// // increment start index value -//// auto newStartVal = builder.CreateAdd(startVal, _env->i64Const(1)); -//// builder.CreateStore(newStartVal, startValPtr); -//// -//// return SerializableValue(retVal, retSize); -// } - -// void IteratorContextProxy::incrementIteratorIndex(const codegen::IRBuilder& builder, -// llvm::Value *iterator, -// const std::shared_ptr &iteratorInfo, -// int offset) { -// using namespace llvm; -// -// auto iteratorName = iteratorInfo->iteratorName; -// auto argsIteratorInfo = iteratorInfo->argsIteratorInfo; -// -// if(iteratorName == "zip") { -// for (int i = 0; i < argsIteratorInfo.size(); ++i) { -// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(i)}); -// -// // get iterator type -// auto llvm_iterator_type = createIteratorContextTypeFromIteratorInfo(*_env, *argsIteratorInfo[i]); -// -// auto currIterator = builder.CreateLoad(llvm_iterator_type->getPointerTo(), currIteratorPtr); -// incrementIteratorIndex(builder, currIterator, argsIteratorInfo[i], offset); -// } -// return; -// } -// -// if(iteratorName == "enumerate") { -// auto currIteratorPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); -// auto currIterator = builder.CreateLoad(currIteratorPtr); -// incrementIteratorIndex(builder, currIterator, argsIteratorInfo.front(), offset); -// return; -// } -// -// auto iterablesType = iteratorInfo->argsType; -// if(iteratorName == "iter") { -// if(iterablesType.isIteratorType()) { -// // iter() call on an iterator, ignore the outer iter and call again -// assert(argsIteratorInfo.front()); -// incrementIteratorIndex(builder, iterator, argsIteratorInfo.front(), offset); -// return; -// } -// } else if(iteratorName == "reversed") { -// // for reverseiterator, need to decrement index by offset -// offset = -offset; -// } else { -// throw std::runtime_error("unsupported iterator" + iteratorName); -// } -// -// // change index field -// auto indexPtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(1)}); -// auto currIndex = builder.CreateLoad(builder.getInt32Ty(), indexPtr); -// if(iterablesType == python::Type::RANGE) { -// // index will change by offset * step -// auto rangePtr = builder.CreateGEP(iterator, {_env->i32Const(0), _env->i32Const(2)}); -// auto range = builder.CreateLoad(rangePtr); -// auto stepPtr = builder.CreateGEP(_env->getRangeObjectType(), range, {_env->i32Const(0), _env->i32Const(2)}); -// auto step = builder.CreateLoad(stepPtr); -// builder.CreateStore(builder.CreateAdd(currIndex, builder.CreateMul(_env->i64Const(offset), step)), indexPtr); -// } else { -// builder.CreateStore(builder.CreateAdd(currIndex, _env->i32Const(offset)), indexPtr); -// } -// } - // helper to retrieve iteratorcontexttype from iteratorInfo llvm::Type* createIteratorContextTypeFromIteratorInfo(LLVMEnvironment& env, const IteratorInfo& iteratorInfo) { // coupled with FunctionRegistry diff --git a/tuplex/core/CMakeLists.txt b/tuplex/core/CMakeLists.txt index 2652c5c2d..e5f323112 100755 --- a/tuplex/core/CMakeLists.txt +++ b/tuplex/core/CMakeLists.txt @@ -18,7 +18,10 @@ if(BUILD_WITH_AWS) # is installed # set(Protobuf_USE_STATIC_LIBS ON) # https://github.com/protocolbuffers/protobuf/issues/12637 - find_package(Protobuf REQUIRED) + find_package(Protobuf CONFIG) + if(NOT Protobuf_FOUND) + find_package(Protobuf REQUIRED) + endif() include_directories(Protobuf_INCLUDE_DIRS) add_library(proto-objects OBJECT "${CMAKE_CURRENT_LIST_DIR}/proto/Lambda.proto") diff --git a/tuplex/runtime/CMakeLists.txt b/tuplex/runtime/CMakeLists.txt index eb70a12fd..6385e6dbc 100644 --- a/tuplex/runtime/CMakeLists.txt +++ b/tuplex/runtime/CMakeLists.txt @@ -18,7 +18,7 @@ message(STATUS "Tuplex python language runtime include dir is: ${RUNTIME_INCLUDE target_include_directories(runtime PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${PCRE2_INCLUDE_DIRS}) # keep dependencies lean... -target_link_libraries(runtime libutils ${PCRE2_LIBRARIES} fmt::fmt) +target_link_libraries(runtime libutils ${PCRE2_LIBRARIES}) # require thread_local and aligned malloc keyword (C11 or C++11) target_compile_features(runtime PRIVATE cxx_thread_local) diff --git a/tuplex/vcpkg.json b/tuplex/vcpkg.json deleted file mode 100644 index 5ab426a41..000000000 --- a/tuplex/vcpkg.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", - "dependencies": [ - "boost-system", - "fmt", - "spdlog", - {"name" : "aws-sdk-cpp", "features": ["s3", "lambda", "transfer"]}, - {"name": "llvm", "features": ["enable-rtti", "enable-zlib", "enable-zstd", "target-aarch64", "target-x86"]}, - {"name": "protobuf", "features": ["zlib"]} - ], - "overrides": [ - { - "name": "fmt", - "version": "10.1.1" - }, - {"name": "llvm", - "version": "17.0.2" - } - ], - "builtin-baseline": "3265c187c74914aa5569b75355badebfdbab7987" -}