From 6ce9b7afaf184df37480c4e5da8a27d414e25efd Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Wed, 2 Jul 2025 16:15:35 +0100 Subject: [PATCH 01/61] v0.6.1 (#463) Update core version to v0.2.9 --- CHANGELOG.md | 5 +++++ go.mod | 4 ++-- go.sum | 8 ++++---- internal/constants/plugin.go | 2 +- tests/acceptance/test_files/file_source.bats | 5 ++++- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e37e6708..185cf1ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ +## v0.6.1 [2025-07-02] +_Bug fixes_ +* Update core version to v0.2.9 - fix issue where collection state is not being saved for zero granularity collection. ([#251](https://github.com/turbot/tailpipe-plugin-sdk/issues/251)) + + ## v0.6.0 [2025-07-02] _What's new_ * Add `--to` flag for `collect`, allowing collection of standalone time ranges. ([#238](https://github.com/turbot/tailpipe/issues/238)) diff --git a/go.mod b/go.mod index 3689e77a..41046e90 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 github.com/turbot/pipe-fittings/v2 v2.5.2 - github.com/turbot/tailpipe-plugin-sdk v0.9.0 + github.com/turbot/tailpipe-plugin-sdk v0.9.1 github.com/zclconf/go-cty v1.14.4 golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c @@ -41,7 +41,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/marcboeker/go-duckdb/v2 v2.1.0 github.com/thediveo/enumflag/v2 v2.0.5 - github.com/turbot/tailpipe-plugin-core v0.2.8 + github.com/turbot/tailpipe-plugin-core v0.2.9 golang.org/x/sync v0.12.0 golang.org/x/text v0.23.0 google.golang.org/grpc v1.69.2 diff --git a/go.sum b/go.sum index d4f2ee82..0d18b9fc 100644 --- a/go.sum +++ b/go.sum @@ -773,10 +773,10 @@ github.com/turbot/pipe-fittings/v2 v2.5.2 h1:qrQQEnD6dmMSQFUw8cKw6/Rb1sWolMru0+x github.com/turbot/pipe-fittings/v2 v2.5.2/go.mod h1:szte433cBDCaZcGe5zMVGG7uTl9HMaEYaQmuvzZRYIQ= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= -github.com/turbot/tailpipe-plugin-core v0.2.8 h1:cStFddCthf/Pwr5M9tEl3k/v5LZg90t2L0umyrt7eNY= -github.com/turbot/tailpipe-plugin-core v0.2.8/go.mod h1://Rab0muTCHXVQlv118bCjTqzvQskcAVm71VYdcJgT8= -github.com/turbot/tailpipe-plugin-sdk v0.9.0 h1:4WTxBdSimiQENVZ0DtzgDOaBLk9k3mHb6YHZXRnS+7M= -github.com/turbot/tailpipe-plugin-sdk v0.9.0/go.mod h1:ch/GVDaLoyau6LCUPbx/aYprNi7+DKhmwPDM0M1QJbM= +github.com/turbot/tailpipe-plugin-core v0.2.9 h1:ntpGASgFpGbARK4RCVsIt96M5IlBY1//JBRr8H9IPVo= +github.com/turbot/tailpipe-plugin-core v0.2.9/go.mod h1:BQb0sF9CJIo93xDdPuOSqCUH19dz3QpG+NrdyKbYM/c= +github.com/turbot/tailpipe-plugin-sdk v0.9.1 h1:MaoN547oqmwPOWa817niKqtif/XMuyl61S1Ykq34LAk= +github.com/turbot/tailpipe-plugin-sdk v0.9.1/go.mod h1:ch/GVDaLoyau6LCUPbx/aYprNi7+DKhmwPDM0M1QJbM= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= diff --git a/internal/constants/plugin.go b/internal/constants/plugin.go index 9b190501..b63c85fe 100644 --- a/internal/constants/plugin.go +++ b/internal/constants/plugin.go @@ -7,7 +7,7 @@ import ( const ( // MinCorePluginVersion should be set for production releases - it is the minimum version of the core plugin that is required - MinCorePluginVersion = "v0.2.8" + MinCorePluginVersion = "v0.2.9" // CorePluginVersion may be set for pre-release versions - it allows us to pin a pre-release version of the core plugin // NOTE: they must NOT both be set CorePluginVersion = "" diff --git a/tests/acceptance/test_files/file_source.bats b/tests/acceptance/test_files/file_source.bats index 66d4e318..d9ab6c23 100644 --- a/tests/acceptance/test_files/file_source.bats +++ b/tests/acceptance/test_files/file_source.bats @@ -29,7 +29,8 @@ EOF } @test "verify file source with multiple paths" { - # Create a second directory with the same files for testing multiple paths + skip "TODO - This test is not working as expected. It needs to be fixed before it can be run." + # Create a second directory with the same files for testing multiple paths mkdir -p $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ cp $SOURCE_FILES_DIR/aws_cloudtrail_flaws/* $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ @@ -61,6 +62,7 @@ EOF } @test "verify file source with custom file layout" { + skip "TODO - This test is not working as expected. It needs to be fixed before it can be run." cat << EOF > $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc partition "aws_cloudtrail_log" "fs" { source "file" { @@ -88,6 +90,7 @@ EOF } @test "verify file source with custom patterns" { + skip "TODO - This test is not working as expected. It needs to be fixed before it can be run." cat << EOF > $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc partition "aws_cloudtrail_log" "fs" { source "file" { From 7796d434148579355239ac2e8d53496c2136c9ff Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Thu, 3 Jul 2025 17:13:38 +0530 Subject: [PATCH 02/61] Fix and re-enable file source acceptance tests --- tests/acceptance/test_files/file_source.bats | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/acceptance/test_files/file_source.bats b/tests/acceptance/test_files/file_source.bats index d9ab6c23..735d56cf 100644 --- a/tests/acceptance/test_files/file_source.bats +++ b/tests/acceptance/test_files/file_source.bats @@ -25,17 +25,16 @@ EOF 200000' # remove the config file - rm -rf $TAILPIPE_INSTALL_DIR/config/cloudtrail_logs.tpc + rm -f $TAILPIPE_INSTALL_DIR/config/cloudtrail_logs.tpc } @test "verify file source with multiple paths" { - skip "TODO - This test is not working as expected. It needs to be fixed before it can be run." # Create a second directory with the same files for testing multiple paths mkdir -p $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ cp $SOURCE_FILES_DIR/aws_cloudtrail_flaws/* $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ cat << EOF > $TAILPIPE_INSTALL_DIR/config/multi_path.tpc -partition "aws_cloudtrail_log" "fs" { +partition "aws_cloudtrail_log" "fs2" { source "file" { file_layout = ".json.gz" paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/", "$SOURCE_FILES_DIR/aws_cloudtrail_flaws2/"] @@ -44,9 +43,12 @@ partition "aws_cloudtrail_log" "fs" { EOF cat $TAILPIPE_INSTALL_DIR/config/multi_path.tpc + ls -al $SOURCE_FILES_DIR/aws_cloudtrail_flaws2 + + tailpipe plugin list # tailpipe collect - tailpipe collect aws_cloudtrail_log.fs --progress=false --from 2014-01-01 + tailpipe collect aws_cloudtrail_log.fs2 --progress=false --from 2014-01-01 # run tailpipe query and verify the row counts run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv @@ -57,14 +59,13 @@ EOF 400000' # remove the config file and test directory - rm -rf $TAILPIPE_INSTALL_DIR/config/multi_path.tpc + rm -f $TAILPIPE_INSTALL_DIR/config/multi_path.tpc rm -rf $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ } @test "verify file source with custom file layout" { - skip "TODO - This test is not working as expected. It needs to be fixed before it can be run." cat << EOF > $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc -partition "aws_cloudtrail_log" "fs" { +partition "aws_cloudtrail_log" "fs3" { source "file" { paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/"] file_layout = \`flaws_cloudtrail%{NUMBER:file_number}.json.gz\` @@ -75,7 +76,7 @@ EOF cat $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc # tailpipe collect - tailpipe collect aws_cloudtrail_log.fs --progress=false --from 2014-01-01 + tailpipe collect aws_cloudtrail_log.fs3 --progress=false --from 2014-01-01 # run tailpipe query and verify the row counts run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv @@ -86,13 +87,12 @@ EOF 200000' # remove the config file - rm -rf $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc + rm -f $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc } @test "verify file source with custom patterns" { - skip "TODO - This test is not working as expected. It needs to be fixed before it can be run." cat << EOF > $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc -partition "aws_cloudtrail_log" "fs" { +partition "aws_cloudtrail_log" "fs4" { source "file" { paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/"] file_layout = \`%{MY_PATTERN}.json.gz\` @@ -106,7 +106,7 @@ EOF cat $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc # tailpipe collect - tailpipe collect aws_cloudtrail_log.fs --progress=false --from 2014-01-01 + tailpipe collect aws_cloudtrail_log.fs4 --progress=false --from 2014-01-01 # run tailpipe query and verify the row counts run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv @@ -117,7 +117,7 @@ EOF 200000' # remove the config file - rm -rf $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc + rm -f $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc } function teardown() { From 91192cefe1cf31d2e39b8afa66af0ece790bc47b Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 8 Jul 2025 21:48:36 +0530 Subject: [PATCH 03/61] Add smoke test automation workflow and scripts (#469) --- .../workflows/02-tailpipe-smoke-tests.yaml | 194 ++++++++++++++++++ scripts/linux_container_info.sh | 7 + scripts/prepare_amazonlinux_container.sh | 20 ++ scripts/prepare_centos_container.sh | 20 ++ scripts/prepare_ubuntu_container.sh | 20 ++ scripts/smoke_test.sh | 78 +++++++ 6 files changed, 339 insertions(+) create mode 100644 .github/workflows/02-tailpipe-smoke-tests.yaml create mode 100755 scripts/linux_container_info.sh create mode 100755 scripts/prepare_amazonlinux_container.sh create mode 100755 scripts/prepare_centos_container.sh create mode 100755 scripts/prepare_ubuntu_container.sh create mode 100755 scripts/smoke_test.sh diff --git a/.github/workflows/02-tailpipe-smoke-tests.yaml b/.github/workflows/02-tailpipe-smoke-tests.yaml new file mode 100644 index 00000000..476fd73b --- /dev/null +++ b/.github/workflows/02-tailpipe-smoke-tests.yaml @@ -0,0 +1,194 @@ +name: "02 - Tailpipe: Smoke Tests" + +on: + workflow_dispatch: + inputs: + version: + description: "Version to test (with 'v' prefix, e.g., v1.0.0)" + required: true + type: string + +env: + # Version from input + VERSION: ${{ github.event.inputs.version }} + # Disable update checks during smoke tests + TAILPIPE_UPDATE_CHECK: false + + jobs: + smoke_test_ubuntu_24: + name: Smoke test (Ubuntu 24, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*linux_amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull Ubuntu latest Image + run: docker pull ubuntu:latest + + - name: Create and Start Ubuntu latest Container + run: | + docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec ubuntu-24-test /scripts/linux_container_info.sh + + - name: Install dependencies, create user, and assign necessary permissions + run: | + docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh + + - name: Run smoke tests + run: | + docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop ubuntu-24-test + docker rm ubuntu-24-test + + smoke_test_centos_9: + name: Smoke test (CentOS Stream 9, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*linux_amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull CentOS Stream 9 image + run: docker pull quay.io/centos/centos:stream9 + + - name: Create and Start CentOS stream9 Container + run: | + docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec centos-stream9-test /scripts/linux_container_info.sh + + - name: Install dependencies, create user, and assign necessary permissions + run: | + docker exec centos-stream9-test /scripts/prepare_centos_container.sh + + - name: Run smoke tests + run: | + docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop centos-stream9-test + docker rm centos-stream9-test + + smoke_test_amazonlinux: + name: Smoke test (Amazon Linux 2023, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*linux_amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull Amazon Linux 2023 Image + run: docker pull amazonlinux:2023 + + - name: Create and Start Amazon Linux 2023 Container + run: | + docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec amazonlinux-2023-test /scripts/linux_container_info.sh + + - name: Install dependencies, create user, and assign necessary permissions + run: | + docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh + + - name: Run smoke tests + run: | + docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop amazonlinux-2023-test + docker rm amazonlinux-2023-test + + smoke_test_darwin_arm64: + name: Smoke test (macOS 14, ARM64) + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Darwin Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*darwin_arm64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Darwin Artifacts and Install Binary + run: | + mkdir -p /tmp/tailpipe + tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe + sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ + sudo chmod +x /usr/local/bin/tailpipe + + - name: Install jq + run: | + brew install jq + + - name: Get runner/container info + run: | + uname -a + sw_vers + + - name: Run smoke tests + run: | + chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh + $GITHUB_WORKSPACE/scripts/smoke_test.sh diff --git a/scripts/linux_container_info.sh b/scripts/linux_container_info.sh new file mode 100755 index 00000000..0a16f884 --- /dev/null +++ b/scripts/linux_container_info.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# This is a script to get the information about the linux container. +# Used in release smoke tests. + +uname -a # uname information +cat /etc/os-release # OS version information +ldd --version # glibc version information \ No newline at end of file diff --git a/scripts/prepare_amazonlinux_container.sh b/scripts/prepare_amazonlinux_container.sh new file mode 100755 index 00000000..75273933 --- /dev/null +++ b/scripts/prepare_amazonlinux_container.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the Amazon Linux 2023 container. +# Used in release smoke tests. + +# update yum and install required packages +yum update -y +yum install -y tar ca-certificates jq + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_centos_container.sh b/scripts/prepare_centos_container.sh new file mode 100755 index 00000000..a6e4903b --- /dev/null +++ b/scripts/prepare_centos_container.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the CentOS Stream 9 container. +# Used in release smoke tests. + +# update yum and install required packages +yum update -y +yum install -y tar ca-certificates jq + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_ubuntu_container.sh b/scripts/prepare_ubuntu_container.sh new file mode 100755 index 00000000..216bc77c --- /dev/null +++ b/scripts/prepare_ubuntu_container.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the ubuntu 24 container. +# Used in release smoke tests. + +# update apt and install required packages +apt-get update +apt-get install -y tar ca-certificates jq + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 00000000..ec8fc523 --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# This is a script with set of commands to smoke test a tailpipe build. +# The plan is to gradually add more tests to this script. + +set -e + +# Ensure the PATH includes the directory where jq is installed +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin + +# Check jq is available +jq --version + +/usr/local/bin/tailpipe --version # check version + +# Test basic query functionality (should work without data) +/usr/local/bin/tailpipe query "SELECT 1 as smoke_test" # verify basic query works + +# Test connect functionality +DB_FILE=$(/usr/local/bin/tailpipe connect --output json | jq -r '.database_filepath') + +# Verify the database file exists +if [ -f "$DB_FILE" ]; then + echo "Database file exists" +else + echo "Database file not found: $DB_FILE" + exit 1 +fi + +# Test plugin installation +/usr/local/bin/tailpipe plugin install chaos # install chaos plugin for testing +/usr/local/bin/tailpipe plugin list # verify plugin is installed + +# Show available tables and sources after plugin installation +/usr/local/bin/tailpipe table list # should now show chaos tables +/usr/local/bin/tailpipe source list # should now show chaos sources + +# Create configuration for testing +# the config path is different for darwin and linux +if [ "$(uname -s)" = "Darwin" ]; then + CONFIG_DIR="/Users/runner/.tailpipe/config" +else + CONFIG_DIR="/home/tailpipe/.tailpipe/config" +fi + +mkdir -p "$CONFIG_DIR" + +# Create chaos.tpc configuration file +cat > "$CONFIG_DIR/chaos.tpc" << 'EOF' +partition "chaos_date_time" "chaos_date_time_range" { + source "chaos_date_time" { + row_count = 100 + } +} +EOF + +cat "$CONFIG_DIR/chaos.tpc" + +# Test partition listing after adding configuration +/usr/local/bin/tailpipe partition list # should now show the chaos partition + +# Show partition details +/usr/local/bin/tailpipe partition show chaos_date_time.chaos_date_time_range + +# Test data collection - this is the main goal! +/usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range + +# Test querying collected data +# Query 1: Count total rows +/usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json + +# Query 2: Show first 5 rows +/usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table + +# Query 3: Basic aggregation +/usr/local/bin/tailpipe query "SELECT date_part('hour', datetime_col) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', datetime_col) ORDER BY hour LIMIT 5" --output json + +# Test plugin show functionality +/usr/local/bin/tailpipe plugin show chaos From b90f4716359994294714c3dcc242ce8504cad6e0 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 8 Jul 2025 21:48:36 +0530 Subject: [PATCH 04/61] Add smoke test automation workflow and scripts (#469) --- .../workflows/02-tailpipe-smoke-tests.yaml | 194 ++++++++++++++++++ scripts/linux_container_info.sh | 7 + scripts/prepare_amazonlinux_container.sh | 20 ++ scripts/prepare_centos_container.sh | 20 ++ scripts/prepare_ubuntu_container.sh | 20 ++ scripts/smoke_test.sh | 78 +++++++ 6 files changed, 339 insertions(+) create mode 100644 .github/workflows/02-tailpipe-smoke-tests.yaml create mode 100755 scripts/linux_container_info.sh create mode 100755 scripts/prepare_amazonlinux_container.sh create mode 100755 scripts/prepare_centos_container.sh create mode 100755 scripts/prepare_ubuntu_container.sh create mode 100755 scripts/smoke_test.sh diff --git a/.github/workflows/02-tailpipe-smoke-tests.yaml b/.github/workflows/02-tailpipe-smoke-tests.yaml new file mode 100644 index 00000000..476fd73b --- /dev/null +++ b/.github/workflows/02-tailpipe-smoke-tests.yaml @@ -0,0 +1,194 @@ +name: "02 - Tailpipe: Smoke Tests" + +on: + workflow_dispatch: + inputs: + version: + description: "Version to test (with 'v' prefix, e.g., v1.0.0)" + required: true + type: string + +env: + # Version from input + VERSION: ${{ github.event.inputs.version }} + # Disable update checks during smoke tests + TAILPIPE_UPDATE_CHECK: false + + jobs: + smoke_test_ubuntu_24: + name: Smoke test (Ubuntu 24, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*linux_amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull Ubuntu latest Image + run: docker pull ubuntu:latest + + - name: Create and Start Ubuntu latest Container + run: | + docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec ubuntu-24-test /scripts/linux_container_info.sh + + - name: Install dependencies, create user, and assign necessary permissions + run: | + docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh + + - name: Run smoke tests + run: | + docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop ubuntu-24-test + docker rm ubuntu-24-test + + smoke_test_centos_9: + name: Smoke test (CentOS Stream 9, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*linux_amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull CentOS Stream 9 image + run: docker pull quay.io/centos/centos:stream9 + + - name: Create and Start CentOS stream9 Container + run: | + docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec centos-stream9-test /scripts/linux_container_info.sh + + - name: Install dependencies, create user, and assign necessary permissions + run: | + docker exec centos-stream9-test /scripts/prepare_centos_container.sh + + - name: Run smoke tests + run: | + docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop centos-stream9-test + docker rm centos-stream9-test + + smoke_test_amazonlinux: + name: Smoke test (Amazon Linux 2023, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*linux_amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull Amazon Linux 2023 Image + run: docker pull amazonlinux:2023 + + - name: Create and Start Amazon Linux 2023 Container + run: | + docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec amazonlinux-2023-test /scripts/linux_container_info.sh + + - name: Install dependencies, create user, and assign necessary permissions + run: | + docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh + + - name: Run smoke tests + run: | + docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop amazonlinux-2023-test + docker rm amazonlinux-2023-test + + smoke_test_darwin_arm64: + name: Smoke test (macOS 14, ARM64) + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Darwin Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "*darwin_arm64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Darwin Artifacts and Install Binary + run: | + mkdir -p /tmp/tailpipe + tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe + sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ + sudo chmod +x /usr/local/bin/tailpipe + + - name: Install jq + run: | + brew install jq + + - name: Get runner/container info + run: | + uname -a + sw_vers + + - name: Run smoke tests + run: | + chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh + $GITHUB_WORKSPACE/scripts/smoke_test.sh diff --git a/scripts/linux_container_info.sh b/scripts/linux_container_info.sh new file mode 100755 index 00000000..0a16f884 --- /dev/null +++ b/scripts/linux_container_info.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# This is a script to get the information about the linux container. +# Used in release smoke tests. + +uname -a # uname information +cat /etc/os-release # OS version information +ldd --version # glibc version information \ No newline at end of file diff --git a/scripts/prepare_amazonlinux_container.sh b/scripts/prepare_amazonlinux_container.sh new file mode 100755 index 00000000..75273933 --- /dev/null +++ b/scripts/prepare_amazonlinux_container.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the Amazon Linux 2023 container. +# Used in release smoke tests. + +# update yum and install required packages +yum update -y +yum install -y tar ca-certificates jq + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_centos_container.sh b/scripts/prepare_centos_container.sh new file mode 100755 index 00000000..a6e4903b --- /dev/null +++ b/scripts/prepare_centos_container.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the CentOS Stream 9 container. +# Used in release smoke tests. + +# update yum and install required packages +yum update -y +yum install -y tar ca-certificates jq + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_ubuntu_container.sh b/scripts/prepare_ubuntu_container.sh new file mode 100755 index 00000000..216bc77c --- /dev/null +++ b/scripts/prepare_ubuntu_container.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the ubuntu 24 container. +# Used in release smoke tests. + +# update apt and install required packages +apt-get update +apt-get install -y tar ca-certificates jq + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 00000000..ec8fc523 --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# This is a script with set of commands to smoke test a tailpipe build. +# The plan is to gradually add more tests to this script. + +set -e + +# Ensure the PATH includes the directory where jq is installed +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin + +# Check jq is available +jq --version + +/usr/local/bin/tailpipe --version # check version + +# Test basic query functionality (should work without data) +/usr/local/bin/tailpipe query "SELECT 1 as smoke_test" # verify basic query works + +# Test connect functionality +DB_FILE=$(/usr/local/bin/tailpipe connect --output json | jq -r '.database_filepath') + +# Verify the database file exists +if [ -f "$DB_FILE" ]; then + echo "Database file exists" +else + echo "Database file not found: $DB_FILE" + exit 1 +fi + +# Test plugin installation +/usr/local/bin/tailpipe plugin install chaos # install chaos plugin for testing +/usr/local/bin/tailpipe plugin list # verify plugin is installed + +# Show available tables and sources after plugin installation +/usr/local/bin/tailpipe table list # should now show chaos tables +/usr/local/bin/tailpipe source list # should now show chaos sources + +# Create configuration for testing +# the config path is different for darwin and linux +if [ "$(uname -s)" = "Darwin" ]; then + CONFIG_DIR="/Users/runner/.tailpipe/config" +else + CONFIG_DIR="/home/tailpipe/.tailpipe/config" +fi + +mkdir -p "$CONFIG_DIR" + +# Create chaos.tpc configuration file +cat > "$CONFIG_DIR/chaos.tpc" << 'EOF' +partition "chaos_date_time" "chaos_date_time_range" { + source "chaos_date_time" { + row_count = 100 + } +} +EOF + +cat "$CONFIG_DIR/chaos.tpc" + +# Test partition listing after adding configuration +/usr/local/bin/tailpipe partition list # should now show the chaos partition + +# Show partition details +/usr/local/bin/tailpipe partition show chaos_date_time.chaos_date_time_range + +# Test data collection - this is the main goal! +/usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range + +# Test querying collected data +# Query 1: Count total rows +/usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json + +# Query 2: Show first 5 rows +/usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table + +# Query 3: Basic aggregation +/usr/local/bin/tailpipe query "SELECT date_part('hour', datetime_col) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', datetime_col) ORDER BY hour LIMIT 5" --output json + +# Test plugin show functionality +/usr/local/bin/tailpipe plugin show chaos From 42feeb0400eba42fa1ce32af2a5767a655dcc8a7 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Thu, 24 Jul 2025 12:56:35 +0530 Subject: [PATCH 05/61] Fix issue where 'to' time was not respected for zero granularity data (#484) Add partition filter for to time --- internal/collector/collector.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 448fa5d1..3c90ffb9 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -158,11 +158,8 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, rec return err } - // if there is a from time, add a filter to the partition - this will be used by the parquet writer - if !resolvedFromTime.Time.IsZero() { - // NOTE: handle null timestamp so we get a validation error for null timestamps, rather than excluding the row - c.partition.AddFilter(fmt.Sprintf("(tp_timestamp is null or tp_timestamp >= '%s')", resolvedFromTime.Time.Format("2006-01-02T15:04:05"))) - } + // if we have a from or to time, add filters to the partition + c.addTimeRangeFilters(resolvedFromTime, toTime) // create a parquet writer parquetConvertor, err := parquet.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount) @@ -177,6 +174,20 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, rec return nil } +// addTimeRangeFilters adds filters to the partition based on the from and to time +func (c *Collector) addTimeRangeFilters(resolvedFromTime *row_source.ResolvedFromTime, toTime time.Time) { + // if there is a from time, add a filter to the partition - this will be used by the parquet writer + if !resolvedFromTime.Time.IsZero() { + // NOTE: handle null timestamp so we get a validation error for null timestamps, rather than excluding the row + c.partition.AddFilter(fmt.Sprintf("(tp_timestamp is null or tp_timestamp >= '%s')", resolvedFromTime.Time.Format("2006-01-02T15:04:05"))) + } + // if to time was set as arg, add that filter as well + if viper.IsSet(pconstants.ArgTo) { + // NOTE: handle null timestamp so we get a validation error for null timestamps, rather than excluding the row + c.partition.AddFilter(fmt.Sprintf("(tp_timestamp is null or tp_timestamp < '%s')", toTime.Format("2006-01-02T15:04:05"))) + } +} + // Notify implements observer.Observer // send an event down the channel to be picked up by the handlePluginEvent goroutine func (c *Collector) Notify(_ context.Context, event events.Event) error { From 0b4e41d5f19fc8a5ddd4a2db46646a0eee5eed98 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Thu, 24 Jul 2025 13:08:07 +0530 Subject: [PATCH 06/61] fix: Normalize relative time to UTC before applying 24h granularity trimming (#486) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 41046e90..5c950f56 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 - github.com/turbot/pipe-fittings/v2 v2.5.2 + github.com/turbot/pipe-fittings/v2 v2.6.0 github.com/turbot/tailpipe-plugin-sdk v0.9.1 github.com/zclconf/go-cty v1.14.4 golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c @@ -220,7 +220,7 @@ require ( golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index 0d18b9fc..fff71c34 100644 --- a/go.sum +++ b/go.sum @@ -769,8 +769,8 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.5.2 h1:qrQQEnD6dmMSQFUw8cKw6/Rb1sWolMru0+xXY/BcPU4= -github.com/turbot/pipe-fittings/v2 v2.5.2/go.mod h1:szte433cBDCaZcGe5zMVGG7uTl9HMaEYaQmuvzZRYIQ= +github.com/turbot/pipe-fittings/v2 v2.6.0 h1:RhCHble2MB7W0l9lE5QQLRQKuMD8xlM7AKuhoFOwqy4= +github.com/turbot/pipe-fittings/v2 v2.6.0/go.mod h1:wcKckD5UUCVWSQkdW6J27cumy5GkACV/wn7FJOajYHE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.9 h1:ntpGASgFpGbARK4RCVsIt96M5IlBY1//JBRr8H9IPVo= @@ -951,8 +951,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From f5fc36ee0bfab6bbcec40e1de9ae1299094df57a Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Thu, 24 Jul 2025 13:11:00 +0530 Subject: [PATCH 07/61] Upgrade go-viper/mapstructure/v2 to remediate moderate vulnerabilities --- go.mod | 2 +- go.sum | 4 ++-- memtest/go.mod | 14 ++------------ memtest/go.sum | 28 ++-------------------------- 4 files changed, 7 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 5c950f56..a31c3110 100644 --- a/go.mod +++ b/go.mod @@ -121,7 +121,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect github.com/goccy/go-yaml v1.11.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index fff71c34..046ef2e2 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= diff --git a/memtest/go.mod b/memtest/go.mod index 4b26d94e..30a39ee5 100644 --- a/memtest/go.mod +++ b/memtest/go.mod @@ -4,10 +4,7 @@ go 1.24 toolchain go1.24.1 -require ( - github.com/marcboeker/go-duckdb/v2 v2.2.0 - github.com/shirou/gopsutil/v3 v3.24.5 -) +require github.com/marcboeker/go-duckdb/v2 v2.2.0 require ( github.com/apache/arrow-go/v18 v18.1.0 // indirect @@ -18,23 +15,16 @@ require ( github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9 // indirect github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9 // indirect github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/flatbuffers v25.1.24+incompatible // indirect github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 // indirect github.com/marcboeker/go-duckdb/mapping v0.0.7 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect golang.org/x/mod v0.22.0 // indirect diff --git a/memtest/go.sum b/memtest/go.sum index e94855b6..9e7b8d71 100644 --- a/memtest/go.sum +++ b/memtest/go.sum @@ -18,17 +18,14 @@ github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9 h1:TVBDwDSanIttQCH76UpDJ github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9 h1:okFoG+evMiXnyUK+cI67V0MpvKbstO6MaXlXXotst3k= github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -39,8 +36,6 @@ github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IX github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 h1:6mq16sPGJPo8Tkkl6UIsXuaNv467LjHLBscRyJl2Qhc= github.com/marcboeker/go-duckdb/arrowmapping v0.0.7/go.mod h1:FdvmqJOwVdfFZLpV+anBFlTUOzfU/NdIRET37mIEczY= github.com/marcboeker/go-duckdb/mapping v0.0.7 h1:t0BaNmLXj76RKs/x80A/ZTe+KzZDimO2Ji8ct4YnPu4= @@ -55,22 +50,8 @@ github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= -github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -81,15 +62,10 @@ golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= From 415db268698bc4c665e93f0a74c51d09a6118e76 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 24 Jul 2025 13:24:14 +0530 Subject: [PATCH 08/61] chore: add post-release linux tests and slack notifications (#468) --- .github/workflows/01-tailpipe-release.yaml | 126 +++++++++++++++++- ...> 12-test-post-release-linux-distros.yaml} | 88 ++++++++---- scripts/prepare_amazonlinux_container.sh | 13 +- scripts/prepare_centos_container.sh | 9 +- scripts/prepare_ubuntu_container.sh | 6 +- scripts/smoke_test.sh | 54 ++++++-- 6 files changed, 234 insertions(+), 62 deletions(-) rename .github/workflows/{02-tailpipe-smoke-tests.yaml => 12-test-post-release-linux-distros.yaml} (64%) diff --git a/.github/workflows/01-tailpipe-release.yaml b/.github/workflows/01-tailpipe-release.yaml index 293b0df3..bdb57cf7 100644 --- a/.github/workflows/01-tailpipe-release.yaml +++ b/.github/workflows/01-tailpipe-release.yaml @@ -5,12 +5,12 @@ on: inputs: environment: type: choice - description: 'Select Release Type' + description: "Select Release Type" options: - # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment - - Development (alpha) - - Development (beta) - - Final (RC and final release) + # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment + - Development (alpha) + - Development (beta) + - Final (RC and final release) required: true version: description: "Version (without 'v')" @@ -132,7 +132,7 @@ jobs: # this is required, check golangci-lint-action docs - uses: actions/setup-go@19bb51245e9c80abacb2e91cc42b33fa478b8639 # v4.2.1 with: - go-version: '1.23' + go-version: "1.23" cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: Setup release environment @@ -223,3 +223,117 @@ jobs: git add . git commit -m "Versioning brew formulas" git push origin $VERSION + + update_homebrew_tap: + name: Update homebrew-tap formula + if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} + needs: update_pr_for_versioning + runs-on: ubuntu-latest + steps: + - name: Calculate version + id: calculate_version + run: | + echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Parse semver string + id: semver_parser + uses: booxmedialtd/ws-action-parse-semver@7784200024d6b3fc01253e617ec0168daf603de3 # v1.4.7 + with: + input_string: ${{ github.event.inputs.version }} + + - name: Checkout + if: steps.semver_parser.outputs.prerelease == '' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: turbot/homebrew-tap + token: ${{ secrets.GH_ACCESS_TOKEN }} + ref: main + + - name: Get pull request title + if: steps.semver_parser.outputs.prerelease == '' + id: pr_title + run: >- + echo "PR_TITLE=$( + gh pr view $VERSION --json title | jq .title | tr -d '"' + )" >> $GITHUB_OUTPUT + + - name: Output + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo ${{ steps.pr_title.outputs.PR_TITLE }} + echo ${{ env.VERSION }} + + - name: Fail if PR title does not match with version + if: steps.semver_parser.outputs.prerelease == '' + run: | + if [[ "${{ steps.pr_title.outputs.PR_TITLE }}" == "Tailpipe ${{ env.VERSION }}" ]]; then + echo "Correct version" + else + echo "Incorrect version" + exit 1 + fi + + - name: Merge pull request to update brew formula + if: steps.semver_parser.outputs.prerelease == '' + run: | + git fetch --all + gh pr merge $VERSION --squash --delete-branch + git push origin --delete bump-brew + + trigger_smoke_tests: + name: Trigger smoke tests + if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} + needs: [update_homebrew_tap] + runs-on: ubuntu-latest + steps: + - name: Calculate version + id: calculate_version + run: | + echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Parse semver string + id: semver_parser + uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 + with: + input_string: ${{ github.event.inputs.version }} + + - name: Trigger smoke test workflow + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo "Triggering smoke test workflow for version $VERSION..." + gh workflow run "12-test-post-release-linux-distros.yaml" \ + --ref ${{ github.ref }} \ + --field version=$VERSION \ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + + - name: Get smoke test workflow run URL + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo "Waiting for smoke test workflow to start..." + sleep 10 + + # Get the most recent run of the smoke test workflow + RUN_ID=$(gh run list \ + --workflow="12-test-post-release-linux-distros.yaml" \ + --repo ${{ github.repository }} \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ -n "$RUN_ID" ]; then + WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID" + echo "✅ Smoke test workflow triggered successfully!" + echo "🔗 Monitor progress at: $WORKFLOW_URL" + echo "" + echo "Workflow details:" + echo " - Version: $VERSION" + echo " - Workflow: 12-test-post-release-linux-distros.yaml" + echo " - Run ID: $RUN_ID" + else + echo "⚠️ Could not retrieve workflow run ID. Check manually at:" + echo "https://github.com/${{ github.repository }}/actions/workflows/12-test-post-release-linux-distros.yaml" + fi + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/02-tailpipe-smoke-tests.yaml b/.github/workflows/12-test-post-release-linux-distros.yaml similarity index 64% rename from .github/workflows/02-tailpipe-smoke-tests.yaml rename to .github/workflows/12-test-post-release-linux-distros.yaml index 476fd73b..4e58287a 100644 --- a/.github/workflows/02-tailpipe-smoke-tests.yaml +++ b/.github/workflows/12-test-post-release-linux-distros.yaml @@ -1,4 +1,4 @@ -name: "02 - Tailpipe: Smoke Tests" +name: "12 - Test: Linux Distros (Post-release)" on: workflow_dispatch: @@ -13,8 +13,9 @@ env: VERSION: ${{ github.event.inputs.version }} # Disable update checks during smoke tests TAILPIPE_UPDATE_CHECK: false + SLACK_WEBHOOK_URL: ${{ secrets.PIPELING_RELEASE_BOT_WEBHOOK_URL }} - jobs: +jobs: smoke_test_ubuntu_24: name: Smoke test (Ubuntu 24, x86_64) runs-on: ubuntu-latest @@ -26,11 +27,11 @@ env: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ + --pattern "tailpipe.linux.amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -48,13 +49,13 @@ env: run: | docker exec ubuntu-24-test /scripts/linux_container_info.sh - - name: Install dependencies, create user, and assign necessary permissions + - name: Install dependencies and setup environment run: | docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh - name: Run smoke tests run: | - docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh + docker exec ubuntu-24-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | @@ -72,11 +73,11 @@ env: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ + --pattern "tailpipe.linux.amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -94,13 +95,13 @@ env: run: | docker exec centos-stream9-test /scripts/linux_container_info.sh - - name: Install dependencies, create user, and assign necessary permissions + - name: Install dependencies and setup environment run: | docker exec centos-stream9-test /scripts/prepare_centos_container.sh - name: Run smoke tests run: | - docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh + docker exec centos-stream9-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | @@ -118,11 +119,11 @@ env: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ + --pattern "tailpipe.linux.amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -140,55 +141,86 @@ env: run: | docker exec amazonlinux-2023-test /scripts/linux_container_info.sh - - name: Install dependencies, create user, and assign necessary permissions + - name: Install dependencies and setup environment run: | docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh - name: Run smoke tests run: | - docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh + docker exec amazonlinux-2023-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | docker stop amazonlinux-2023-test docker rm amazonlinux-2023-test - smoke_test_darwin_arm64: - name: Smoke test (macOS 14, ARM64) - runs-on: macos-latest + smoke_test_linux_arm64: + name: Smoke test (Ubuntu 24, ARM64) + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Download Darwin Release Artifact + - name: Download Linux ARM64 Release Artifact run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*darwin_arm64.tar.gz" \ + --pattern "tailpipe.linux.arm64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz + mv ./artifacts/tailpipe.linux.arm64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Darwin Artifacts and Install Binary + - name: Extract Linux Artifacts and Install Binary run: | - mkdir -p /tmp/tailpipe - tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe - sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ + sudo tar -xzf ./artifacts/linux.tar.gz -C /usr/local/bin sudo chmod +x /usr/local/bin/tailpipe - name: Install jq run: | - brew install jq + sudo apt-get update + sudo apt-get install -y jq - name: Get runner/container info run: | uname -a - sw_vers + cat /etc/os-release - name: Run smoke tests run: | - chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh - $GITHUB_WORKSPACE/scripts/smoke_test.sh + chmod +x ${{ github.workspace }}/scripts/smoke_test.sh + ${{ github.workspace }}/scripts/smoke_test.sh + + notify_completion: + name: Notify completion + runs-on: ubuntu-latest + needs: + [ + smoke_test_ubuntu_24, + smoke_test_centos_9, + smoke_test_amazonlinux, + smoke_test_linux_arm64, + ] + if: always() + steps: + - name: Check results and notify + run: | + # Check if all jobs succeeded + UBUNTU_24_RESULT="${{ needs.smoke_test_ubuntu_24.result }}" + CENTOS_9_RESULT="${{ needs.smoke_test_centos_9.result }}" + AMAZONLINUX_RESULT="${{ needs.smoke_test_amazonlinux.result }}" + ARM64_RESULT="${{ needs.smoke_test_linux_arm64.result }}" + + WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + if [ "$UBUNTU_24_RESULT" = "success" ] && [ "$CENTOS_9_RESULT" = "success" ] && [ "$AMAZONLINUX_RESULT" = "success" ] && [ "$ARM64_RESULT" = "success" ]; then + MESSAGE="✅ Tailpipe ${{ env.VERSION }} smoke tests passed!\n\n🔗 View details: $WORKFLOW_URL" + else + MESSAGE="❌ Tailpipe ${{ env.VERSION }} smoke tests failed!\n\n🔗 View details: $WORKFLOW_URL" + fi + + curl -X POST -H 'Content-type: application/json' \ + --data "{\"text\":\"$MESSAGE\"}" \ + ${{ env.SLACK_WEBHOOK_URL }} diff --git a/scripts/prepare_amazonlinux_container.sh b/scripts/prepare_amazonlinux_container.sh index 75273933..7ca92813 100755 --- a/scripts/prepare_amazonlinux_container.sh +++ b/scripts/prepare_amazonlinux_container.sh @@ -2,9 +2,10 @@ # This is a script to install dependencies/packages, create user, and assign necessary permissions in the Amazon Linux 2023 container. # Used in release smoke tests. +set -e # Exit on any error + # update yum and install required packages -yum update -y -yum install -y tar ca-certificates jq +yum install -y shadow-utils tar gzip ca-certificates jq curl --allowerasing # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,9 +13,7 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Create user, since tailpipe cannot be run as root -useradd -m tailpipe - # Make the scripts executable -chown tailpipe:tailpipe /scripts/smoke_test.sh -chmod +x /scripts/smoke_test.sh \ No newline at end of file +chmod +x /scripts/smoke_test.sh + +echo "Amazon Linux container preparation completed successfully" \ No newline at end of file diff --git a/scripts/prepare_centos_container.sh b/scripts/prepare_centos_container.sh index a6e4903b..4a07bbbb 100755 --- a/scripts/prepare_centos_container.sh +++ b/scripts/prepare_centos_container.sh @@ -2,9 +2,10 @@ # This is a script to install dependencies/packages, create user, and assign necessary permissions in the CentOS Stream 9 container. # Used in release smoke tests. +set -e + # update yum and install required packages -yum update -y -yum install -y tar ca-certificates jq +yum install -y epel-release tar ca-certificates jq curl --allowerasing # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,9 +13,5 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Create user, since tailpipe cannot be run as root -useradd -m tailpipe - # Make the scripts executable -chown tailpipe:tailpipe /scripts/smoke_test.sh chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_ubuntu_container.sh b/scripts/prepare_ubuntu_container.sh index 216bc77c..ee1a420c 100755 --- a/scripts/prepare_ubuntu_container.sh +++ b/scripts/prepare_ubuntu_container.sh @@ -4,7 +4,7 @@ # update apt and install required packages apt-get update -apt-get install -y tar ca-certificates jq +apt-get install -y tar ca-certificates jq gzip # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,9 +12,5 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Create user, since tailpipe cannot be run as root -useradd -m tailpipe - # Make the scripts executable -chown tailpipe:tailpipe /scripts/smoke_test.sh chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index ec8fc523..5b283a48 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -37,9 +37,9 @@ fi # Create configuration for testing # the config path is different for darwin and linux if [ "$(uname -s)" = "Darwin" ]; then - CONFIG_DIR="/Users/runner/.tailpipe/config" + CONFIG_DIR="$HOME/.tailpipe/config" else - CONFIG_DIR="/home/tailpipe/.tailpipe/config" + CONFIG_DIR="$HOME/.tailpipe/config" fi mkdir -p "$CONFIG_DIR" @@ -62,17 +62,51 @@ cat "$CONFIG_DIR/chaos.tpc" /usr/local/bin/tailpipe partition show chaos_date_time.chaos_date_time_range # Test data collection - this is the main goal! -/usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range +# The chaos plugin generates dates around 2006-2007, so we need to collect from that range +echo "Starting data collection..." +# Use different timeout commands for macOS vs Linux +if [ "$(uname -s)" = "Darwin" ]; then + # macOS - try gtimeout first, fallback to no timeout + if command -v gtimeout >/dev/null 2>&1; then + gtimeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { + echo "Collection timed out or failed, trying without timeout..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false + } + else + echo "No timeout command available on macOS, running without timeout..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false + fi +else + # Linux - use timeout + timeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { + echo "Collection timed out or failed, trying without progress bar..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false 2>&1 | head -50 + echo "Collection attempt completed" + } +fi -# Test querying collected data -# Query 1: Count total rows -/usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json +# Verify data was collected before proceeding +echo "Checking if data was collected..." +DATA_COUNT=$(/usr/local/bin/tailpipe query "SELECT COUNT(*) as count FROM chaos_date_time" --output json 2>/dev/null | jq -r '.rows[0].count' || echo "0") +echo "Data count: $DATA_COUNT" -# Query 2: Show first 5 rows -/usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table +if [ "$DATA_COUNT" -gt 0 ]; then + echo "Data collection successful, proceeding with queries..." + + # Test querying collected data + # Query 1: Count total rows + /usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json -# Query 3: Basic aggregation -/usr/local/bin/tailpipe query "SELECT date_part('hour', datetime_col) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', datetime_col) ORDER BY hour LIMIT 5" --output json + # Query 2: Show first 5 rows + /usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table + + # Query 3: Basic aggregation using the correct column name + /usr/local/bin/tailpipe query "SELECT date_part('hour', timestamp) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', timestamp) ORDER BY hour LIMIT 5" --output json +else + echo "No data collected, skipping query tests..." + echo "Available tables after collection attempt:" + /usr/local/bin/tailpipe table list +fi # Test plugin show functionality /usr/local/bin/tailpipe plugin show chaos From 52ca9ccf07469de657411bb4330b114dd70bcd6c Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Thu, 24 Jul 2025 13:28:05 +0530 Subject: [PATCH 09/61] Revert "chore: add post-release linux tests and slack notifications (#468)" This reverts commit 415db268698bc4c665e93f0a74c51d09a6118e76. --- .github/workflows/01-tailpipe-release.yaml | 126 +----------------- ...tros.yaml => 02-tailpipe-smoke-tests.yaml} | 88 ++++-------- scripts/prepare_amazonlinux_container.sh | 13 +- scripts/prepare_centos_container.sh | 9 +- scripts/prepare_ubuntu_container.sh | 6 +- scripts/smoke_test.sh | 54 ++------ 6 files changed, 62 insertions(+), 234 deletions(-) rename .github/workflows/{12-test-post-release-linux-distros.yaml => 02-tailpipe-smoke-tests.yaml} (64%) diff --git a/.github/workflows/01-tailpipe-release.yaml b/.github/workflows/01-tailpipe-release.yaml index bdb57cf7..293b0df3 100644 --- a/.github/workflows/01-tailpipe-release.yaml +++ b/.github/workflows/01-tailpipe-release.yaml @@ -5,12 +5,12 @@ on: inputs: environment: type: choice - description: "Select Release Type" + description: 'Select Release Type' options: - # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment - - Development (alpha) - - Development (beta) - - Final (RC and final release) + # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment + - Development (alpha) + - Development (beta) + - Final (RC and final release) required: true version: description: "Version (without 'v')" @@ -132,7 +132,7 @@ jobs: # this is required, check golangci-lint-action docs - uses: actions/setup-go@19bb51245e9c80abacb2e91cc42b33fa478b8639 # v4.2.1 with: - go-version: "1.23" + go-version: '1.23' cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: Setup release environment @@ -223,117 +223,3 @@ jobs: git add . git commit -m "Versioning brew formulas" git push origin $VERSION - - update_homebrew_tap: - name: Update homebrew-tap formula - if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} - needs: update_pr_for_versioning - runs-on: ubuntu-latest - steps: - - name: Calculate version - id: calculate_version - run: | - echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV - - - name: Parse semver string - id: semver_parser - uses: booxmedialtd/ws-action-parse-semver@7784200024d6b3fc01253e617ec0168daf603de3 # v1.4.7 - with: - input_string: ${{ github.event.inputs.version }} - - - name: Checkout - if: steps.semver_parser.outputs.prerelease == '' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: turbot/homebrew-tap - token: ${{ secrets.GH_ACCESS_TOKEN }} - ref: main - - - name: Get pull request title - if: steps.semver_parser.outputs.prerelease == '' - id: pr_title - run: >- - echo "PR_TITLE=$( - gh pr view $VERSION --json title | jq .title | tr -d '"' - )" >> $GITHUB_OUTPUT - - - name: Output - if: steps.semver_parser.outputs.prerelease == '' - run: | - echo ${{ steps.pr_title.outputs.PR_TITLE }} - echo ${{ env.VERSION }} - - - name: Fail if PR title does not match with version - if: steps.semver_parser.outputs.prerelease == '' - run: | - if [[ "${{ steps.pr_title.outputs.PR_TITLE }}" == "Tailpipe ${{ env.VERSION }}" ]]; then - echo "Correct version" - else - echo "Incorrect version" - exit 1 - fi - - - name: Merge pull request to update brew formula - if: steps.semver_parser.outputs.prerelease == '' - run: | - git fetch --all - gh pr merge $VERSION --squash --delete-branch - git push origin --delete bump-brew - - trigger_smoke_tests: - name: Trigger smoke tests - if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} - needs: [update_homebrew_tap] - runs-on: ubuntu-latest - steps: - - name: Calculate version - id: calculate_version - run: | - echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV - - - name: Parse semver string - id: semver_parser - uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 - with: - input_string: ${{ github.event.inputs.version }} - - - name: Trigger smoke test workflow - if: steps.semver_parser.outputs.prerelease == '' - run: | - echo "Triggering smoke test workflow for version $VERSION..." - gh workflow run "12-test-post-release-linux-distros.yaml" \ - --ref ${{ github.ref }} \ - --field version=$VERSION \ - --repo ${{ github.repository }} - env: - GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} - - - name: Get smoke test workflow run URL - if: steps.semver_parser.outputs.prerelease == '' - run: | - echo "Waiting for smoke test workflow to start..." - sleep 10 - - # Get the most recent run of the smoke test workflow - RUN_ID=$(gh run list \ - --workflow="12-test-post-release-linux-distros.yaml" \ - --repo ${{ github.repository }} \ - --limit 1 \ - --json databaseId \ - --jq '.[0].databaseId') - - if [ -n "$RUN_ID" ]; then - WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID" - echo "✅ Smoke test workflow triggered successfully!" - echo "🔗 Monitor progress at: $WORKFLOW_URL" - echo "" - echo "Workflow details:" - echo " - Version: $VERSION" - echo " - Workflow: 12-test-post-release-linux-distros.yaml" - echo " - Run ID: $RUN_ID" - else - echo "⚠️ Could not retrieve workflow run ID. Check manually at:" - echo "https://github.com/${{ github.repository }}/actions/workflows/12-test-post-release-linux-distros.yaml" - fi - env: - GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/12-test-post-release-linux-distros.yaml b/.github/workflows/02-tailpipe-smoke-tests.yaml similarity index 64% rename from .github/workflows/12-test-post-release-linux-distros.yaml rename to .github/workflows/02-tailpipe-smoke-tests.yaml index 4e58287a..476fd73b 100644 --- a/.github/workflows/12-test-post-release-linux-distros.yaml +++ b/.github/workflows/02-tailpipe-smoke-tests.yaml @@ -1,4 +1,4 @@ -name: "12 - Test: Linux Distros (Post-release)" +name: "02 - Tailpipe: Smoke Tests" on: workflow_dispatch: @@ -13,9 +13,8 @@ env: VERSION: ${{ github.event.inputs.version }} # Disable update checks during smoke tests TAILPIPE_UPDATE_CHECK: false - SLACK_WEBHOOK_URL: ${{ secrets.PIPELING_RELEASE_BOT_WEBHOOK_URL }} -jobs: + jobs: smoke_test_ubuntu_24: name: Smoke test (Ubuntu 24, x86_64) runs-on: ubuntu-latest @@ -27,11 +26,11 @@ jobs: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "tailpipe.linux.amd64.tar.gz" \ + --pattern "*linux_amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -49,13 +48,13 @@ jobs: run: | docker exec ubuntu-24-test /scripts/linux_container_info.sh - - name: Install dependencies and setup environment + - name: Install dependencies, create user, and assign necessary permissions run: | docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh - name: Run smoke tests run: | - docker exec ubuntu-24-test /scripts/smoke_test.sh + docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | @@ -73,11 +72,11 @@ jobs: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "tailpipe.linux.amd64.tar.gz" \ + --pattern "*linux_amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -95,13 +94,13 @@ jobs: run: | docker exec centos-stream9-test /scripts/linux_container_info.sh - - name: Install dependencies and setup environment + - name: Install dependencies, create user, and assign necessary permissions run: | docker exec centos-stream9-test /scripts/prepare_centos_container.sh - name: Run smoke tests run: | - docker exec centos-stream9-test /scripts/smoke_test.sh + docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | @@ -119,11 +118,11 @@ jobs: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "tailpipe.linux.amd64.tar.gz" \ + --pattern "*linux_amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -141,86 +140,55 @@ jobs: run: | docker exec amazonlinux-2023-test /scripts/linux_container_info.sh - - name: Install dependencies and setup environment + - name: Install dependencies, create user, and assign necessary permissions run: | docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh - name: Run smoke tests run: | - docker exec amazonlinux-2023-test /scripts/smoke_test.sh + docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | docker stop amazonlinux-2023-test docker rm amazonlinux-2023-test - smoke_test_linux_arm64: - name: Smoke test (Ubuntu 24, ARM64) - runs-on: ubuntu-24.04-arm + smoke_test_darwin_arm64: + name: Smoke test (macOS 14, ARM64) + runs-on: macos-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Download Linux ARM64 Release Artifact + - name: Download Darwin Release Artifact run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "tailpipe.linux.arm64.tar.gz" \ + --pattern "*darwin_arm64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/tailpipe.linux.arm64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Linux Artifacts and Install Binary + - name: Extract Darwin Artifacts and Install Binary run: | - sudo tar -xzf ./artifacts/linux.tar.gz -C /usr/local/bin + mkdir -p /tmp/tailpipe + tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe + sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ sudo chmod +x /usr/local/bin/tailpipe - name: Install jq run: | - sudo apt-get update - sudo apt-get install -y jq + brew install jq - name: Get runner/container info run: | uname -a - cat /etc/os-release + sw_vers - name: Run smoke tests run: | - chmod +x ${{ github.workspace }}/scripts/smoke_test.sh - ${{ github.workspace }}/scripts/smoke_test.sh - - notify_completion: - name: Notify completion - runs-on: ubuntu-latest - needs: - [ - smoke_test_ubuntu_24, - smoke_test_centos_9, - smoke_test_amazonlinux, - smoke_test_linux_arm64, - ] - if: always() - steps: - - name: Check results and notify - run: | - # Check if all jobs succeeded - UBUNTU_24_RESULT="${{ needs.smoke_test_ubuntu_24.result }}" - CENTOS_9_RESULT="${{ needs.smoke_test_centos_9.result }}" - AMAZONLINUX_RESULT="${{ needs.smoke_test_amazonlinux.result }}" - ARM64_RESULT="${{ needs.smoke_test_linux_arm64.result }}" - - WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - - if [ "$UBUNTU_24_RESULT" = "success" ] && [ "$CENTOS_9_RESULT" = "success" ] && [ "$AMAZONLINUX_RESULT" = "success" ] && [ "$ARM64_RESULT" = "success" ]; then - MESSAGE="✅ Tailpipe ${{ env.VERSION }} smoke tests passed!\n\n🔗 View details: $WORKFLOW_URL" - else - MESSAGE="❌ Tailpipe ${{ env.VERSION }} smoke tests failed!\n\n🔗 View details: $WORKFLOW_URL" - fi - - curl -X POST -H 'Content-type: application/json' \ - --data "{\"text\":\"$MESSAGE\"}" \ - ${{ env.SLACK_WEBHOOK_URL }} + chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh + $GITHUB_WORKSPACE/scripts/smoke_test.sh diff --git a/scripts/prepare_amazonlinux_container.sh b/scripts/prepare_amazonlinux_container.sh index 7ca92813..75273933 100755 --- a/scripts/prepare_amazonlinux_container.sh +++ b/scripts/prepare_amazonlinux_container.sh @@ -2,10 +2,9 @@ # This is a script to install dependencies/packages, create user, and assign necessary permissions in the Amazon Linux 2023 container. # Used in release smoke tests. -set -e # Exit on any error - # update yum and install required packages -yum install -y shadow-utils tar gzip ca-certificates jq curl --allowerasing +yum update -y +yum install -y tar ca-certificates jq # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -13,7 +12,9 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Make the scripts executable -chmod +x /scripts/smoke_test.sh +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe -echo "Amazon Linux container preparation completed successfully" \ No newline at end of file +# Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_centos_container.sh b/scripts/prepare_centos_container.sh index 4a07bbbb..a6e4903b 100755 --- a/scripts/prepare_centos_container.sh +++ b/scripts/prepare_centos_container.sh @@ -2,10 +2,9 @@ # This is a script to install dependencies/packages, create user, and assign necessary permissions in the CentOS Stream 9 container. # Used in release smoke tests. -set -e - # update yum and install required packages -yum install -y epel-release tar ca-certificates jq curl --allowerasing +yum update -y +yum install -y tar ca-certificates jq # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -13,5 +12,9 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + # Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_ubuntu_container.sh b/scripts/prepare_ubuntu_container.sh index ee1a420c..216bc77c 100755 --- a/scripts/prepare_ubuntu_container.sh +++ b/scripts/prepare_ubuntu_container.sh @@ -4,7 +4,7 @@ # update apt and install required packages apt-get update -apt-get install -y tar ca-certificates jq gzip +apt-get install -y tar ca-certificates jq # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,5 +12,9 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe +# Create user, since tailpipe cannot be run as root +useradd -m tailpipe + # Make the scripts executable +chown tailpipe:tailpipe /scripts/smoke_test.sh chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index 5b283a48..ec8fc523 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -37,9 +37,9 @@ fi # Create configuration for testing # the config path is different for darwin and linux if [ "$(uname -s)" = "Darwin" ]; then - CONFIG_DIR="$HOME/.tailpipe/config" + CONFIG_DIR="/Users/runner/.tailpipe/config" else - CONFIG_DIR="$HOME/.tailpipe/config" + CONFIG_DIR="/home/tailpipe/.tailpipe/config" fi mkdir -p "$CONFIG_DIR" @@ -62,51 +62,17 @@ cat "$CONFIG_DIR/chaos.tpc" /usr/local/bin/tailpipe partition show chaos_date_time.chaos_date_time_range # Test data collection - this is the main goal! -# The chaos plugin generates dates around 2006-2007, so we need to collect from that range -echo "Starting data collection..." -# Use different timeout commands for macOS vs Linux -if [ "$(uname -s)" = "Darwin" ]; then - # macOS - try gtimeout first, fallback to no timeout - if command -v gtimeout >/dev/null 2>&1; then - gtimeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { - echo "Collection timed out or failed, trying without timeout..." - /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false - } - else - echo "No timeout command available on macOS, running without timeout..." - /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false - fi -else - # Linux - use timeout - timeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { - echo "Collection timed out or failed, trying without progress bar..." - /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false 2>&1 | head -50 - echo "Collection attempt completed" - } -fi +/usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range -# Verify data was collected before proceeding -echo "Checking if data was collected..." -DATA_COUNT=$(/usr/local/bin/tailpipe query "SELECT COUNT(*) as count FROM chaos_date_time" --output json 2>/dev/null | jq -r '.rows[0].count' || echo "0") -echo "Data count: $DATA_COUNT" +# Test querying collected data +# Query 1: Count total rows +/usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json -if [ "$DATA_COUNT" -gt 0 ]; then - echo "Data collection successful, proceeding with queries..." - - # Test querying collected data - # Query 1: Count total rows - /usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json +# Query 2: Show first 5 rows +/usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table - # Query 2: Show first 5 rows - /usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table - - # Query 3: Basic aggregation using the correct column name - /usr/local/bin/tailpipe query "SELECT date_part('hour', timestamp) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', timestamp) ORDER BY hour LIMIT 5" --output json -else - echo "No data collected, skipping query tests..." - echo "Available tables after collection attempt:" - /usr/local/bin/tailpipe table list -fi +# Query 3: Basic aggregation +/usr/local/bin/tailpipe query "SELECT date_part('hour', datetime_col) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', datetime_col) ORDER BY hour LIMIT 5" --output json # Test plugin show functionality /usr/local/bin/tailpipe plugin show chaos From 6c0494cbe73ec521ce50468de3a75ee020b9b6f4 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 24 Jul 2025 19:03:55 +0530 Subject: [PATCH 10/61] chore: add post-release linux distro tests (#487) --- .github/workflows/01-tailpipe-release.yaml | 126 +++++++++++++++++- ...> 12-test-post-release-linux-distros.yaml} | 88 ++++++++---- scripts/prepare_amazonlinux_container.sh | 13 +- scripts/prepare_centos_container.sh | 9 +- scripts/prepare_ubuntu_container.sh | 6 +- scripts/smoke_test.sh | 54 ++++++-- 6 files changed, 234 insertions(+), 62 deletions(-) rename .github/workflows/{02-tailpipe-smoke-tests.yaml => 12-test-post-release-linux-distros.yaml} (64%) diff --git a/.github/workflows/01-tailpipe-release.yaml b/.github/workflows/01-tailpipe-release.yaml index 293b0df3..bdb57cf7 100644 --- a/.github/workflows/01-tailpipe-release.yaml +++ b/.github/workflows/01-tailpipe-release.yaml @@ -5,12 +5,12 @@ on: inputs: environment: type: choice - description: 'Select Release Type' + description: "Select Release Type" options: - # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment - - Development (alpha) - - Development (beta) - - Final (RC and final release) + # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment + - Development (alpha) + - Development (beta) + - Final (RC and final release) required: true version: description: "Version (without 'v')" @@ -132,7 +132,7 @@ jobs: # this is required, check golangci-lint-action docs - uses: actions/setup-go@19bb51245e9c80abacb2e91cc42b33fa478b8639 # v4.2.1 with: - go-version: '1.23' + go-version: "1.23" cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: Setup release environment @@ -223,3 +223,117 @@ jobs: git add . git commit -m "Versioning brew formulas" git push origin $VERSION + + update_homebrew_tap: + name: Update homebrew-tap formula + if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} + needs: update_pr_for_versioning + runs-on: ubuntu-latest + steps: + - name: Calculate version + id: calculate_version + run: | + echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Parse semver string + id: semver_parser + uses: booxmedialtd/ws-action-parse-semver@7784200024d6b3fc01253e617ec0168daf603de3 # v1.4.7 + with: + input_string: ${{ github.event.inputs.version }} + + - name: Checkout + if: steps.semver_parser.outputs.prerelease == '' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: turbot/homebrew-tap + token: ${{ secrets.GH_ACCESS_TOKEN }} + ref: main + + - name: Get pull request title + if: steps.semver_parser.outputs.prerelease == '' + id: pr_title + run: >- + echo "PR_TITLE=$( + gh pr view $VERSION --json title | jq .title | tr -d '"' + )" >> $GITHUB_OUTPUT + + - name: Output + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo ${{ steps.pr_title.outputs.PR_TITLE }} + echo ${{ env.VERSION }} + + - name: Fail if PR title does not match with version + if: steps.semver_parser.outputs.prerelease == '' + run: | + if [[ "${{ steps.pr_title.outputs.PR_TITLE }}" == "Tailpipe ${{ env.VERSION }}" ]]; then + echo "Correct version" + else + echo "Incorrect version" + exit 1 + fi + + - name: Merge pull request to update brew formula + if: steps.semver_parser.outputs.prerelease == '' + run: | + git fetch --all + gh pr merge $VERSION --squash --delete-branch + git push origin --delete bump-brew + + trigger_smoke_tests: + name: Trigger smoke tests + if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} + needs: [update_homebrew_tap] + runs-on: ubuntu-latest + steps: + - name: Calculate version + id: calculate_version + run: | + echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Parse semver string + id: semver_parser + uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 + with: + input_string: ${{ github.event.inputs.version }} + + - name: Trigger smoke test workflow + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo "Triggering smoke test workflow for version $VERSION..." + gh workflow run "12-test-post-release-linux-distros.yaml" \ + --ref ${{ github.ref }} \ + --field version=$VERSION \ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + + - name: Get smoke test workflow run URL + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo "Waiting for smoke test workflow to start..." + sleep 10 + + # Get the most recent run of the smoke test workflow + RUN_ID=$(gh run list \ + --workflow="12-test-post-release-linux-distros.yaml" \ + --repo ${{ github.repository }} \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ -n "$RUN_ID" ]; then + WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID" + echo "✅ Smoke test workflow triggered successfully!" + echo "🔗 Monitor progress at: $WORKFLOW_URL" + echo "" + echo "Workflow details:" + echo " - Version: $VERSION" + echo " - Workflow: 12-test-post-release-linux-distros.yaml" + echo " - Run ID: $RUN_ID" + else + echo "⚠️ Could not retrieve workflow run ID. Check manually at:" + echo "https://github.com/${{ github.repository }}/actions/workflows/12-test-post-release-linux-distros.yaml" + fi + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/02-tailpipe-smoke-tests.yaml b/.github/workflows/12-test-post-release-linux-distros.yaml similarity index 64% rename from .github/workflows/02-tailpipe-smoke-tests.yaml rename to .github/workflows/12-test-post-release-linux-distros.yaml index 476fd73b..4e58287a 100644 --- a/.github/workflows/02-tailpipe-smoke-tests.yaml +++ b/.github/workflows/12-test-post-release-linux-distros.yaml @@ -1,4 +1,4 @@ -name: "02 - Tailpipe: Smoke Tests" +name: "12 - Test: Linux Distros (Post-release)" on: workflow_dispatch: @@ -13,8 +13,9 @@ env: VERSION: ${{ github.event.inputs.version }} # Disable update checks during smoke tests TAILPIPE_UPDATE_CHECK: false + SLACK_WEBHOOK_URL: ${{ secrets.PIPELING_RELEASE_BOT_WEBHOOK_URL }} - jobs: +jobs: smoke_test_ubuntu_24: name: Smoke test (Ubuntu 24, x86_64) runs-on: ubuntu-latest @@ -26,11 +27,11 @@ env: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ + --pattern "tailpipe.linux.amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -48,13 +49,13 @@ env: run: | docker exec ubuntu-24-test /scripts/linux_container_info.sh - - name: Install dependencies, create user, and assign necessary permissions + - name: Install dependencies and setup environment run: | docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh - name: Run smoke tests run: | - docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh + docker exec ubuntu-24-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | @@ -72,11 +73,11 @@ env: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ + --pattern "tailpipe.linux.amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -94,13 +95,13 @@ env: run: | docker exec centos-stream9-test /scripts/linux_container_info.sh - - name: Install dependencies, create user, and assign necessary permissions + - name: Install dependencies and setup environment run: | docker exec centos-stream9-test /scripts/prepare_centos_container.sh - name: Run smoke tests run: | - docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh + docker exec centos-stream9-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | @@ -118,11 +119,11 @@ env: run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ + --pattern "tailpipe.linux.amd64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -140,55 +141,86 @@ env: run: | docker exec amazonlinux-2023-test /scripts/linux_container_info.sh - - name: Install dependencies, create user, and assign necessary permissions + - name: Install dependencies and setup environment run: | docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh - name: Run smoke tests run: | - docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh + docker exec amazonlinux-2023-test /scripts/smoke_test.sh - name: Stop and Remove Container run: | docker stop amazonlinux-2023-test docker rm amazonlinux-2023-test - smoke_test_darwin_arm64: - name: Smoke test (macOS 14, ARM64) - runs-on: macos-latest + smoke_test_linux_arm64: + name: Smoke test (Ubuntu 24, ARM64) + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Download Darwin Release Artifact + - name: Download Linux ARM64 Release Artifact run: | mkdir -p ./artifacts gh release download ${{ env.VERSION }} \ - --pattern "*darwin_arm64.tar.gz" \ + --pattern "tailpipe.linux.arm64.tar.gz" \ --dir ./artifacts \ --repo ${{ github.repository }} # Rename to expected format - mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz + mv ./artifacts/tailpipe.linux.arm64.tar.gz ./artifacts/linux.tar.gz env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Darwin Artifacts and Install Binary + - name: Extract Linux Artifacts and Install Binary run: | - mkdir -p /tmp/tailpipe - tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe - sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ + sudo tar -xzf ./artifacts/linux.tar.gz -C /usr/local/bin sudo chmod +x /usr/local/bin/tailpipe - name: Install jq run: | - brew install jq + sudo apt-get update + sudo apt-get install -y jq - name: Get runner/container info run: | uname -a - sw_vers + cat /etc/os-release - name: Run smoke tests run: | - chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh - $GITHUB_WORKSPACE/scripts/smoke_test.sh + chmod +x ${{ github.workspace }}/scripts/smoke_test.sh + ${{ github.workspace }}/scripts/smoke_test.sh + + notify_completion: + name: Notify completion + runs-on: ubuntu-latest + needs: + [ + smoke_test_ubuntu_24, + smoke_test_centos_9, + smoke_test_amazonlinux, + smoke_test_linux_arm64, + ] + if: always() + steps: + - name: Check results and notify + run: | + # Check if all jobs succeeded + UBUNTU_24_RESULT="${{ needs.smoke_test_ubuntu_24.result }}" + CENTOS_9_RESULT="${{ needs.smoke_test_centos_9.result }}" + AMAZONLINUX_RESULT="${{ needs.smoke_test_amazonlinux.result }}" + ARM64_RESULT="${{ needs.smoke_test_linux_arm64.result }}" + + WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + if [ "$UBUNTU_24_RESULT" = "success" ] && [ "$CENTOS_9_RESULT" = "success" ] && [ "$AMAZONLINUX_RESULT" = "success" ] && [ "$ARM64_RESULT" = "success" ]; then + MESSAGE="✅ Tailpipe ${{ env.VERSION }} smoke tests passed!\n\n🔗 View details: $WORKFLOW_URL" + else + MESSAGE="❌ Tailpipe ${{ env.VERSION }} smoke tests failed!\n\n🔗 View details: $WORKFLOW_URL" + fi + + curl -X POST -H 'Content-type: application/json' \ + --data "{\"text\":\"$MESSAGE\"}" \ + ${{ env.SLACK_WEBHOOK_URL }} diff --git a/scripts/prepare_amazonlinux_container.sh b/scripts/prepare_amazonlinux_container.sh index 75273933..7ca92813 100755 --- a/scripts/prepare_amazonlinux_container.sh +++ b/scripts/prepare_amazonlinux_container.sh @@ -2,9 +2,10 @@ # This is a script to install dependencies/packages, create user, and assign necessary permissions in the Amazon Linux 2023 container. # Used in release smoke tests. +set -e # Exit on any error + # update yum and install required packages -yum update -y -yum install -y tar ca-certificates jq +yum install -y shadow-utils tar gzip ca-certificates jq curl --allowerasing # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,9 +13,7 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Create user, since tailpipe cannot be run as root -useradd -m tailpipe - # Make the scripts executable -chown tailpipe:tailpipe /scripts/smoke_test.sh -chmod +x /scripts/smoke_test.sh \ No newline at end of file +chmod +x /scripts/smoke_test.sh + +echo "Amazon Linux container preparation completed successfully" \ No newline at end of file diff --git a/scripts/prepare_centos_container.sh b/scripts/prepare_centos_container.sh index a6e4903b..4a07bbbb 100755 --- a/scripts/prepare_centos_container.sh +++ b/scripts/prepare_centos_container.sh @@ -2,9 +2,10 @@ # This is a script to install dependencies/packages, create user, and assign necessary permissions in the CentOS Stream 9 container. # Used in release smoke tests. +set -e + # update yum and install required packages -yum update -y -yum install -y tar ca-certificates jq +yum install -y epel-release tar ca-certificates jq curl --allowerasing # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,9 +13,5 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Create user, since tailpipe cannot be run as root -useradd -m tailpipe - # Make the scripts executable -chown tailpipe:tailpipe /scripts/smoke_test.sh chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_ubuntu_container.sh b/scripts/prepare_ubuntu_container.sh index 216bc77c..ee1a420c 100755 --- a/scripts/prepare_ubuntu_container.sh +++ b/scripts/prepare_ubuntu_container.sh @@ -4,7 +4,7 @@ # update apt and install required packages apt-get update -apt-get install -y tar ca-certificates jq +apt-get install -y tar ca-certificates jq gzip # Extract the tailpipe binary tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin @@ -12,9 +12,5 @@ tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin # Make the binary executable chmod +x /usr/local/bin/tailpipe -# Create user, since tailpipe cannot be run as root -useradd -m tailpipe - # Make the scripts executable -chown tailpipe:tailpipe /scripts/smoke_test.sh chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index ec8fc523..5b283a48 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -37,9 +37,9 @@ fi # Create configuration for testing # the config path is different for darwin and linux if [ "$(uname -s)" = "Darwin" ]; then - CONFIG_DIR="/Users/runner/.tailpipe/config" + CONFIG_DIR="$HOME/.tailpipe/config" else - CONFIG_DIR="/home/tailpipe/.tailpipe/config" + CONFIG_DIR="$HOME/.tailpipe/config" fi mkdir -p "$CONFIG_DIR" @@ -62,17 +62,51 @@ cat "$CONFIG_DIR/chaos.tpc" /usr/local/bin/tailpipe partition show chaos_date_time.chaos_date_time_range # Test data collection - this is the main goal! -/usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range +# The chaos plugin generates dates around 2006-2007, so we need to collect from that range +echo "Starting data collection..." +# Use different timeout commands for macOS vs Linux +if [ "$(uname -s)" = "Darwin" ]; then + # macOS - try gtimeout first, fallback to no timeout + if command -v gtimeout >/dev/null 2>&1; then + gtimeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { + echo "Collection timed out or failed, trying without timeout..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false + } + else + echo "No timeout command available on macOS, running without timeout..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false + fi +else + # Linux - use timeout + timeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { + echo "Collection timed out or failed, trying without progress bar..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false 2>&1 | head -50 + echo "Collection attempt completed" + } +fi -# Test querying collected data -# Query 1: Count total rows -/usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json +# Verify data was collected before proceeding +echo "Checking if data was collected..." +DATA_COUNT=$(/usr/local/bin/tailpipe query "SELECT COUNT(*) as count FROM chaos_date_time" --output json 2>/dev/null | jq -r '.rows[0].count' || echo "0") +echo "Data count: $DATA_COUNT" -# Query 2: Show first 5 rows -/usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table +if [ "$DATA_COUNT" -gt 0 ]; then + echo "Data collection successful, proceeding with queries..." + + # Test querying collected data + # Query 1: Count total rows + /usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json -# Query 3: Basic aggregation -/usr/local/bin/tailpipe query "SELECT date_part('hour', datetime_col) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', datetime_col) ORDER BY hour LIMIT 5" --output json + # Query 2: Show first 5 rows + /usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table + + # Query 3: Basic aggregation using the correct column name + /usr/local/bin/tailpipe query "SELECT date_part('hour', timestamp) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', timestamp) ORDER BY hour LIMIT 5" --output json +else + echo "No data collected, skipping query tests..." + echo "Available tables after collection attempt:" + /usr/local/bin/tailpipe table list +fi # Test plugin show functionality /usr/local/bin/tailpipe plugin show chaos From d239d0baab7445d5fe4597e05ac667c9ff273c7d Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 24 Jul 2025 19:12:02 +0530 Subject: [PATCH 11/61] Fix `.inspect` command to show plugin info for custom tables (#471) --- internal/metaquery/handler_inspect.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/metaquery/handler_inspect.go b/internal/metaquery/handler_inspect.go index 62ea964e..1c1fc631 100644 --- a/internal/metaquery/handler_inspect.go +++ b/internal/metaquery/handler_inspect.go @@ -10,6 +10,7 @@ import ( "github.com/turbot/tailpipe/internal/plugin" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" ) @@ -76,6 +77,13 @@ func listViewSchema(ctx context.Context, input *HandlerInput, viewName string) e // getPluginForTable returns the plugin name and version for a given table name. // note - this looks at the installed plugins and their version file entry, not only the version file func getPluginForTable(ctx context.Context, tableName string) (string, error) { + // First check if this is a custom table + if _, isCustom := config.GlobalConfig.CustomTables[tableName]; isCustom { + // Custom tables use the core plugin + corePluginName := constants.CorePluginInstallStream() + return corePluginName, nil + } + prefix := strings.Split(tableName, "_")[0] ps, err := plugin.GetInstalledPlugins(ctx, config.GlobalConfig.PluginVersions) From 713223eb9cae8ddb1a057cdeff82c0aea3df61cc Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 24 Jul 2025 19:22:36 +0530 Subject: [PATCH 12/61] fix: improve JSON column display formatting to match DuckDB output (#482) --- internal/query/execute.go | 96 ++++--- internal/query/execute_test.go | 454 +++++++++++++++++++++++++++++++++ 2 files changed, 511 insertions(+), 39 deletions(-) create mode 100644 internal/query/execute_test.go diff --git a/internal/query/execute.go b/internal/query/execute.go index 6857827a..bdfc874b 100644 --- a/internal/query/execute.go +++ b/internal/query/execute.go @@ -43,18 +43,26 @@ func RunBatchSession(ctx context.Context, args []string, db *database.DuckDb) (i } func ExecuteQuery(ctx context.Context, query string, db *database.DuckDb) (int, error) { + // Get column definitions first + colDefs, err := GetColumnDefsForQuery(query, db) + if err != nil { + // if this error is due to trying to select a table which exists in partition config, + // but there is no view defined (as no rows have been collected), return a special error + err := handleMissingViewError(err) + return 0, err + } + // Run the query rows, err := db.QueryContext(ctx, query) if err != nil { // if this error is due to trying to select a table which exists in partition config, // but there is no view defined (as no rows have been collected), return a special error err := handleMissingViewError(err) - return 0, err } // Execute the query - result, err := Execute(ctx, rows) + result, err := Execute(ctx, rows, colDefs) if err != nil { return 0, err } @@ -68,6 +76,52 @@ func ExecuteQuery(ctx context.Context, query string, db *database.DuckDb) (int, return 0, nil } +// GetColumnDefsForQuery executes a DESCRIBE query to get column definitions +func GetColumnDefsForQuery(query string, db *database.DuckDb) ([]*queryresult.ColumnDef, error) { + // Remove trailing semicolon from query to avoid DESCRIBE syntax errors + cleanQuery := strings.TrimSpace(query) + cleanQuery = strings.TrimSuffix(cleanQuery, ";") + + // Create DESCRIBE query + describeQuery := fmt.Sprintf("DESCRIBE (%s)", cleanQuery) + + // Execute the describe query + rows, err := db.Query(describeQuery) + if err != nil { + return nil, err + } + defer rows.Close() + + // Initialize a slice to hold column definitions + var columnDefs []*queryresult.ColumnDef + + // Process the DESCRIBE results + for rows.Next() { + var columnName, columnType string + var nullable, key, defaultValue, extra sql.NullString + + // DESCRIBE returns: column_name, column_type, null, key, default, extra + err := rows.Scan(&columnName, &columnType, &nullable, &key, &defaultValue, &extra) + if err != nil { + return nil, err + } + + columnDef := &queryresult.ColumnDef{ + Name: columnName, + DataType: columnType, + OriginalName: columnName, + } + + columnDefs = append(columnDefs, columnDef) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return columnDefs, nil +} + func handleMissingViewError(err error) error { errorMessage := err.Error() // Define the regex to match the table name @@ -97,12 +151,7 @@ func (t TimingMetadata) GetTiming() any { return t } -func Execute(ctx context.Context, rows *sql.Rows) (res *queryresult.Result[TimingMetadata], err error) { - - colDefs, err := fetchColumnDefs(rows) - if err != nil { - return nil, err - } +func Execute(ctx context.Context, rows *sql.Rows, colDefs []*queryresult.ColumnDef) (res *queryresult.Result[TimingMetadata], err error) { result := queryresult.NewResult[TimingMetadata](colDefs, TimingMetadata{}) @@ -151,34 +200,3 @@ func streamResults(ctx context.Context, rows *sql.Rows, result *queryresult.Resu } statushooks.Done(ctx) } - -// FetchColumnDefs extracts column definitions from sql.Rows and returns a slice of ColumnDef. -func fetchColumnDefs(rows *sql.Rows) ([]*queryresult.ColumnDef, error) { - // Get column names - columnNames, err := rows.Columns() - if err != nil { - return nil, err - } - - // Get column types - columnTypes, err := rows.ColumnTypes() - if err != nil { - return nil, err - } - - // Initialize a slice to hold column definitions - var columnDefs []*queryresult.ColumnDef - - for i, colType := range columnTypes { - columnDef := &queryresult.ColumnDef{ - Name: columnNames[i], - DataType: colType.DatabaseTypeName(), - OriginalName: columnNames[i], // Set this if you have a way to obtain the original name (optional) - this would be needed when multiple same columns are requested - } - - // Append to the list of column definitions - columnDefs = append(columnDefs, columnDef) - } - - return columnDefs, nil -} diff --git a/internal/query/execute_test.go b/internal/query/execute_test.go new file mode 100644 index 00000000..e4b172c7 --- /dev/null +++ b/internal/query/execute_test.go @@ -0,0 +1,454 @@ +package query + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/turbot/tailpipe/internal/database" +) + +func TestGetColumnDefsForQuery(t *testing.T) { + // Create a temporary DuckDB instance for testing + db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + // Create test tables with sample data + setupTestTables(t, db) + + tests := []struct { + name string + query string + expectedCols []string + expectError bool + }{ + { + name: "simple select", + query: "SELECT id, name, value FROM test_table", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "select with aliases", + query: "SELECT id AS user_id, name AS user_name, value AS score FROM test_table", + expectedCols: []string{"user_id", "user_name", "score"}, + expectError: false, + }, + { + name: "select with functions", + query: "SELECT COUNT(*), AVG(value), MAX(name) FROM test_table", + expectedCols: []string{"count_star()", "avg(\"value\")", "max(\"name\")"}, + expectError: false, + }, + { + name: "inner join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 INNER JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "left join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 LEFT JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "right join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 RIGHT JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "full outer join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 FULL OUTER JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "cross join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 CROSS JOIN category_table t2", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "group by", + query: "SELECT category, COUNT(*) as count, AVG(value) as avg_value FROM test_table t1 JOIN category_table t2 ON t1.id = t2.id GROUP BY category", + expectedCols: []string{"category", "count", "avg_value"}, + expectError: false, + }, + { + name: "group by with having", + query: "SELECT category, COUNT(*) as count FROM test_table t1 JOIN category_table t2 ON t1.id = t2.id GROUP BY category HAVING COUNT(*) > 1", + expectedCols: []string{"category", "count"}, + expectError: false, + }, + { + name: "order by", + query: "SELECT id, name, value FROM test_table ORDER BY value DESC", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "limit and offset", + query: "SELECT id, name, value FROM test_table ORDER BY id LIMIT 5 OFFSET 2", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "subquery in select", + query: "SELECT id, name, (SELECT AVG(value) FROM test_table) as avg_all FROM test_table", + expectedCols: []string{"id", "name", "avg_all"}, + expectError: false, + }, + { + name: "subquery in from", + query: "SELECT * FROM (SELECT id, name, value FROM test_table WHERE value > 5) as sub", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "subquery in where", + query: "SELECT id, name, value FROM test_table WHERE value > (SELECT AVG(value) FROM test_table)", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "union", + query: "SELECT id, name FROM test_table UNION SELECT id, category as name FROM category_table", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "union all", + query: "SELECT id, name FROM test_table UNION ALL SELECT id, category as name FROM category_table", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "case statement", + query: "SELECT id, name, CASE WHEN value > 5 THEN 'high' WHEN value > 2 THEN 'medium' ELSE 'low' END as level FROM test_table", + expectedCols: []string{"id", "name", "level"}, + expectError: false, + }, + { + name: "window function", + query: "SELECT id, name, value, ROW_NUMBER() OVER (ORDER BY value DESC) as rank FROM test_table", + expectedCols: []string{"id", "name", "value", "rank"}, + expectError: false, + }, + { + name: "window function with partition", + query: "SELECT id, name, value, ROW_NUMBER() OVER (PARTITION BY name ORDER BY value DESC) as rank FROM test_table", + expectedCols: []string{"id", "name", "value", "rank"}, + expectError: false, + }, + { + name: "aggregate with window function", + query: "SELECT id, name, value, AVG(value) OVER (PARTITION BY name) as avg_by_name FROM test_table", + expectedCols: []string{"id", "name", "value", "avg_by_name"}, + expectError: false, + }, + { + name: "cte (common table expression)", + query: "WITH cte AS (SELECT id, name, value FROM test_table WHERE value > 3) SELECT * FROM cte", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "multiple ctes", + query: "WITH cte1 AS (SELECT id, name FROM test_table), cte2 AS (SELECT id, category FROM category_table) SELECT cte1.name, cte2.category FROM cte1 JOIN cte2 ON cte1.id = cte2.id", + expectedCols: []string{"name", "category"}, + expectError: false, + }, + { + name: "exists subquery", + query: "SELECT id, name FROM test_table WHERE EXISTS (SELECT 1 FROM category_table WHERE category_table.id = test_table.id)", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "in subquery", + query: "SELECT id, name FROM test_table WHERE id IN (SELECT id FROM category_table WHERE category = 'A')", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "complex nested query", + query: "SELECT t1.id, t1.name, t2.category, (SELECT COUNT(*) FROM test_table WHERE value > t1.value) as higher_count FROM test_table t1 LEFT JOIN category_table t2 ON t1.id = t2.id WHERE t1.value > (SELECT AVG(value) FROM test_table) ORDER BY t1.value DESC LIMIT 10", + expectedCols: []string{"id", "name", "category", "higher_count"}, + expectError: false, + }, + { + name: "query with semicolon", + query: "SELECT id, name, value FROM test_table;", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "query with extra whitespace", + query: " SELECT id, name, value FROM test_table ", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "invalid query", + query: "SELECT * FROM non_existent_table", + expectedCols: nil, + expectError: true, + }, + { + name: "syntax error", + query: "SELECT id, name FROM test_table WHERE", + expectedCols: nil, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + colDefs, err := GetColumnDefsForQuery(tt.query, db) + + if tt.expectError { + assert.Error(t, err) + assert.Nil(t, colDefs) + } else { + assert.NoError(t, err) + assert.NotNil(t, colDefs) + assert.Len(t, colDefs, len(tt.expectedCols)) + + // Verify column names match expected + for i, expectedCol := range tt.expectedCols { + if i < len(colDefs) { + assert.Equal(t, expectedCol, colDefs[i].Name, "Column name mismatch at position %d", i) + assert.Equal(t, expectedCol, colDefs[i].OriginalName, "Original name should match name for position %d", i) + } + } + + // Verify all column definitions have data types + for i, colDef := range colDefs { + assert.NotEmpty(t, colDef.DataType, "Column %d (%s) should have a data type", i, colDef.Name) + } + } + }) + } +} + +func TestGetColumnDefsForQuery_EdgeCases(t *testing.T) { + // Create a temporary DuckDB instance for testing + db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + // Create test tables with sample data + setupTestTables(t, db) + + tests := []struct { + name string + query string + description string + expectError bool + }{ + { + name: "empty query", + query: "", + description: "Empty query should return error", + expectError: true, + }, + { + name: "whitespace only query", + query: " \t\n ", + description: "Whitespace-only query should return error", + expectError: true, + }, + { + name: "query with comments", + query: "SELECT id, name -- comment\nFROM test_table /* another comment */", + description: "Query with comments should work", + expectError: false, + }, + { + name: "query with special characters in column names", + query: "SELECT id as \"user-id\", name as \"user_name\", value as \"score_value\" FROM test_table", + description: "Query with quoted column names should work", + expectError: false, + }, + { + name: "query with numeric column names", + query: "SELECT 1 as \"1\", 2 as \"2\", 3 as \"3\" FROM test_table", + description: "Query with numeric column names should work", + expectError: false, + }, + { + name: "query with very long column names", + query: "SELECT id as \"very_long_column_name_that_exceeds_normal_length_limits_for_testing_purposes\", name, value FROM test_table", + description: "Query with very long column names should work", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + colDefs, err := GetColumnDefsForQuery(tt.query, db) + + if tt.expectError { + assert.Error(t, err, tt.description) + assert.Nil(t, colDefs) + } else { + assert.NoError(t, err, tt.description) + assert.NotNil(t, colDefs) + assert.Greater(t, len(colDefs), 0, "Should return at least one column definition") + } + }) + } +} + +func TestGetColumnDefsForQuery_DataTypes(t *testing.T) { + // Create a temporary DuckDB instance for testing + db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + // Create test table with various data types + _, err = db.Exec(` + CREATE TABLE data_types_test ( + id INTEGER, + name VARCHAR, + value DOUBLE, + is_active BOOLEAN, + created_at TIMESTAMP, + data BLOB, + json_data JSON, + uuid_val UUID + ) + `) + require.NoError(t, err) + + // Insert test data + _, err = db.Exec(` + INSERT INTO data_types_test VALUES + (1, 'test', 3.14, true, '2024-01-01 10:00:00', 'binary_data', '{"key": "value"}', '123e4567-e89b-12d3-a456-426614174000') + `) + require.NoError(t, err) + + tests := []struct { + name string + query string + expectedTypes []string + }{ + { + name: "all data types", + query: "SELECT * FROM data_types_test", + expectedTypes: []string{"INTEGER", "VARCHAR", "DOUBLE", "BOOLEAN", "TIMESTAMP", "BLOB", "JSON", "UUID"}, + }, + { + name: "type casting", + query: "SELECT CAST(id AS BIGINT) as big_id, CAST(value AS DECIMAL(10,2)) as decimal_value FROM data_types_test", + expectedTypes: []string{"BIGINT", "DECIMAL(10,2)"}, + }, + { + name: "string functions", + query: "SELECT UPPER(name) as upper_name, LENGTH(name) as name_length, SUBSTRING(name, 1, 2) as name_sub FROM data_types_test", + expectedTypes: []string{"VARCHAR", "BIGINT", "VARCHAR"}, + }, + { + name: "numeric functions", + query: "SELECT ABS(value) as abs_value, ROUND(value, 2) as rounded_value, CEIL(value) as ceil_value FROM data_types_test", + expectedTypes: []string{"DOUBLE", "DOUBLE", "DOUBLE"}, + }, + { + name: "date functions", + query: "SELECT YEAR(created_at) as year, MONTH(created_at) as month, DAY(created_at) as day FROM data_types_test", + expectedTypes: []string{"BIGINT", "BIGINT", "BIGINT"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + colDefs, err := GetColumnDefsForQuery(tt.query, db) + require.NoError(t, err) + require.Len(t, colDefs, len(tt.expectedTypes)) + + for i, expectedType := range tt.expectedTypes { + // DuckDB might return slightly different type names, so we check if the type contains our expected type + assert.Contains(t, colDefs[i].DataType, expectedType, + "Column %d (%s) should have data type containing %s, got %s", + i, colDefs[i].Name, expectedType, colDefs[i].DataType) + } + }) + } +} + +// setupTestTables creates test tables with sample data for testing +func setupTestTables(t testing.TB, db *database.DuckDb) { + // Create test_table + _, err := db.Exec(` + CREATE TABLE test_table ( + id INTEGER, + name VARCHAR, + value DOUBLE + ) + `) + require.NoError(t, err) + + // Create category_table + _, err = db.Exec(` + CREATE TABLE category_table ( + id INTEGER, + category VARCHAR + ) + `) + require.NoError(t, err) + + // Insert test data into test_table + _, err = db.Exec(` + INSERT INTO test_table VALUES + (1, 'Alice', 10.5), + (2, 'Bob', 7.2), + (3, 'Charlie', 15.8), + (4, 'David', 3.1), + (5, 'Eve', 12.3) + `) + require.NoError(t, err) + + // Insert test data into category_table + _, err = db.Exec(` + INSERT INTO category_table VALUES + (1, 'A'), + (2, 'B'), + (3, 'A'), + (4, 'C'), + (5, 'B') + `) + require.NoError(t, err) +} + +// BenchmarkGetColumnDefsForQuery benchmarks the GetColumnDefsForQuery function +func BenchmarkGetColumnDefsForQuery(b *testing.B) { + // Create a temporary DuckDB instance for benchmarking + db, err := database.NewDuckDb(database.WithTempDir(b.TempDir())) + require.NoError(b, err) + defer db.Close() + + // Setup test tables + setupTestTables(b, db) + + queries := []string{ + "SELECT id, name, value FROM test_table", + "SELECT t1.id, t1.name, t2.category FROM test_table t1 INNER JOIN category_table t2 ON t1.id = t2.id", + "SELECT category, COUNT(*) as count, AVG(value) as avg_value FROM test_table t1 JOIN category_table t2 ON t1.id = t2.id GROUP BY category", + "SELECT id, name, value, ROW_NUMBER() OVER (ORDER BY value DESC) as rank FROM test_table", + } + + for i, query := range queries { + b.Run(fmt.Sprintf("Query_%d", i+1), func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := GetColumnDefsForQuery(query, db) + if err != nil { + b.Fatalf("GetColumnDefsForQuery failed: %v", err) + } + } + }) + } +} From 83182e80876fcbef3b02598cc824e8be9b185912 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Thu, 24 Jul 2025 19:42:46 +0530 Subject: [PATCH 13/61] Update core to v0.2.10 --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/constants/plugin.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index a31c3110..753eba49 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 github.com/turbot/pipe-fittings/v2 v2.6.0 - github.com/turbot/tailpipe-plugin-sdk v0.9.1 + github.com/turbot/tailpipe-plugin-sdk v0.9.2 github.com/zclconf/go-cty v1.14.4 golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c @@ -41,7 +41,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/marcboeker/go-duckdb/v2 v2.1.0 github.com/thediveo/enumflag/v2 v2.0.5 - github.com/turbot/tailpipe-plugin-core v0.2.9 + github.com/turbot/tailpipe-plugin-core v0.2.10 golang.org/x/sync v0.12.0 golang.org/x/text v0.23.0 google.golang.org/grpc v1.69.2 diff --git a/go.sum b/go.sum index 046ef2e2..50eee3dc 100644 --- a/go.sum +++ b/go.sum @@ -773,10 +773,10 @@ github.com/turbot/pipe-fittings/v2 v2.6.0 h1:RhCHble2MB7W0l9lE5QQLRQKuMD8xlM7AKu github.com/turbot/pipe-fittings/v2 v2.6.0/go.mod h1:wcKckD5UUCVWSQkdW6J27cumy5GkACV/wn7FJOajYHE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= -github.com/turbot/tailpipe-plugin-core v0.2.9 h1:ntpGASgFpGbARK4RCVsIt96M5IlBY1//JBRr8H9IPVo= -github.com/turbot/tailpipe-plugin-core v0.2.9/go.mod h1:BQb0sF9CJIo93xDdPuOSqCUH19dz3QpG+NrdyKbYM/c= -github.com/turbot/tailpipe-plugin-sdk v0.9.1 h1:MaoN547oqmwPOWa817niKqtif/XMuyl61S1Ykq34LAk= -github.com/turbot/tailpipe-plugin-sdk v0.9.1/go.mod h1:ch/GVDaLoyau6LCUPbx/aYprNi7+DKhmwPDM0M1QJbM= +github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= +github.com/turbot/tailpipe-plugin-core v0.2.10/go.mod h1:dHzPUR1p5GksSvDqqEeZEvvJX6wTEwK/ZDev//9nSLw= +github.com/turbot/tailpipe-plugin-sdk v0.9.2 h1:bsivlduG4BSYlyjYIKGCiFHiYsrLhtuZbimjv1TnUOQ= +github.com/turbot/tailpipe-plugin-sdk v0.9.2/go.mod h1:Egojp0j7+th/4Bh6muMuF6aZa5iE3MuiJ4pzBo0J2mg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= diff --git a/internal/constants/plugin.go b/internal/constants/plugin.go index b63c85fe..fc44e390 100644 --- a/internal/constants/plugin.go +++ b/internal/constants/plugin.go @@ -7,7 +7,7 @@ import ( const ( // MinCorePluginVersion should be set for production releases - it is the minimum version of the core plugin that is required - MinCorePluginVersion = "v0.2.9" + MinCorePluginVersion = "v0.2.10" // CorePluginVersion may be set for pre-release versions - it allows us to pin a pre-release version of the core plugin // NOTE: they must NOT both be set CorePluginVersion = "" From 38379996f1b2875a5eab55a875b18b6326a18d9e Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Thu, 24 Jul 2025 20:01:18 +0530 Subject: [PATCH 14/61] v0.6.2 --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 185cf1ea..03a15fd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,18 @@ +## v0.6.2 [2025-07-24] +_Bug fixes_ +* Fix issue where `--to` was not respected for zero granularity data. ([#483](https://github.com/turbot/tailpipe/issues/483)) +* Fix issue where the relative time passed to `from/to` args were getting parsed incorrectly. ([#485](https://github.com/turbot/tailpipe/issues/485)) +* Fix issue where Tailpipe was crashing if the collection state file had nil trunk states from the previous collection. ([#489](https://github.com/turbot/tailpipe/issues/489)) +* Fix `.inspect` output to show the plugin name for custom tables. ([#360](https://github.com/turbot/tailpipe/issues/360)) +* Fix query JSON outputs to be consistent with DuckDB. ([#432](https://github.com/turbot/tailpipe/issues/432)) + +_Dependencies_ +* Upgrade `go-viper/mapstructure/v2` and `oauth2` packages to remediate high and moderate vulnerabilities. ## v0.6.1 [2025-07-02] _Bug fixes_ * Update core version to v0.2.9 - fix issue where collection state is not being saved for zero granularity collection. ([#251](https://github.com/turbot/tailpipe-plugin-sdk/issues/251)) - ## v0.6.0 [2025-07-02] _What's new_ * Add `--to` flag for `collect`, allowing collection of standalone time ranges. ([#238](https://github.com/turbot/tailpipe/issues/238)) From b953dca43311bfa1b93edfb911f8b62484577005 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Mon, 11 Aug 2025 17:23:20 +0530 Subject: [PATCH 15/61] chore: add diagnostic mode to display viper configuration for various commands --- cmd/collect.go | 9 +++++++++ cmd/compact.go | 11 ++++++++++- cmd/connect.go | 9 ++++++++- cmd/format.go | 8 ++++++++ cmd/partition.go | 19 ++++++++++++++++++ cmd/plugin.go | 32 +++++++++++++++++++++++++++++++ cmd/query.go | 8 ++++++++ cmd/root.go | 9 --------- cmd/source.go | 14 ++++++++++++++ cmd/table.go | 14 ++++++++++++++ internal/cmdconfig/diagnostics.go | 2 +- 11 files changed, 123 insertions(+), 12 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 0a10da1b..3b276e4e 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "os" "strings" "time" @@ -17,8 +18,10 @@ import ( "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/parse" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" @@ -71,6 +74,12 @@ func runCollectCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + err = doCollect(ctx, cancel, args) if errors.Is(err, context.Canceled) { // clear error so we don't show it with normal error reporting diff --git a/cmd/compact.go b/cmd/compact.go index 272dd08e..87cad3fa 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "golang.org/x/exp/maps" "log/slog" "os" "time" + "golang.org/x/exp/maps" + "github.com/briandowns/spinner" "github.com/spf13/cobra" "github.com/turbot/go-kit/helpers" @@ -16,7 +17,9 @@ import ( pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/parquet" ) @@ -49,6 +52,12 @@ func runCompactCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + slog.Info("Compacting parquet files") // verify that the provided args resolve to at least one partition diff --git a/cmd/connect.go b/cmd/connect.go index 2dcafd46..7bfe3455 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "golang.org/x/exp/maps" "io" "log" "os" @@ -22,11 +21,13 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/parse" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/filepaths" "github.com/turbot/tailpipe/internal/parquet" + "golang.org/x/exp/maps" ) // variable used to assign the output mode flag @@ -73,6 +74,12 @@ func runConnectCmd(cmd *cobra.Command, _ []string) { displayOutput(ctx, databaseFilePath, err) }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + databaseFilePath, err = generateDbFile(ctx) // we are done - the defer block will print either the filepath (if successful) or the error (if not) diff --git a/cmd/format.go b/cmd/format.go index 15165e23..97543dd2 100644 --- a/cmd/format.go +++ b/cmd/format.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -79,6 +81,12 @@ func runFormatListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListFormatResources(ctx) error_helpers.FailOnError(err) diff --git a/cmd/partition.go b/cmd/partition.go index 24bcdc62..66bc2fa6 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -18,6 +18,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" @@ -85,6 +86,12 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListPartitionResources(ctx) error_helpers.FailOnError(err) @@ -135,6 +142,12 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources partitionName := args[0] resource, err := display.GetPartitionResource(partitionName) @@ -186,6 +199,12 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // arg `fromTime` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) var fromTime time.Time diff --git a/cmd/plugin.go b/cmd/plugin.go index 55878629..8332baba 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "sync" "time" @@ -23,6 +24,7 @@ import ( "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/pipe-fittings/v2/versionfile" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" @@ -246,6 +248,12 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args to 'plugin install' -- one or more plugins to install // plugin names can be simple names for "standard" plugins, constraint suffixed names // or full refs to the OCI image @@ -373,6 +381,12 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args to 'plugin update' -- one or more plugins to update // These can be simple names for "standard" plugins, constraint suffixed names // or full refs to the OCI image @@ -623,6 +637,12 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + if len(args) == 0 { fmt.Println() //nolint:forbidigo // ui output error_helpers.ShowError(ctx, fmt.Errorf("you need to provide at least one plugin to uninstall")) @@ -685,6 +705,12 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resource(s) resources, err := display.ListPlugins(ctx) error_helpers.FailOnError(err) @@ -727,6 +753,12 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resource(s) resource, err := display.GetPluginResource(ctx, args[0]) error_helpers.FailOnError(err) diff --git a/cmd/query.go b/cmd/query.go index d767de7d..8bb0b3ea 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -12,6 +13,7 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/error_helpers" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/interactive" @@ -76,6 +78,12 @@ func runQueryCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // get a connection to the database var db *database.DuckDb db, err = openDatabaseConnection(ctx) diff --git a/cmd/root.go b/cmd/root.go index d33ca2fa..258b3a8b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,8 +1,6 @@ package cmd import ( - "os" - "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/turbot/pipe-fittings/v2/cmdconfig" @@ -10,7 +8,6 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/utils" - localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" ) @@ -63,12 +60,6 @@ func rootCommand() *cobra.Command { } func Execute() int { - // if diagnostic mode is set, print out config and return - if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { - localcmdconfig.DisplayConfig() - return 0 - } - rootCmd := rootCommand() utils.LogTime("cmd.root.Execute start") defer utils.LogTime("cmd.root.Execute end") diff --git a/cmd/source.go b/cmd/source.go index 1067c36d..972bfc97 100644 --- a/cmd/source.go +++ b/cmd/source.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -76,6 +78,12 @@ func runSourceListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListSourceResources(ctx) error_helpers.FailOnError(err) @@ -125,6 +133,12 @@ func runSourceShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resourceName := args[0] resource, err := display.GetSourceResource(ctx, resourceName) diff --git a/cmd/table.go b/cmd/table.go index 1ee88718..0ff868ff 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -77,6 +79,12 @@ func runTableListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListTableResources(ctx) error_helpers.FailOnError(err) @@ -127,6 +135,12 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resource, err := display.GetTableResource(ctx, args[0]) error_helpers.FailOnError(err) diff --git a/internal/cmdconfig/diagnostics.go b/internal/cmdconfig/diagnostics.go index b40a97ec..b5f4fe9d 100644 --- a/internal/cmdconfig/diagnostics.go +++ b/internal/cmdconfig/diagnostics.go @@ -51,7 +51,7 @@ func DisplayConfig() { sort.Strings(lines) var b strings.Builder - b.WriteString("\n================\nSteampipe Config\n================\n\n") + b.WriteString("\n================\nTailpipe Config\n================\n\n") for _, line := range lines { b.WriteString(line) From 3ca9e3e19277225e375bdad17c5647c41dc0d563 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 18 Aug 2025 13:04:02 +0530 Subject: [PATCH 16/61] test: config precedence acceptance tests (#494) --- .github/workflows/11-test-acceptance.yaml | 1 + .../config_tests/workspace_tests.json | 155 ++++++++++++++++++ .../source_files/config_tests/workspaces.tpc | 16 ++ .../test_files/config_precedence.bats | 121 ++++++++++++++ 4 files changed, 293 insertions(+) create mode 100644 tests/acceptance/test_data/source_files/config_tests/workspace_tests.json create mode 100755 tests/acceptance/test_data/source_files/config_tests/workspaces.tpc create mode 100644 tests/acceptance/test_files/config_precedence.bats diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 6717c664..053af91e 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -97,6 +97,7 @@ jobs: - "partition_delete" - "core_formats" - "table_block" + - "config_precedence" runs-on: ${{ matrix.platform }} steps: - name: Checkout diff --git a/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json b/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json new file mode 100644 index 00000000..f03b0759 --- /dev/null +++ b/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json @@ -0,0 +1,155 @@ +[ { + "test": "env variables set, no command line arguments set and no workspace env variable set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_PLUGIN_MEMORY_MAX_MB=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [] + }, + "expected": { + "update-check": "false", + "memory-max-mb": 16384, + "memory-max-mb-plugin": 2048, + "temp-dir-max-mb": 8192 + } + }, + { + "test": "only command line arguments set and no env variables set", + "description": "", + "cmd": "query", + "setup": { + "env": [], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "workspace env variable set and no command line arguments set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=workspace_profiles" + ], + "args": [] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "All env variables set and command line argument set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development", + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "workspace env variable set and --workspace arg passed", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "all env variables set including workspace env variable and no command line arguments set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development", + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [] + }, + "expected": { + "workspace": "development", + "log-level": "warn", + "update-check": "false", + "memory-max-mb": 512, + "memory-max-mb-plugin": 206, + "temp-dir-max-mb": 512 + } + }, + { + "test": "all env variables set except workspace env variable and --workspace arg passed", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + } +] \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc b/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc new file mode 100755 index 00000000..d38c3592 --- /dev/null +++ b/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc @@ -0,0 +1,16 @@ + +workspace "workspace_profiles" { + log_level = "trace" + update_check = false + memory_max_mb = 1024 + plugin_memory_max_mb = 512 + temp_dir_max_mb = 1024 +} + +workspace "development"{ + log_level = "warn" + update_check = false + memory_max_mb = 512 + plugin_memory_max_mb = 206 + temp_dir_max_mb = 512 +} \ No newline at end of file diff --git a/tests/acceptance/test_files/config_precedence.bats b/tests/acceptance/test_files/config_precedence.bats new file mode 100644 index 00000000..32c1e28e --- /dev/null +++ b/tests/acceptance/test_files/config_precedence.bats @@ -0,0 +1,121 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +## workspace tests + +@test "generic config precedence test" { + cp $SOURCE_FILES_DIR/config_tests/workspaces.tpc $TAILPIPE_INSTALL_DIR/config/workspaces.tpc + # setup test folder and read the test-cases file + cd $SOURCE_FILES_DIR/config_tests + tests=$(cat workspace_tests.json) + # echo $tests + + # to create the failure message + err="" + flag=0 + + # fetch the keys(test names) + test_keys=$(echo $tests | jq '. | keys[]') + # echo $test_keys + + for i in $test_keys; do + # each test case do the following + unset TAILPIPE_INSTALL_DIR + cwd=$(pwd) + export TAILPIPE_CONFIG_DUMP=config_json + + # command accordingly + cmd=$(echo $tests | jq -c ".[${i}]" | jq ".cmd") + if [[ $cmd == '"query"' ]]; then + tp_cmd='tailpipe query "select 1"' + fi + # echo $tp_cmd + + # key=$(echo $i) + echo -e "\n" + test_name=$(echo $tests | jq -c ".[${i}]" | jq ".test") + echo ">>> TEST NAME: $test_name" + + # env variables needed for setup + env=$(echo $tests | jq -c ".[${i}]" | jq ".setup.env") + # echo $env + + # set env variables + for e in $(echo "${env}" | jq -r '.[]'); do + export $e + done + + # args to run with tailpipe query command + args=$(echo $tests | jq -c ".[${i}]" | jq ".setup.args") + echo $args + + # construct the tailpipe command to be run with the args + for arg in $(echo "${args}" | jq -r '.[]'); do + tp_cmd="${tp_cmd} ${arg}" + done + echo "tailpipe command: $tp_cmd" # help debugging in case of failures + + # get the actual config by running the constructed tailpipe command + run $tp_cmd + echo "output from tailpipe command: $output" # help debugging in case of failures + + # The output contains log lines followed by a JSON object + # Find the start of the JSON (line starting with '{') and extract from there to the end + # Then use jq to parse and compact it + json_start_line=$(echo "$output" | grep -n '^{' | tail -1 | cut -d: -f1) + if [[ -n "$json_start_line" ]]; then + config_json=$(echo "$output" | tail -n +$json_start_line) + else + # Fallback: try to find any JSON-like content + config_json=$(echo "$output" | grep -A 1000 '{' | head -1000) + fi + + # Parse with jq and handle errors gracefully + actual_config=$(echo "$config_json" | jq -c '.' 2>/dev/null) + if [[ $? -ne 0 ]] || [[ -z "$actual_config" ]]; then + echo "Failed to parse JSON config, raw output:" + echo "$config_json" + actual_config="{}" + fi + echo "actual config: \n$actual_config" # help debugging in case of failures + + # get expected config from test case + expected_config=$(echo $tests | jq -c ".[${i}]" | jq ".expected") + # echo $expected_config + + # fetch only keys from expected config + exp_keys=$(echo $expected_config | jq '. | keys[]' | jq -s 'flatten | @sh' | tr -d '\'\' | tr -d '"') + + for key in $exp_keys; do + # get the expected and the actual value for the keys + exp_val=$(echo $(echo $expected_config | jq --arg KEY $key '.[$KEY]' | tr -d '"')) + act_val=$(echo $(echo $actual_config | jq --arg KEY $key '.[$KEY]' | tr -d '"')) + + # get the absolute paths for install-dir and mod-location + if [[ $key == "install-dir" ]] || [[ $key == "mod-location" ]]; then + exp_val="${cwd}/${exp_val}" + fi + echo "expected $key: $exp_val" + echo "actual $key: $act_val" + + # check the values + if [[ "$exp_val" != "$act_val" ]]; then + flag=1 + err="FAILED: $test_name >> key: $key ; expected: $exp_val ; actual: $act_val \n${err}" + fi + done + + # check if all passed + if [[ $flag -eq 0 ]]; then + echo "PASSED ✅" + else + echo "FAILED ❌" + fi + # reset flag back to 0 for the next test case + flag=0 + done + echo -e "\n" + echo -e "$err" + assert_equal "$err" "" + rm -f err +} From 115f985746aae730eab311ea2093036afda7b1a8 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 19 Aug 2025 14:49:12 +0530 Subject: [PATCH 17/61] Removing the deletion of stale issues in tailpipe for now (#515) * Removing the deletion of stale issues in tailpipe for now * add TODO comment as a reminder to add back the closing of stale issues part later on --- .github/workflows/30-stale.yaml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/30-stale.yaml b/.github/workflows/30-stale.yaml index 2b4cbbb1..36e32042 100644 --- a/.github/workflows/30-stale.yaml +++ b/.github/workflows/30-stale.yaml @@ -18,22 +18,26 @@ jobs: id: stale-issues-and-prs uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8.0.0 with: - close-issue-message: | - This issue was closed because it has been stalled for 90 days with no activity. - close-issue-reason: 'not_planned' - close-pr-message: | - This PR was closed because it has been stalled for 90 days with no activity. - # Set days-before-close to 30 because we want to close the issue/PR after 90 days total, since days-before-stale is set to 60 - days-before-close: 30 + # TODO: Add back the closing of stale issue part later on + # close-issue-message: | + # This issue was closed because it has been stalled for 90 days with no activity. + # close-issue-reason: "not_planned" + # close-pr-message: | + # This PR was closed because it has been stalled for 90 days with no activity. + # # Set days-before-close to 30 because we want to close the issue/PR after 90 days total, since days-before-stale is set to 60 + # days-before-close: 30 + days-before-close: -1 days-before-stale: 60 debug-only: ${{ inputs.dryRun }} - exempt-issue-labels: 'good first issue,help wanted,blocker' + exempt-issue-labels: "good first issue,help wanted,blocker" repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-label: 'stale' + stale-issue-label: "stale" stale-issue-message: | - This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. - stale-pr-label: 'stale' + This issue is stale because it has been open 60 days with no activity. + # This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. + stale-pr-label: "stale" stale-pr-message: | - This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. + This PR is stale because it has been open 60 days with no activity. + # This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. start-date: "2021-02-09" operations-per-run: 1000 From 0038f6e9f0371c64a1606ca3ac750f684a72a542 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 26 Aug 2025 11:58:56 +0530 Subject: [PATCH 18/61] chore: remove redundant Tailpipe smoke test workflow (#526) --- .../workflows/02-tailpipe-smoke-tests.yaml | 194 ------------------ 1 file changed, 194 deletions(-) delete mode 100644 .github/workflows/02-tailpipe-smoke-tests.yaml diff --git a/.github/workflows/02-tailpipe-smoke-tests.yaml b/.github/workflows/02-tailpipe-smoke-tests.yaml deleted file mode 100644 index 476fd73b..00000000 --- a/.github/workflows/02-tailpipe-smoke-tests.yaml +++ /dev/null @@ -1,194 +0,0 @@ -name: "02 - Tailpipe: Smoke Tests" - -on: - workflow_dispatch: - inputs: - version: - description: "Version to test (with 'v' prefix, e.g., v1.0.0)" - required: true - type: string - -env: - # Version from input - VERSION: ${{ github.event.inputs.version }} - # Disable update checks during smoke tests - TAILPIPE_UPDATE_CHECK: false - - jobs: - smoke_test_ubuntu_24: - name: Smoke test (Ubuntu 24, x86_64) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Linux Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Pull Ubuntu latest Image - run: docker pull ubuntu:latest - - - name: Create and Start Ubuntu latest Container - run: | - docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null - - - name: Get runner/container info - run: | - docker exec ubuntu-24-test /scripts/linux_container_info.sh - - - name: Install dependencies, create user, and assign necessary permissions - run: | - docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh - - - name: Run smoke tests - run: | - docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh - - - name: Stop and Remove Container - run: | - docker stop ubuntu-24-test - docker rm ubuntu-24-test - - smoke_test_centos_9: - name: Smoke test (CentOS Stream 9, x86_64) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Linux Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Pull CentOS Stream 9 image - run: docker pull quay.io/centos/centos:stream9 - - - name: Create and Start CentOS stream9 Container - run: | - docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null - - - name: Get runner/container info - run: | - docker exec centos-stream9-test /scripts/linux_container_info.sh - - - name: Install dependencies, create user, and assign necessary permissions - run: | - docker exec centos-stream9-test /scripts/prepare_centos_container.sh - - - name: Run smoke tests - run: | - docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh - - - name: Stop and Remove Container - run: | - docker stop centos-stream9-test - docker rm centos-stream9-test - - smoke_test_amazonlinux: - name: Smoke test (Amazon Linux 2023, x86_64) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Linux Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Pull Amazon Linux 2023 Image - run: docker pull amazonlinux:2023 - - - name: Create and Start Amazon Linux 2023 Container - run: | - docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null - - - name: Get runner/container info - run: | - docker exec amazonlinux-2023-test /scripts/linux_container_info.sh - - - name: Install dependencies, create user, and assign necessary permissions - run: | - docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh - - - name: Run smoke tests - run: | - docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh - - - name: Stop and Remove Container - run: | - docker stop amazonlinux-2023-test - docker rm amazonlinux-2023-test - - smoke_test_darwin_arm64: - name: Smoke test (macOS 14, ARM64) - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Darwin Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*darwin_arm64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Darwin Artifacts and Install Binary - run: | - mkdir -p /tmp/tailpipe - tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe - sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ - sudo chmod +x /usr/local/bin/tailpipe - - - name: Install jq - run: | - brew install jq - - - name: Get runner/container info - run: | - uname -a - sw_vers - - - name: Run smoke tests - run: | - chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh - $GITHUB_WORKSPACE/scripts/smoke_test.sh From 3343844de21e2d5892f302f50740eb1af996c054 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:32:50 +0530 Subject: [PATCH 19/61] Merge branch 'v0.6.x' into develop (#527) --- .../workflows/02-tailpipe-smoke-tests.yaml | 194 ------------------ .github/workflows/11-test-acceptance.yaml | 1 + .github/workflows/30-stale.yaml | 28 +-- cmd/collect.go | 9 + cmd/compact.go | 11 +- cmd/connect.go | 9 +- cmd/format.go | 8 + cmd/partition.go | 19 ++ cmd/plugin.go | 32 +++ cmd/query.go | 8 + cmd/root.go | 9 - cmd/source.go | 14 ++ cmd/table.go | 14 ++ internal/cmdconfig/diagnostics.go | 2 +- .../config_tests/workspace_tests.json | 155 ++++++++++++++ .../source_files/config_tests/workspaces.tpc | 16 ++ .../test_files/config_precedence.bats | 121 +++++++++++ 17 files changed, 432 insertions(+), 218 deletions(-) delete mode 100644 .github/workflows/02-tailpipe-smoke-tests.yaml create mode 100644 tests/acceptance/test_data/source_files/config_tests/workspace_tests.json create mode 100755 tests/acceptance/test_data/source_files/config_tests/workspaces.tpc create mode 100644 tests/acceptance/test_files/config_precedence.bats diff --git a/.github/workflows/02-tailpipe-smoke-tests.yaml b/.github/workflows/02-tailpipe-smoke-tests.yaml deleted file mode 100644 index 476fd73b..00000000 --- a/.github/workflows/02-tailpipe-smoke-tests.yaml +++ /dev/null @@ -1,194 +0,0 @@ -name: "02 - Tailpipe: Smoke Tests" - -on: - workflow_dispatch: - inputs: - version: - description: "Version to test (with 'v' prefix, e.g., v1.0.0)" - required: true - type: string - -env: - # Version from input - VERSION: ${{ github.event.inputs.version }} - # Disable update checks during smoke tests - TAILPIPE_UPDATE_CHECK: false - - jobs: - smoke_test_ubuntu_24: - name: Smoke test (Ubuntu 24, x86_64) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Linux Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Pull Ubuntu latest Image - run: docker pull ubuntu:latest - - - name: Create and Start Ubuntu latest Container - run: | - docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null - - - name: Get runner/container info - run: | - docker exec ubuntu-24-test /scripts/linux_container_info.sh - - - name: Install dependencies, create user, and assign necessary permissions - run: | - docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh - - - name: Run smoke tests - run: | - docker exec -u tailpipe ubuntu-24-test /scripts/smoke_test.sh - - - name: Stop and Remove Container - run: | - docker stop ubuntu-24-test - docker rm ubuntu-24-test - - smoke_test_centos_9: - name: Smoke test (CentOS Stream 9, x86_64) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Linux Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Pull CentOS Stream 9 image - run: docker pull quay.io/centos/centos:stream9 - - - name: Create and Start CentOS stream9 Container - run: | - docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null - - - name: Get runner/container info - run: | - docker exec centos-stream9-test /scripts/linux_container_info.sh - - - name: Install dependencies, create user, and assign necessary permissions - run: | - docker exec centos-stream9-test /scripts/prepare_centos_container.sh - - - name: Run smoke tests - run: | - docker exec -u tailpipe centos-stream9-test /scripts/smoke_test.sh - - - name: Stop and Remove Container - run: | - docker stop centos-stream9-test - docker rm centos-stream9-test - - smoke_test_amazonlinux: - name: Smoke test (Amazon Linux 2023, x86_64) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Linux Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*linux_amd64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*linux_amd64.tar.gz ./artifacts/linux.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - - name: Pull Amazon Linux 2023 Image - run: docker pull amazonlinux:2023 - - - name: Create and Start Amazon Linux 2023 Container - run: | - docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null - - - name: Get runner/container info - run: | - docker exec amazonlinux-2023-test /scripts/linux_container_info.sh - - - name: Install dependencies, create user, and assign necessary permissions - run: | - docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh - - - name: Run smoke tests - run: | - docker exec -u tailpipe amazonlinux-2023-test /scripts/smoke_test.sh - - - name: Stop and Remove Container - run: | - docker stop amazonlinux-2023-test - docker rm amazonlinux-2023-test - - smoke_test_darwin_arm64: - name: Smoke test (macOS 14, ARM64) - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download Darwin Release Artifact - run: | - mkdir -p ./artifacts - gh release download ${{ env.VERSION }} \ - --pattern "*darwin_arm64.tar.gz" \ - --dir ./artifacts \ - --repo ${{ github.repository }} - # Rename to expected format - mv ./artifacts/*darwin_arm64.tar.gz ./artifacts/darwin.tar.gz - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Darwin Artifacts and Install Binary - run: | - mkdir -p /tmp/tailpipe - tar -xzf ./artifacts/darwin.tar.gz -C /tmp/tailpipe - sudo cp /tmp/tailpipe/tailpipe /usr/local/bin/ - sudo chmod +x /usr/local/bin/tailpipe - - - name: Install jq - run: | - brew install jq - - - name: Get runner/container info - run: | - uname -a - sw_vers - - - name: Run smoke tests - run: | - chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh - $GITHUB_WORKSPACE/scripts/smoke_test.sh diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 6717c664..053af91e 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -97,6 +97,7 @@ jobs: - "partition_delete" - "core_formats" - "table_block" + - "config_precedence" runs-on: ${{ matrix.platform }} steps: - name: Checkout diff --git a/.github/workflows/30-stale.yaml b/.github/workflows/30-stale.yaml index 2b4cbbb1..36e32042 100644 --- a/.github/workflows/30-stale.yaml +++ b/.github/workflows/30-stale.yaml @@ -18,22 +18,26 @@ jobs: id: stale-issues-and-prs uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8.0.0 with: - close-issue-message: | - This issue was closed because it has been stalled for 90 days with no activity. - close-issue-reason: 'not_planned' - close-pr-message: | - This PR was closed because it has been stalled for 90 days with no activity. - # Set days-before-close to 30 because we want to close the issue/PR after 90 days total, since days-before-stale is set to 60 - days-before-close: 30 + # TODO: Add back the closing of stale issue part later on + # close-issue-message: | + # This issue was closed because it has been stalled for 90 days with no activity. + # close-issue-reason: "not_planned" + # close-pr-message: | + # This PR was closed because it has been stalled for 90 days with no activity. + # # Set days-before-close to 30 because we want to close the issue/PR after 90 days total, since days-before-stale is set to 60 + # days-before-close: 30 + days-before-close: -1 days-before-stale: 60 debug-only: ${{ inputs.dryRun }} - exempt-issue-labels: 'good first issue,help wanted,blocker' + exempt-issue-labels: "good first issue,help wanted,blocker" repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-label: 'stale' + stale-issue-label: "stale" stale-issue-message: | - This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. - stale-pr-label: 'stale' + This issue is stale because it has been open 60 days with no activity. + # This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. + stale-pr-label: "stale" stale-pr-message: | - This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. + This PR is stale because it has been open 60 days with no activity. + # This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. start-date: "2021-02-09" operations-per-run: 1000 diff --git a/cmd/collect.go b/cmd/collect.go index 0a10da1b..3b276e4e 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "os" "strings" "time" @@ -17,8 +18,10 @@ import ( "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/parse" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" @@ -71,6 +74,12 @@ func runCollectCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + err = doCollect(ctx, cancel, args) if errors.Is(err, context.Canceled) { // clear error so we don't show it with normal error reporting diff --git a/cmd/compact.go b/cmd/compact.go index 272dd08e..87cad3fa 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "golang.org/x/exp/maps" "log/slog" "os" "time" + "golang.org/x/exp/maps" + "github.com/briandowns/spinner" "github.com/spf13/cobra" "github.com/turbot/go-kit/helpers" @@ -16,7 +17,9 @@ import ( pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/parquet" ) @@ -49,6 +52,12 @@ func runCompactCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + slog.Info("Compacting parquet files") // verify that the provided args resolve to at least one partition diff --git a/cmd/connect.go b/cmd/connect.go index 2dcafd46..7bfe3455 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "golang.org/x/exp/maps" "io" "log" "os" @@ -22,11 +21,13 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/parse" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/filepaths" "github.com/turbot/tailpipe/internal/parquet" + "golang.org/x/exp/maps" ) // variable used to assign the output mode flag @@ -73,6 +74,12 @@ func runConnectCmd(cmd *cobra.Command, _ []string) { displayOutput(ctx, databaseFilePath, err) }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + databaseFilePath, err = generateDbFile(ctx) // we are done - the defer block will print either the filepath (if successful) or the error (if not) diff --git a/cmd/format.go b/cmd/format.go index 15165e23..97543dd2 100644 --- a/cmd/format.go +++ b/cmd/format.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -79,6 +81,12 @@ func runFormatListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListFormatResources(ctx) error_helpers.FailOnError(err) diff --git a/cmd/partition.go b/cmd/partition.go index 24bcdc62..66bc2fa6 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -18,6 +18,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" @@ -85,6 +86,12 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListPartitionResources(ctx) error_helpers.FailOnError(err) @@ -135,6 +142,12 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources partitionName := args[0] resource, err := display.GetPartitionResource(partitionName) @@ -186,6 +199,12 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // arg `fromTime` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) var fromTime time.Time diff --git a/cmd/plugin.go b/cmd/plugin.go index 55878629..8332baba 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "sync" "time" @@ -23,6 +24,7 @@ import ( "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/pipe-fittings/v2/versionfile" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" @@ -246,6 +248,12 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args to 'plugin install' -- one or more plugins to install // plugin names can be simple names for "standard" plugins, constraint suffixed names // or full refs to the OCI image @@ -373,6 +381,12 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args to 'plugin update' -- one or more plugins to update // These can be simple names for "standard" plugins, constraint suffixed names // or full refs to the OCI image @@ -623,6 +637,12 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + if len(args) == 0 { fmt.Println() //nolint:forbidigo // ui output error_helpers.ShowError(ctx, fmt.Errorf("you need to provide at least one plugin to uninstall")) @@ -685,6 +705,12 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resource(s) resources, err := display.ListPlugins(ctx) error_helpers.FailOnError(err) @@ -727,6 +753,12 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resource(s) resource, err := display.GetPluginResource(ctx, args[0]) error_helpers.FailOnError(err) diff --git a/cmd/query.go b/cmd/query.go index d767de7d..8bb0b3ea 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -12,6 +13,7 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/error_helpers" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/interactive" @@ -76,6 +78,12 @@ func runQueryCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // get a connection to the database var db *database.DuckDb db, err = openDatabaseConnection(ctx) diff --git a/cmd/root.go b/cmd/root.go index d33ca2fa..258b3a8b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,8 +1,6 @@ package cmd import ( - "os" - "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/turbot/pipe-fittings/v2/cmdconfig" @@ -10,7 +8,6 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/utils" - localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" ) @@ -63,12 +60,6 @@ func rootCommand() *cobra.Command { } func Execute() int { - // if diagnostic mode is set, print out config and return - if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { - localcmdconfig.DisplayConfig() - return 0 - } - rootCmd := rootCommand() utils.LogTime("cmd.root.Execute start") defer utils.LogTime("cmd.root.Execute end") diff --git a/cmd/source.go b/cmd/source.go index 1067c36d..972bfc97 100644 --- a/cmd/source.go +++ b/cmd/source.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -76,6 +78,12 @@ func runSourceListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListSourceResources(ctx) error_helpers.FailOnError(err) @@ -125,6 +133,12 @@ func runSourceShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resourceName := args[0] resource, err := display.GetSourceResource(ctx, resourceName) diff --git a/cmd/table.go b/cmd/table.go index 1ee88718..0ff868ff 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -14,6 +15,7 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -77,6 +79,12 @@ func runTableListCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListTableResources(ctx) error_helpers.FailOnError(err) @@ -127,6 +135,12 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resource, err := display.GetTableResource(ctx, args[0]) error_helpers.FailOnError(err) diff --git a/internal/cmdconfig/diagnostics.go b/internal/cmdconfig/diagnostics.go index b40a97ec..b5f4fe9d 100644 --- a/internal/cmdconfig/diagnostics.go +++ b/internal/cmdconfig/diagnostics.go @@ -51,7 +51,7 @@ func DisplayConfig() { sort.Strings(lines) var b strings.Builder - b.WriteString("\n================\nSteampipe Config\n================\n\n") + b.WriteString("\n================\nTailpipe Config\n================\n\n") for _, line := range lines { b.WriteString(line) diff --git a/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json b/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json new file mode 100644 index 00000000..f03b0759 --- /dev/null +++ b/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json @@ -0,0 +1,155 @@ +[ { + "test": "env variables set, no command line arguments set and no workspace env variable set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_PLUGIN_MEMORY_MAX_MB=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [] + }, + "expected": { + "update-check": "false", + "memory-max-mb": 16384, + "memory-max-mb-plugin": 2048, + "temp-dir-max-mb": 8192 + } + }, + { + "test": "only command line arguments set and no env variables set", + "description": "", + "cmd": "query", + "setup": { + "env": [], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "workspace env variable set and no command line arguments set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=workspace_profiles" + ], + "args": [] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "All env variables set and command line argument set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development", + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "workspace env variable set and --workspace arg passed", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "all env variables set including workspace env variable and no command line arguments set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development", + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [] + }, + "expected": { + "workspace": "development", + "log-level": "warn", + "update-check": "false", + "memory-max-mb": 512, + "memory-max-mb-plugin": 206, + "temp-dir-max-mb": 512 + } + }, + { + "test": "all env variables set except workspace env variable and --workspace arg passed", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + } +] \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc b/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc new file mode 100755 index 00000000..d38c3592 --- /dev/null +++ b/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc @@ -0,0 +1,16 @@ + +workspace "workspace_profiles" { + log_level = "trace" + update_check = false + memory_max_mb = 1024 + plugin_memory_max_mb = 512 + temp_dir_max_mb = 1024 +} + +workspace "development"{ + log_level = "warn" + update_check = false + memory_max_mb = 512 + plugin_memory_max_mb = 206 + temp_dir_max_mb = 512 +} \ No newline at end of file diff --git a/tests/acceptance/test_files/config_precedence.bats b/tests/acceptance/test_files/config_precedence.bats new file mode 100644 index 00000000..32c1e28e --- /dev/null +++ b/tests/acceptance/test_files/config_precedence.bats @@ -0,0 +1,121 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +## workspace tests + +@test "generic config precedence test" { + cp $SOURCE_FILES_DIR/config_tests/workspaces.tpc $TAILPIPE_INSTALL_DIR/config/workspaces.tpc + # setup test folder and read the test-cases file + cd $SOURCE_FILES_DIR/config_tests + tests=$(cat workspace_tests.json) + # echo $tests + + # to create the failure message + err="" + flag=0 + + # fetch the keys(test names) + test_keys=$(echo $tests | jq '. | keys[]') + # echo $test_keys + + for i in $test_keys; do + # each test case do the following + unset TAILPIPE_INSTALL_DIR + cwd=$(pwd) + export TAILPIPE_CONFIG_DUMP=config_json + + # command accordingly + cmd=$(echo $tests | jq -c ".[${i}]" | jq ".cmd") + if [[ $cmd == '"query"' ]]; then + tp_cmd='tailpipe query "select 1"' + fi + # echo $tp_cmd + + # key=$(echo $i) + echo -e "\n" + test_name=$(echo $tests | jq -c ".[${i}]" | jq ".test") + echo ">>> TEST NAME: $test_name" + + # env variables needed for setup + env=$(echo $tests | jq -c ".[${i}]" | jq ".setup.env") + # echo $env + + # set env variables + for e in $(echo "${env}" | jq -r '.[]'); do + export $e + done + + # args to run with tailpipe query command + args=$(echo $tests | jq -c ".[${i}]" | jq ".setup.args") + echo $args + + # construct the tailpipe command to be run with the args + for arg in $(echo "${args}" | jq -r '.[]'); do + tp_cmd="${tp_cmd} ${arg}" + done + echo "tailpipe command: $tp_cmd" # help debugging in case of failures + + # get the actual config by running the constructed tailpipe command + run $tp_cmd + echo "output from tailpipe command: $output" # help debugging in case of failures + + # The output contains log lines followed by a JSON object + # Find the start of the JSON (line starting with '{') and extract from there to the end + # Then use jq to parse and compact it + json_start_line=$(echo "$output" | grep -n '^{' | tail -1 | cut -d: -f1) + if [[ -n "$json_start_line" ]]; then + config_json=$(echo "$output" | tail -n +$json_start_line) + else + # Fallback: try to find any JSON-like content + config_json=$(echo "$output" | grep -A 1000 '{' | head -1000) + fi + + # Parse with jq and handle errors gracefully + actual_config=$(echo "$config_json" | jq -c '.' 2>/dev/null) + if [[ $? -ne 0 ]] || [[ -z "$actual_config" ]]; then + echo "Failed to parse JSON config, raw output:" + echo "$config_json" + actual_config="{}" + fi + echo "actual config: \n$actual_config" # help debugging in case of failures + + # get expected config from test case + expected_config=$(echo $tests | jq -c ".[${i}]" | jq ".expected") + # echo $expected_config + + # fetch only keys from expected config + exp_keys=$(echo $expected_config | jq '. | keys[]' | jq -s 'flatten | @sh' | tr -d '\'\' | tr -d '"') + + for key in $exp_keys; do + # get the expected and the actual value for the keys + exp_val=$(echo $(echo $expected_config | jq --arg KEY $key '.[$KEY]' | tr -d '"')) + act_val=$(echo $(echo $actual_config | jq --arg KEY $key '.[$KEY]' | tr -d '"')) + + # get the absolute paths for install-dir and mod-location + if [[ $key == "install-dir" ]] || [[ $key == "mod-location" ]]; then + exp_val="${cwd}/${exp_val}" + fi + echo "expected $key: $exp_val" + echo "actual $key: $act_val" + + # check the values + if [[ "$exp_val" != "$act_val" ]]; then + flag=1 + err="FAILED: $test_name >> key: $key ; expected: $exp_val ; actual: $act_val \n${err}" + fi + done + + # check if all passed + if [[ $flag -eq 0 ]]; then + echo "PASSED ✅" + else + echo "FAILED ❌" + fi + # reset flag back to 0 for the next test case + flag=0 + done + echo -e "\n" + echo -e "$err" + assert_equal "$err" "" + rm -f err +} From 69130369fc104605a2ea8d93553508d67a59320f Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Wed, 27 Aug 2025 20:51:08 +0530 Subject: [PATCH 20/61] Fix issue where the tea app was not closing correctly, resulting in a broken formatting in the terminal (#528) --- cmd/collect.go | 18 ++++++++++++------ internal/collector/collector.go | 5 +++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 3b276e4e..8a9a1c57 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -69,7 +69,12 @@ func runCollectCmd(cmd *cobra.Command, args []string) { } if err != nil { - error_helpers.ShowError(ctx, err) + if errors.Is(err, context.Canceled) { + err = nil + fmt.Println("Collection cancelled.") //nolint:forbidigo // ui output + } else { + error_helpers.ShowError(ctx, err) + } setExitCodeForCollectError(err) } }() @@ -81,11 +86,7 @@ func runCollectCmd(cmd *cobra.Command, args []string) { } err = doCollect(ctx, cancel, args) - if errors.Is(err, context.Canceled) { - // clear error so we don't show it with normal error reporting - err = nil - fmt.Println("Collection cancelled.") //nolint:forbidigo // ui output - } + } func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) error { @@ -302,6 +303,11 @@ func setExitCodeForCollectError(err error) { if exitCode != 0 || err == nil { return } + // TODO Set exit code for cancellation + if errors.Is(err, context.Canceled) { + exitCode = 0 + return + } // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/106 exitCode = 1 diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 3c90ffb9..c3630f44 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -99,6 +99,11 @@ func (c *Collector) Close() { c.parquetConvertor.Close() } + // close the tea app + if c.app != nil { + c.app.Quit() + } + // if inbox path is empty, remove it (ignore errors) _ = os.Remove(c.sourcePath) From 2d7dea7eb05c9f24510de585d7889492a470fce4 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 28 Aug 2025 19:30:27 +0530 Subject: [PATCH 21/61] Add GitHub Actions workflow to assign newly opened issues to the project (#518) --- .../31-add-issues-to-pipeling-issue-tracker.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml diff --git a/.github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml b/.github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml new file mode 100644 index 00000000..594defea --- /dev/null +++ b/.github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml @@ -0,0 +1,13 @@ +name: Assign Issue to Project + +on: + issues: + types: [opened] + +jobs: + add-to-project: + uses: turbot/steampipe-workflows/.github/workflows/assign-issue-to-pipeling-issue-tracker.yml@main + with: + issue_number: ${{ github.event.issue.number }} + repository: ${{ github.repository }} + secrets: inherit \ No newline at end of file From a9131993fff8ef2192791630dbb3aa17223f24f5 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 28 Aug 2025 19:34:47 +0530 Subject: [PATCH 22/61] test: add acceptance test for plugin metadata verification in versions.json (#495) --- .github/workflows/11-test-acceptance.yaml | 9 ++-- tests/acceptance/test_files/plugin.bats | 59 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/acceptance/test_files/plugin.bats diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 053af91e..69e4d59e 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -47,10 +47,10 @@ jobs: token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main - # this is required, check golangci-lint-action docs + # this is required, check golangci-lint-action docs - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: - go-version: '1.23' + go-version: "1.23" cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: Run CLI Unit Tests @@ -98,6 +98,7 @@ jobs: - "core_formats" - "table_block" - "config_precedence" + - "plugin" runs-on: ${{ matrix.platform }} steps: - name: Checkout @@ -116,7 +117,7 @@ jobs: id: prepare-for-downloads run: | mkdir ~/artifacts - + - name: Download Linux Build Artifacts uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 if: ${{ matrix.platform == 'ubuntu-latest' }} @@ -129,7 +130,7 @@ jobs: run: | mkdir ~/build tar -xf ~/artifacts/linux.tar.gz -C ~/build - + - name: Set PATH run: | echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/tailpipe/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV diff --git a/tests/acceptance/test_files/plugin.bats b/tests/acceptance/test_files/plugin.bats new file mode 100644 index 00000000..419efd0a --- /dev/null +++ b/tests/acceptance/test_files/plugin.bats @@ -0,0 +1,59 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify metadata in versions.json file after plugin install" { + # Ensure chaos plugin is installed (it should already be in acceptance tests) + run tailpipe plugin list --output json + echo $output + + # Verify chaos plugin is in the list + assert_output --partial "hub.tailpipe.io/plugins/turbot/chaos@latest" + + # Read the versions.json file + versions_file="$TAILPIPE_INSTALL_DIR/plugins/versions.json" + + # Verify the file exists + [ -f "$versions_file" ] + + # Read the file content + versions_content=$(cat "$versions_file") + echo "Versions file content: $versions_content" + + # Extract metadata for chaos plugin using jq + chaos_plugin_key="hub.tailpipe.io/plugins/turbot/chaos@latest" + + # Verify that metadata exists for the chaos plugin + metadata_exists=$(echo "$versions_content" | jq -r --arg key "$chaos_plugin_key" '.plugins | has($key) and (.[$key] | has("metadata"))') + assert_equal "$metadata_exists" "true" + + # Verify tables metadata - chaos plugin should have specific tables + tables=$(echo "$versions_content" | jq -r --arg key "$chaos_plugin_key" '.plugins[$key].metadata.tables // [] | sort | join(",")') + assert_equal "$tables" "chaos_all_columns,chaos_date_time,chaos_struct_columns" + + # Verify sources metadata - chaos plugin should have specific sources + sources=$(echo "$versions_content" | jq -r --arg key "$chaos_plugin_key" '.plugins[$key].metadata.sources // [] | sort | join(",")') + assert_equal "$sources" "chaos_all_columns,chaos_date_time,chaos_struct_columns" +} + +@test "verify format types and presets metadata exists in versions.json file after plugin install" { + # Read the versions.json file + versions_file="$TAILPIPE_INSTALL_DIR/plugins/versions.json" + + # Verify the file exists + [ -f "$versions_file" ] + + # Read the file content + versions_content=$(cat "$versions_file") + echo "Versions file content: $versions_content" + + # Test format_types and format_presets from core plugin (which has them) + core_plugin_key="hub.tailpipe.io/plugins/turbot/core@latest" + + # Verify format_types content - should contain the expected types + format_types=$(echo "$versions_content" | jq -r --arg key "$core_plugin_key" '.plugins[$key].metadata.format_types // [] | sort | join(",")') + assert_equal "$format_types" "delimited,grok,jsonl,regex" + + # Verify format_presets content - should contain the expected presets + format_presets=$(echo "$versions_content" | jq -r --arg key "$core_plugin_key" '.plugins[$key].metadata.format_presets // [] | sort | join(",")') + assert_equal "$format_presets" "delimited.default,jsonl.default" +} \ No newline at end of file From 438a7bf11fd2b56889ff430e542949e48be1b968 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 2 Sep 2025 16:52:01 +0530 Subject: [PATCH 23/61] fix(display): include partitions for local plugins in tailpipe plugin list command (#538) --- internal/display/plugin.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/display/plugin.go b/internal/display/plugin.go index 968928cb..56d49ddc 100644 --- a/internal/display/plugin.go +++ b/internal/display/plugin.go @@ -29,7 +29,7 @@ func (r *PluginListDetails) GetListData() *printers.RowData { func (r *PluginListDetails) setPartitions() { for _, partition := range config.GlobalConfig.Partitions { - if partition.Plugin.Plugin == r.Name { + if partition.Plugin.Plugin == r.Name || isLocalPluginPartition(r, partition.Plugin.Alias) { r.Partitions = append(r.Partitions, strings.TrimPrefix(partition.FullName, "partition.")) } } @@ -37,6 +37,18 @@ func (r *PluginListDetails) setPartitions() { slices.Sort(r.Partitions) } +// handle local plugins: r.Name (from filesystem) can be like "local/plugin-name" +// while partition.Plugin.Plugin is a full image ref like +// "hub.tailpipe.io/plugins/plugin-name/test@latest"; compare alias to last path segment +func isLocalPluginPartition(r *PluginListDetails, partitionAlias string) bool { + return r.Version == "local" && lastSegment(r.Name) == partitionAlias +} + +func lastSegment(s string) string { + p := strings.Split(strings.Trim(s, "/"), "/") + return p[len(p)-1] +} + func ListPlugins(ctx context.Context) ([]*PluginListDetails, error) { var res []*PluginListDetails From 043a1a1405c71c01387660d29ea48fb9082d8f89 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 2 Sep 2025 16:52:29 +0530 Subject: [PATCH 24/61] Update partition command usage to include required arguments for 'show' and 'delete' commands (#536) --- cmd/partition.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/partition.go b/cmd/partition.go index 66bc2fa6..44a3cb0f 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -112,7 +112,7 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { // Show Partition func partitionShowCmd() *cobra.Command { var cmd = &cobra.Command{ - Use: "show", + Use: "show ", Args: cobra.ExactArgs(1), Run: runPartitionShowCmd, Short: "Show details for a specific partition", @@ -168,7 +168,7 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { func partitionDeleteCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "delete ", + Use: "delete ", Args: cobra.ExactArgs(1), Run: runPartitionDeleteCmd, Short: "Delete a partition for the specified period", From 393b3080d72f98ceaddbfb6a13500f520057d649 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Tue, 2 Sep 2025 18:50:24 +0530 Subject: [PATCH 25/61] Remove redundant GH_ACCESS_TOKEN from workflow checkout steps (#542) --- .github/workflows/10-test-lint.yaml | 1 - .github/workflows/11-test-acceptance.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/10-test-lint.yaml b/.github/workflows/10-test-lint.yaml index 821ef7fc..6d753bac 100644 --- a/.github/workflows/10-test-lint.yaml +++ b/.github/workflows/10-test-lint.yaml @@ -31,7 +31,6 @@ jobs: with: repository: turbot/tailpipe-plugin-sdk path: tailpipe-plugin-sdk - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: develop - name: Checkout Tailpipe Core Plugin repository diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 69e4d59e..bb8aaff5 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -36,7 +36,6 @@ jobs: with: repository: turbot/tailpipe-plugin-sdk path: tailpipe-plugin-sdk - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: develop - name: Checkout Tailpipe Core Plugin repository From 3f711d8577dfd22ef1ff000701c8126ed3d0bab7 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Tue, 2 Sep 2025 21:03:41 +0530 Subject: [PATCH 26/61] Remove redundant GH_ACCESS_TOKEN from workflow checkout steps (#545) --- .github/workflows/10-test-lint.yaml | 1 - .github/workflows/11-test-acceptance.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/10-test-lint.yaml b/.github/workflows/10-test-lint.yaml index 6d753bac..049ba71e 100644 --- a/.github/workflows/10-test-lint.yaml +++ b/.github/workflows/10-test-lint.yaml @@ -38,7 +38,6 @@ jobs: with: repository: turbot/tailpipe-plugin-core path: tailpipe-plugin-core - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main # this is required, check golangci-lint-action docs diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index bb8aaff5..7217cf74 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -43,7 +43,6 @@ jobs: with: repository: turbot/tailpipe-plugin-core path: tailpipe-plugin-core - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main # this is required, check golangci-lint-action docs From a5fc23a04f57fef3a45545cf4a537cd3a3ffa77d Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 2 Sep 2025 16:48:39 +0100 Subject: [PATCH 27/61] Initial integration of DuckLake database backend. #546 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace native parquet conversion with a DuckLake database backend. Main Changes: - Data is now stored in a ducklake database rather than directly converting to parquet - Remove generation and usage of tailpipe.db DuckDb database - Update connect command to return duck db connection string - Implement partition deletion using DuckLake backend - Remove conversion workers - conversion now performed in a single thread by the Convertor - Minimise creation of DB connections, reusue where possible - Add schema change detection code [incomplete] - Add support for S3 for parquet data storage [incomplete] - Add schema comparison and validation capabilities [incomplete] - Add duration tracking to CompactionStatus - Move deletion of parquet files for collection range into collector - Update introspection commands to use ducklake - Remove DeleteParquetFiles manual deletion code NOTE: this feature is incomplete. Remaining key changes: • Add migration from native parquet to ducklake — https://github.com/turbot/tailpipe/issues/511 • Update index migration code for ducklake — https://github.com/turbot/tailpipe/issues/475 • Add support for backup and revert of metadata db — https://github.com/turbot/tailpipe/issues/512 • Implement/test Powerpipe Ducklake backend support (done but needs testing) — https://github.com/turbot/powerpipe/issues/889 • Review/test ducklake schema change detection (done but needs testing) — https://github.com/turbot/tailpipe/issues/481 • Re-add required column validation to ducklake conversion — https://github.com/turbot/tailpipe/issues/479 • Re-add error handling to ducklake conversion process — https://github.com/turbot/tailpipe/issues/480 --- .acceptance.goreleaser.yml | 12 +- .cursor/rules/general.mdc | 20 + .darwin.goreleaser.yml | 58 + .github/workflows/01-tailpipe-release.yaml | 2 + .github/workflows/11-test-acceptance.yaml | 15 +- .gitignore | 8 +- .golangci.yml | 1 + .goreleaser.yml | 64 +- Dockerfile.goreleaser-cross | 40 + Makefile | 100 +- cmd/collect.go | 118 +- cmd/collect_test.go | 136 ++ cmd/compact.go | 51 +- cmd/connect.go | 372 +--- cmd/connect_test.go | 194 --- cmd/partition.go | 82 +- cmd/query.go | 19 +- cmd/table.go | 13 +- go.mod | 126 +- go.sum | 995 +++++++++-- internal/cmdconfig/cmd_hooks.go | 33 +- internal/collector/collector.go | 253 +-- internal/collector/collector_synthetic.go | 584 +++++++ internal/collector/status.go | 34 +- internal/config/connection.go | 8 +- internal/config/partition.go | 10 + internal/constants/database.go | 8 - internal/constants/duckdb_extensions.go | 3 - internal/constants/metaquery_commands.go | 5 +- internal/database/create.go | 27 - internal/database/duck_db.go | 53 +- internal/database/duck_db_error.go | 27 +- internal/database/duck_db_options.go | 7 + internal/database/duck_db_test.go | 28 +- internal/database/partitions.go | 52 - internal/database/tables.go | 205 +-- internal/display/partition.go | 69 +- internal/display/shared.go | 37 +- internal/display/table.go | 84 +- internal/filepaths/database.go | 13 - internal/filepaths/parquet.go | 37 - internal/filepaths/partition_fields.go | 72 - internal/filepaths/partition_fields_test.go | 116 -- internal/helpers/sort.go | 24 - internal/interactive/interactive_client.go | 29 +- .../interactive_client_autocomplete.go | 3 - internal/interactive/interactive_helpers.go | 5 - internal/metaquery/handler_input.go | 13 +- internal/metaquery/handler_inspect.go | 26 +- internal/parquet/compact.go | 394 +++-- internal/parquet/compaction_status.go | 87 +- internal/parquet/compaction_types.go | 136 ++ internal/parquet/conversion_error.go | 33 +- internal/parquet/conversion_worker.go | 584 ------- internal/parquet/conversion_worker_test.go | 1521 ----------------- internal/parquet/convertor.go | 324 ++-- internal/parquet/convertor_convert.go | 265 +++ internal/parquet/convertor_ducklake.go | 167 ++ internal/parquet/convertor_infer.go | 186 -- internal/parquet/convertor_schema.go | 275 ++- internal/parquet/convertor_validate.go | 117 ++ internal/parquet/delete.go | 153 -- internal/parquet/delete_test.go | 356 ---- internal/parquet/ducklake.go | 141 ++ internal/parquet/file_helpers.go | 20 - internal/parquet/file_helpers_test.go | 112 -- internal/parquet/file_metadata.go | 103 ++ internal/parquet/file_root_provider.go | 35 - internal/parquet/migrate_tpindex.go | 333 ++-- internal/parquet/partition_key.go | 259 +++ internal/parquet/partition_key_test.go | 383 +++++ internal/parquet/read_json_query.go | 176 ++ ...schema_test.go => read_json_query_test.go} | 0 internal/parquet/schema_comparison.go | 64 + internal/parse/load_config_test.go | 1022 +++++++++-- .../connections_config/resources.tpc | 13 + .../custom_table_config/resources.tpc | 4 +- .../invalid_partition_labels/resources.tpc | 4 + .../test_data/malformed_config/resources.tpc | 5 + internal/plugin/installation_actions.go | 2 +- internal/plugin/plugin_manager.go | 2 +- internal/query/execute.go | 2 +- memtest/go.mod | 35 - memtest/go.sum | 74 - memtest/main.go | 254 --- memtest/memory_results_.csv | 37 - memtest/run_mem.sh | 59 - memtest/short.jsonl | 1 - memtest/short2.jsonl | 1 - memtest/testdata/generate.go | 167 -- memtest/testdata/generate_all.sh | 32 - memtest/testdata/subset/subset_creator.go | 73 - tailpipe_data_generator/go.mod | 5 - tailpipe_data_generator/main.go | 129 -- tests/acceptance/test_files/from_and_to.bats | 4 + .../test_files/partition_tests.bats | 1 + 96 files changed, 6046 insertions(+), 6395 deletions(-) create mode 100644 .cursor/rules/general.mdc create mode 100644 .darwin.goreleaser.yml create mode 100644 Dockerfile.goreleaser-cross delete mode 100644 cmd/connect_test.go create mode 100644 internal/collector/collector_synthetic.go delete mode 100644 internal/constants/database.go delete mode 100644 internal/constants/duckdb_extensions.go delete mode 100644 internal/database/create.go delete mode 100644 internal/database/partitions.go delete mode 100644 internal/filepaths/database.go delete mode 100644 internal/filepaths/parquet.go delete mode 100644 internal/filepaths/partition_fields.go delete mode 100644 internal/filepaths/partition_fields_test.go delete mode 100644 internal/helpers/sort.go create mode 100644 internal/parquet/compaction_types.go delete mode 100644 internal/parquet/conversion_worker.go delete mode 100644 internal/parquet/conversion_worker_test.go create mode 100644 internal/parquet/convertor_convert.go create mode 100644 internal/parquet/convertor_ducklake.go delete mode 100644 internal/parquet/convertor_infer.go create mode 100644 internal/parquet/convertor_validate.go delete mode 100644 internal/parquet/delete.go delete mode 100644 internal/parquet/delete_test.go create mode 100644 internal/parquet/ducklake.go create mode 100644 internal/parquet/file_metadata.go delete mode 100644 internal/parquet/file_root_provider.go create mode 100644 internal/parquet/partition_key.go create mode 100644 internal/parquet/partition_key_test.go create mode 100644 internal/parquet/read_json_query.go rename internal/parquet/{convertor_schema_test.go => read_json_query_test.go} (100%) create mode 100644 internal/parquet/schema_comparison.go create mode 100644 internal/parse/test_data/connections_config/resources.tpc create mode 100644 internal/parse/test_data/invalid_partition_labels/resources.tpc create mode 100644 internal/parse/test_data/malformed_config/resources.tpc delete mode 100644 memtest/go.mod delete mode 100644 memtest/go.sum delete mode 100644 memtest/main.go delete mode 100644 memtest/memory_results_.csv delete mode 100755 memtest/run_mem.sh delete mode 100644 memtest/short.jsonl delete mode 100644 memtest/short2.jsonl delete mode 100644 memtest/testdata/generate.go delete mode 100755 memtest/testdata/generate_all.sh delete mode 100644 memtest/testdata/subset/subset_creator.go delete mode 100644 tailpipe_data_generator/go.mod delete mode 100644 tailpipe_data_generator/main.go diff --git a/.acceptance.goreleaser.yml b/.acceptance.goreleaser.yml index f96f03d5..d764494f 100644 --- a/.acceptance.goreleaser.yml +++ b/.acceptance.goreleaser.yml @@ -1,3 +1,6 @@ +# Acceptance testing configuration - builds only Linux AMD64 for faster testing +version: 2 + before: hooks: - go mod tidy @@ -10,12 +13,17 @@ builds: - amd64 env: - - CC=x86_64-linux-gnu-gcc - - CXX=x86_64-linux-gnu-g++ + - CC=x86_64-linux-gnu-gcc-13 + - CXX=x86_64-linux-gnu-g++-13 + - CGO_CXXFLAGS=-std=c++17 + - CGO_LDFLAGS=-lstdc++ -static-libstdc++ ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + flags: + - -buildvcs=false + archives: - id: homebrew format: tar.gz diff --git a/.cursor/rules/general.mdc b/.cursor/rules/general.mdc new file mode 100644 index 00000000..1e956cb2 --- /dev/null +++ b/.cursor/rules/general.mdc @@ -0,0 +1,20 @@ +--- +description: +globs: +alwaysApply: false +--- +# general rules to always apply +## confirmation/avoid too much initiative +- DO not make any change I have not explicitly asked for +- NEVER make any changes if I have only asked you a question but not explicitly asked you to make an action +- Ask for confirmation before making ANY changes, with a summary of what you will do +## format +- Use lower case for sql always +## general attitude +- Use a neutral tone of voice and do not be too positive/enthusiastic. + - When I report a problem, do NOT say "perfect I see the problem" as that sounds like you know the solution + - When you have made a change do NOT say "now everything will be working" until you have confirmation that it does work + - Always look at my ideas and suggestions critically and look for flaws in my logic + - + + \ No newline at end of file diff --git a/.darwin.goreleaser.yml b/.darwin.goreleaser.yml new file mode 100644 index 00000000..b1932ae1 --- /dev/null +++ b/.darwin.goreleaser.yml @@ -0,0 +1,58 @@ +# Darwin-only goreleaser configuration +version: 2 + +before: + hooks: + - go mod tidy + +builds: + # Darwin AMD64 build with clang + - id: tailpipe-darwin-amd64 + binary: tailpipe + goos: + - darwin + goarch: + - amd64 + env: + - CC=o64-clang + - CXX=o64-clang++ + ldflags: + - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + flags: + - -buildvcs=false + + # Darwin ARM64 build with clang + - id: tailpipe-darwin-arm64 + binary: tailpipe + goos: + - darwin + goarch: + - arm64 + env: + - CC=oa64-clang + - CXX=oa64-clang++ + ldflags: + - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + flags: + - -buildvcs=false + +archives: + - id: darwin + format: tar.gz + name_template: "{{ .ProjectName }}.{{ .Os }}.{{ .Arch }}" + files: + - none* + +checksum: + name_template: 'checksums.txt' + +snapshot: + name_template: "{{ incpatch .Version }}-next" + +changelog: + disable: true + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/.github/workflows/01-tailpipe-release.yaml b/.github/workflows/01-tailpipe-release.yaml index bdb57cf7..c2e8e2ea 100644 --- a/.github/workflows/01-tailpipe-release.yaml +++ b/.github/workflows/01-tailpipe-release.yaml @@ -143,6 +143,8 @@ jobs: - name: Release publish run: |- cd tailpipe + git config --global user.name "Tailpipe GitHub Actions Bot" + git config --global user.email noreply@github.com make release create_pr_in_homebrew: diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 7217cf74..22c89beb 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -57,6 +57,13 @@ jobs: go clean -testcache go test -timeout 30s ./... -test.v + - name: Build Custom Goreleaser Cross Image for Linux + run: |- + cd tailpipe + make build-goreleaser-image + echo "=== Verifying GCC 13 ===" + docker run --rm tailpipe-goreleaser-cross:gcc13 gcc-13 --version | head -1 + - name: Build run: |- cd tailpipe @@ -123,12 +130,12 @@ jobs: name: build-artifact-linux path: ~/artifacts - - name: Extract Ubuntu Artifacts and Install Binary + - name: Extract Linux Artifacts and Install Binary if: ${{ matrix.platform == 'ubuntu-latest' }} run: | mkdir ~/build tar -xf ~/artifacts/linux.tar.gz -C ~/build - + - name: Set PATH run: | echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/tailpipe/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV @@ -181,8 +188,8 @@ jobs: name: build-artifact-linux failOnError: true - - name: Clean up Darwin Build + - name: Clean up Linux Build uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0 with: - name: build-artifact-darwin + name: build-artifact-linux failOnError: true diff --git a/.gitignore b/.gitignore index 6f1db318..e16ec82a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,8 @@ *.dll *.so *.dylib - +/test_apps/ +/memtest # Editor cache and lock files *.swp *.swo @@ -28,4 +29,7 @@ go.work # Dist directory is created by goreleaser -/dist \ No newline at end of file +/dist + +# Sysroot directory is created by make build-sysroot +/sysroot \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index b6748a19..f42ba77e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -70,3 +70,4 @@ run: issues: exclude-dirs: - "tests/acceptance" + - "test_apps" diff --git a/.goreleaser.yml b/.goreleaser.yml index d8e8a667..2ac9f062 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,67 +1,43 @@ version: 2 -builds: - - id: tailpipe-linux-arm64 - binary: tailpipe - goos: - - linux - goarch: - - arm64 - - env: - - CC=aarch64-linux-gnu-gcc - - CXX=aarch64-linux-gnu-g++ - - # Custom ldflags. - # - # Default: '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser' - # Templates: allowed - ldflags: - # Go Releaser analyzes your Git repository and identifies the most recent Git tag (typically the highest version number) as the version for your release. - # This is how it determines the value of {{.Version}}. - - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser +before: + hooks: + - go mod tidy +builds: + # Linux AMD64 build with GCC 13+ - id: tailpipe-linux-amd64 binary: tailpipe goos: - linux goarch: - amd64 - env: - - CC=x86_64-linux-gnu-gcc - - CXX=x86_64-linux-gnu-g++ - + - CC=x86_64-linux-gnu-gcc-13 + - CXX=x86_64-linux-gnu-g++-13 + - CGO_CXXFLAGS=-std=c++17 + - CGO_LDFLAGS=-lstdc++ -static-libstdc++ ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + flags: + - -buildvcs=false - - id: tailpipe-darwin-arm64 + # Linux ARM64 build with GCC 13+ + - id: tailpipe-linux-arm64 binary: tailpipe goos: - - darwin + - linux goarch: - arm64 - - env: - - CC=oa64-clang - - CXX=oa64-clang++ - - ldflags: - - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - - - id: tailpipe-darwin-amd64 - binary: tailpipe - goos: - - darwin - goarch: - - amd64 - env: - - CC=o64-clang - - CXX=o64-clang++ - + - CC=aarch64-linux-gnu-gcc-13 + - CXX=aarch64-linux-gnu-g++-13 + - CGO_CXXFLAGS=-std=c++17 + - CGO_LDFLAGS=-lstdc++ -static-libstdc++ ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + flags: + - -buildvcs=false release: prerelease: auto diff --git a/Dockerfile.goreleaser-cross b/Dockerfile.goreleaser-cross new file mode 100644 index 00000000..72dba423 --- /dev/null +++ b/Dockerfile.goreleaser-cross @@ -0,0 +1,40 @@ +# Use Ubuntu 24.04 as base to get GCC 13+ +FROM ubuntu:noble + +# Install essential packages and cross-compilation tools +RUN apt-get update && apt-get install -y \ + wget curl git build-essential \ + gcc-13 g++-13 \ + gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \ + gcc-x86-64-linux-gnu g++-x86-64-linux-gnu \ + gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf \ + && rm -rf /var/lib/apt/lists/* + +# Install Go 1.24.5 +RUN wget https://go.dev/dl/go1.24.5.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go1.24.5.linux-amd64.tar.gz && \ + rm go1.24.5.linux-amd64.tar.gz + +# Install goreleaser 2.11.2 +RUN wget https://github.com/goreleaser/goreleaser/releases/download/v2.11.2/goreleaser_Linux_x86_64.tar.gz && \ + tar -xzf goreleaser_Linux_x86_64.tar.gz && \ + mv goreleaser /usr/local/bin/ && \ + rm goreleaser_Linux_x86_64.tar.gz + +# Set up environment +ENV PATH="/usr/local/go/bin:${PATH}" +ENV GOROOT="/usr/local/go" +ENV GOPATH="/go" +ENV CGO_ENABLED=1 + +# Set default cross-compilation environment variables for Linux AMD64 +ENV CC_linux_amd64=x86_64-linux-gnu-gcc-13 +ENV CXX_linux_amd64=x86_64-linux-gnu-g++-13 +ENV CGO_CXXFLAGS_linux_amd64="-std=c++17" +ENV CGO_LDFLAGS_linux_amd64="-lstdc++ -static-libstdc++" + +# Create working directory +WORKDIR /go/src + +# Set entrypoint to goreleaser +ENTRYPOINT ["goreleaser"] diff --git a/Makefile b/Makefile index be9846cb..83337548 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ OUTPUT_DIR?=/usr/local/bin PACKAGE_NAME := github.com/turbot/tailpipe -GOLANG_CROSS_VERSION ?= v1.23.2 +GOLANG_CROSS_VERSION ?= v1.25.0 # sed 's/[\/_]/-/g': Replaces both slashes (/) and underscores (_) with hyphens (-). -# sed 's/[^a-zA-Z0-9.-]//g': Removes any character that isn’t alphanumeric, a dot (.), or a hyphen (-). +# sed 's/[^a-zA-Z0-9.-]//g': Removes any character that isn't alphanumeric, a dot (.), or a hyphen (-). # This is to ensure that the branch name is a valid semver pre-release identifier. .PHONY: build build: @@ -12,8 +12,13 @@ build: go build -o $(OUTPUT_DIR) -ldflags "-X main.version=0.0.0-dev-$(GIT_BRANCH).$(TIMESTAMP)" . +.PHONY: build-goreleaser-image +build-goreleaser-image: + docker build -f Dockerfile.goreleaser-cross -t tailpipe-goreleaser-cross:gcc13 . + .PHONY: release-dry-run -release-dry-run: +release-dry-run: build-goreleaser-image + @echo "Building for Linux platforms using custom image with GCC 13+..." @docker run \ --rm \ -e CGO_ENABLED=1 \ @@ -23,11 +28,12 @@ release-dry-run: -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + tailpipe-goreleaser-cross:gcc13 \ --clean --skip=validate --skip=publish --snapshot .PHONY: release-acceptance -release-acceptance: +release-acceptance: build-goreleaser-image + @echo "Building for acceptance testing using custom image with GCC 13+..." @docker run \ --rm \ -e CGO_ENABLED=1 \ @@ -37,16 +43,21 @@ release-acceptance: -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + tailpipe-goreleaser-cross:gcc13 \ --clean --skip=validate --skip=publish --snapshot --config=.acceptance.goreleaser.yml .PHONY: release -release: +release: build-goreleaser-image @if [ ! -f ".release-env" ]; then \ echo ".release-env is required for release";\ exit 1;\ fi - docker run \ + @echo "Building for all platforms (Linux + Darwin) for release..." + @echo "Linux builds: Using custom image with GCC 13+" + @echo "Darwin builds: Using standard goreleaser-cross" + @echo "" + @echo "Building Linux targets..." + @docker run \ --rm \ -e CGO_ENABLED=1 \ --env-file .release-env \ @@ -56,5 +67,76 @@ release: -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + tailpipe-goreleaser-cross:gcc13 \ release --clean --skip=validate + @echo "" + @echo "Building Darwin targets..." + @docker run \ + --rm \ + -e CGO_ENABLED=1 \ + --env-file .release-env \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/tailpipe \ + -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ + -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ + -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ + -w /go/src/tailpipe \ + ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + release --clean --skip=validate --config=.darwin.goreleaser.yml + @echo "" + @echo "✅ Release builds completed successfully!" + @echo "📦 Linux builds: AMD64, ARM64" + @echo "🍎 Darwin builds: AMD64, ARM64" + +# Darwin-only builds using standard goreleaser-cross +.PHONY: release-darwin +release-darwin: + @echo "Building Darwin targets using standard goreleaser-cross..." + @docker run \ + --rm \ + -e CGO_ENABLED=1 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/tailpipe \ + -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ + -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ + -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ + -w /go/src/tailpipe \ + ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + --clean --skip=validate --skip=publish --snapshot --config=.darwin.goreleaser.yml + +# Build for all platforms (Linux + Darwin) - UNIFIED APPROACH +.PHONY: release-all-platforms +release-all-platforms: build-goreleaser-image + @echo "Building for all platforms using unified approach..." + @echo "Linux builds: Using custom image with GCC 13+" + @echo "Darwin builds: Using standard goreleaser-cross" + @echo "" + @echo "Building Linux targets..." + @docker run \ + --rm \ + -e CGO_ENABLED=1 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/tailpipe \ + -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ + -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ + -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ + -w /go/src/tailpipe \ + tailpipe-goreleaser-cross:gcc13 \ + --clean --skip=validate --skip=publish --snapshot + @echo "" + @echo "Building Darwin targets..." + @docker run \ + --rm \ + -e CGO_ENABLED=1 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/tailpipe \ + -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ + -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ + -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ + -w /go/src/tailpipe \ + ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + --clean --skip=validate --skip=publish --snapshot --config=.darwin.goreleaser.yml + @echo "" + @echo "✅ All platform builds completed successfully!" + @echo "📦 Linux builds: AMD64, ARM64" + @echo "🍎 Darwin builds: AMD64, ARM64" diff --git a/cmd/collect.go b/cmd/collect.go index 8a9a1c57..7ad4f440 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,11 +5,13 @@ import ( "errors" "fmt" "log/slog" + "strconv" "os" "strings" "time" "github.com/danwakefield/fnmatch" + "github.com/hashicorp/hcl/v2" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" @@ -17,12 +19,12 @@ import ( pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" ) @@ -131,18 +133,8 @@ func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) er // collect each partition serially var errList []error + for _, partition := range partitions { - // if a from time is set, clear the partition data from that time forward - if !fromTime.IsZero() && viper.GetBool(pconstants.ArgOverwrite) { - slog.Info("Deleting parquet files after the from time", "partition", partition.Name, "from", fromTime) - _, err := parquet.DeleteParquetFiles(partition, fromTime) - if err != nil { - slog.Warn("Failed to delete parquet files after the from time", "partition", partition.Name, "from", fromTime, "error", err) - errList = append(errList, err) - continue - } - slog.Info("Completed deleting parquet files after the from time", "partition", partition.Name, "from", fromTime) - } // do the collection err = collectPartition(ctx, cancel, partition, fromTime, toTime, pluginManager) if err != nil { @@ -169,15 +161,16 @@ func validateCollectionTimeRange(fromTime time.Time, toTime time.Time) error { } func collectPartition(ctx context.Context, cancel context.CancelFunc, partition *config.Partition, fromTime time.Time, toTime time.Time, pluginManager *plugin.PluginManager) (err error) { + t := time.Now() c, err := collector.New(pluginManager, partition, cancel) if err != nil { return fmt.Errorf("failed to create collector: %w", err) } defer c.Close() - recollect := viper.GetBool(pconstants.ArgOverwrite) + overwrite := viper.GetBool(pconstants.ArgOverwrite) - if err = c.Collect(ctx, fromTime, toTime, recollect); err != nil { + if err = c.Collect(ctx, fromTime, toTime, overwrite); err != nil { return err } @@ -188,13 +181,14 @@ func collectPartition(ctx context.Context, cancel context.CancelFunc, partition return err } - slog.Info("Collection complete", "partition", partition.Name) + slog.Info("Collection complete", "partition", partition.Name, "duration", time.Since(t).Seconds()) // compact the parquet files if viper.GetBool(pconstants.ArgCompact) { err = c.Compact(ctx) if err != nil { return err } + } // update status to show complete and display collection summary @@ -203,6 +197,7 @@ func collectPartition(ctx context.Context, cancel context.CancelFunc, partition return nil } +// getPartitions resolves the provided args to a list of partitions. func getPartitions(args []string) ([]*config.Partition, error) { // we have loaded tailpipe config by this time tailpipeConfig := config.GlobalConfig @@ -216,6 +211,11 @@ func getPartitions(args []string) ([]*config.Partition, error) { var partitions []*config.Partition for _, arg := range args { + if syntheticPartition, ok := getSyntheticPartition(arg); ok { + partitions = append(partitions, syntheticPartition) + continue + } + partitionNames, err := getPartitionsForArg(maps.Keys(tailpipeConfig.Partitions), arg) if err != nil { errorList = append(errorList, err) @@ -229,13 +229,97 @@ func getPartitions(args []string) ([]*config.Partition, error) { } if len(errorList) > 0 { - // TODO #errors better formating/error message https://github.com/turbot/tailpipe/issues/106 + // TODO #errors better formating/error message https://github.com/turbot/tailpipe/issues/497 return nil, errors.Join(errorList...) } return partitions, nil } +func getSyntheticPartition(arg string) (*config.Partition, bool) { + // synthetic partitions are of form synthetic_50cols_2000000rows_10000chunk_100ms + // determine if this partition is synthetic and if so try to parse the params + + // Check if this is a synthetic partition + if !strings.HasPrefix(arg, "synthetic_") { + return nil, false + } + + // Parse the synthetic partition parameters + // Format: synthetic_cols_rows_chunk_ms + parts := strings.Split(arg, "_") + if len(parts) != 5 { + // Invalid format, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid format", "arg", arg, "parts", len(parts), "expected", 5) + return nil, false + } + + // Extract and parse the numeric values + colsStr := strings.TrimSuffix(parts[1], "cols") + rowsStr := strings.TrimSuffix(parts[2], "rows") + chunkStr := strings.TrimSuffix(parts[3], "chunk") + intervalStr := strings.TrimSuffix(parts[4], "ms") + + cols, err := strconv.Atoi(colsStr) + if err != nil { + // Invalid columns value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid columns value", "arg", arg, "colsStr", colsStr, "error", err) + return nil, false + } + + rows, err := strconv.Atoi(rowsStr) + if err != nil { + // Invalid rows value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid rows value", "arg", arg, "rowsStr", rowsStr, "error", err) + return nil, false + } + + chunk, err := strconv.Atoi(chunkStr) + if err != nil { + // Invalid chunk value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid chunk value", "arg", arg, "chunkStr", chunkStr, "error", err) + return nil, false + } + + interval, err := strconv.Atoi(intervalStr) + if err != nil { + // Invalid interval value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid interval value", "arg", arg, "intervalStr", intervalStr, "error", err) + return nil, false + } + + // Validate the parsed values + if cols <= 0 || rows <= 0 || chunk <= 0 || interval <= 0 { + // Invalid values, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid values", "arg", arg, "cols", cols, "rows", rows, "chunk", chunk, "interval", interval) + return nil, false + } + + // Create a synthetic partition with proper HCL block structure + block := &hcl.Block{ + Type: "partition", + Labels: []string{"synthetic", arg}, + } + + partition := &config.Partition{ + HclResourceImpl: modconfig.NewHclResourceImpl(block, fmt.Sprintf("partition.synthetic.%s", arg)), + TableName: "synthetic", + TpIndexColumn: "'default'", + SyntheticMetadata: &config.SyntheticMetadata{ + Columns: cols, + Rows: rows, + ChunkSize: chunk, + DeliveryIntervalMs: interval, + }, + } + + // Set the unqualified name + partition.UnqualifiedName = fmt.Sprintf("%s.%s", partition.TableName, partition.ShortName) + + slog.Debug("Synthetic partition parsed successfully", "arg", arg, "columns", cols, "rows", rows, "chunkSize", chunk, "deliveryIntervalMs", interval) + return partition, true +} + func getPartitionsForArg(partitions []string, arg string) ([]string, error) { tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) if err != nil { @@ -309,7 +393,7 @@ func setExitCodeForCollectError(err error) { return } - // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/106 + // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/496 exitCode = 1 } diff --git a/cmd/collect_test.go b/cmd/collect_test.go index 725c27dc..a5b27db8 100644 --- a/cmd/collect_test.go +++ b/cmd/collect_test.go @@ -3,6 +3,8 @@ package cmd import ( "reflect" "testing" + + "github.com/turbot/tailpipe/internal/config" ) func Test_getPartition(t *testing.T) { @@ -253,3 +255,137 @@ func Test_getPartitionMatchPatternsForArg(t *testing.T) { }) } } + +func Test_getSyntheticPartition(t *testing.T) { + tests := []struct { + name string + arg string + wantPart *config.Partition + wantOk bool + }{ + { + name: "Valid synthetic partition", + arg: "synthetic_50cols_2000000rows_10000chunk_100ms", + wantOk: true, + wantPart: &config.Partition{ + TableName: "synthetic", + SyntheticMetadata: &config.SyntheticMetadata{ + Columns: 50, + Rows: 2000000, + ChunkSize: 10000, + DeliveryIntervalMs: 100, + }, + }, + }, + { + name: "Not a synthetic partition", + arg: "aws_cloudtrail_log.p1", + wantOk: false, + }, + { + name: "Invalid synthetic partition format - too few parts", + arg: "synthetic_50cols_2000000rows_10000chunk", + wantOk: false, + }, + { + name: "Invalid synthetic partition format - too many parts", + arg: "synthetic_50cols_2000000rows_10000chunk_100ms_extra", + wantOk: false, + }, + { + name: "Invalid synthetic partition - non-numeric columns", + arg: "synthetic_abccols_2000000rows_10000chunk_100ms", + wantOk: false, + }, + { + name: "Invalid synthetic partition - non-numeric rows", + arg: "synthetic_50cols_abcrows_10000chunk_100ms", + wantOk: false, + }, + { + name: "Invalid synthetic partition - non-numeric chunk", + arg: "synthetic_50cols_2000000rows_abcchunk_100ms", + wantOk: false, + }, + { + name: "Invalid synthetic partition - non-numeric interval", + arg: "synthetic_50cols_2000000rows_10000chunk_abcms", + wantOk: false, + }, + { + name: "Invalid synthetic partition - zero values", + arg: "synthetic_0cols_2000000rows_10000chunk_100ms", + wantOk: false, + }, + { + name: "Invalid synthetic partition - negative values", + arg: "synthetic_-50cols_2000000rows_10000chunk_100ms", + wantOk: false, + }, + { + name: "Invalid synthetic partition - zero interval", + arg: "synthetic_50cols_2000000rows_10000chunk_0ms", + wantOk: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPart, gotOk := getSyntheticPartition(tt.arg) + if gotOk != tt.wantOk { + t.Errorf("getSyntheticPartition() gotOk = %v, want %v", gotOk, tt.wantOk) + return + } + if gotOk { + if gotPart.TableName != tt.wantPart.TableName { + t.Errorf("getSyntheticPartition() TableName = %v, want %v", gotPart.TableName, tt.wantPart.TableName) + } + if gotPart.SyntheticMetadata == nil { + t.Errorf("getSyntheticPartition() SyntheticMetadata is nil") + return + } + if gotPart.SyntheticMetadata.Columns != tt.wantPart.SyntheticMetadata.Columns { + t.Errorf("getSyntheticPartition() Columns = %v, want %v", gotPart.SyntheticMetadata.Columns, tt.wantPart.SyntheticMetadata.Columns) + } + if gotPart.SyntheticMetadata.Rows != tt.wantPart.SyntheticMetadata.Rows { + t.Errorf("getSyntheticPartition() Rows = %v, want %v", gotPart.SyntheticMetadata.Rows, tt.wantPart.SyntheticMetadata.Rows) + } + if gotPart.SyntheticMetadata.ChunkSize != tt.wantPart.SyntheticMetadata.ChunkSize { + t.Errorf("getSyntheticPartition() ChunkSize = %v, want %v", gotPart.SyntheticMetadata.ChunkSize, tt.wantPart.SyntheticMetadata.ChunkSize) + } + if gotPart.SyntheticMetadata.DeliveryIntervalMs != tt.wantPart.SyntheticMetadata.DeliveryIntervalMs { + t.Errorf("getSyntheticPartition() DeliveryIntervalMs = %v, want %v", gotPart.SyntheticMetadata.DeliveryIntervalMs, tt.wantPart.SyntheticMetadata.DeliveryIntervalMs) + } + } + }) + } +} + +func Test_getSyntheticPartition_Logging(t *testing.T) { + // Test that logging works for various failure scenarios + testCases := []struct { + name string + arg string + }{ + {"Invalid format", "synthetic_50cols_2000000rows_10000chunk"}, + {"Invalid columns", "synthetic_abccols_2000000rows_10000chunk_100ms"}, + {"Invalid rows", "synthetic_50cols_abcrows_10000chunk_100ms"}, + {"Invalid chunk", "synthetic_50cols_2000000rows_abcchunk_100ms"}, + {"Invalid interval", "synthetic_50cols_2000000rows_10000chunk_abcms"}, + {"Zero values", "synthetic_0cols_2000000rows_10000chunk_100ms"}, + {"Valid partition", "synthetic_50cols_2000000rows_10000chunk_100ms"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // This test ensures the function doesn't panic and handles logging gracefully + // The actual log output would be visible when running with debug level enabled + _, ok := getSyntheticPartition(tc.arg) + + // Just verify the function completes without error + // The logging is a side effect that we can't easily test without capturing log output + if tc.name == "Valid partition" && !ok { + t.Errorf("Expected valid partition to return true") + } + }) + } +} diff --git a/cmd/compact.go b/cmd/compact.go index 87cad3fa..10ffd7e0 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -8,11 +8,11 @@ import ( "os" "time" - "golang.org/x/exp/maps" - "github.com/briandowns/spinner" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" + "github.com/turbot/go-kit/types" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" @@ -20,9 +20,14 @@ import ( localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/parquet" + "golang.org/x/exp/maps" ) +// TODO #DL update docs - no longer support compacting single partition +// +// https://github.com/turbot/tailpipe/issues/474 func compactCmd() *cobra.Command { cmd := &cobra.Command{ Use: "compact [table|table.partition] [flags]", @@ -60,6 +65,17 @@ func runCompactCmd(cmd *cobra.Command, args []string) { slog.Info("Compacting parquet files") + // if the flag was provided, migrate the tp_index files + if viper.GetBool(pconstants.ArgReindex) { + // TODO #DL update tpIndex migration for ducklake + // https://github.com/turbot/tailpipe/issues/475 + panic("Reindexing is not yet implemented for ducklake") + } + + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + error_helpers.FailOnError(err) + defer db.Close() + // verify that the provided args resolve to at least one partition if _, err := getPartitions(args); err != nil { error_helpers.FailOnError(err) @@ -69,8 +85,10 @@ func runCompactCmd(cmd *cobra.Command, args []string) { patterns, err := getPartitionPatterns(args, maps.Keys(config.GlobalConfig.Partitions)) error_helpers.FailOnErrorWithMessage(err, "failed to get partition patterns") - status, err := doCompaction(ctx, patterns...) + // do the compaction + status, err := doCompaction(ctx, db, patterns) if errors.Is(err, context.Canceled) { + // TODO verify // clear error so we don't show it with normal error reporting err = nil } @@ -92,7 +110,7 @@ func runCompactCmd(cmd *cobra.Command, args []string) { // defer block will show the error } -func doCompaction(ctx context.Context, patterns ...parquet.PartitionPattern) (*parquet.CompactionStatus, error) { +func doCompaction(ctx context.Context, db *database.DuckDb, patterns []parquet.PartitionPattern) (*parquet.CompactionStatus, error) { s := spinner.New( spinner.CharSets[14], 100*time.Millisecond, @@ -104,20 +122,35 @@ func doCompaction(ctx context.Context, patterns ...parquet.PartitionPattern) (*p s.Start() defer s.Stop() s.Suffix = " compacting parquet files" - // define func to update the spinner suffix with the number of files compacted var status = parquet.NewCompactionStatus() - updateTotals := func(counts parquet.CompactionStatus) { - status.Update(counts) - s.Suffix = fmt.Sprintf(" compacting parquet files (%d files -> %d files)", status.Source, status.Dest) + + updateTotals := func(updatedStatus parquet.CompactionStatus) { + status = &updatedStatus + s.Suffix = fmt.Sprintf(" compacting parquet files (%0.1f%% of %s rows)", status.ProgressPercent, types.ToHumanisedString(status.TotalRows)) } // do compaction - err := parquet.CompactDataFiles(ctx, updateTotals, patterns...) + err := parquet.CompactDataFiles(ctx, db, updateTotals, patterns...) return status, err } +// getPartitionPatterns returns the table and partition patterns for the given partition args +func getPartitionPatterns(partitionArgs []string, partitions []string) ([]parquet.PartitionPattern, error) { + var res []parquet.PartitionPattern + for _, arg := range partitionArgs { + tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) + if err != nil { + return nil, fmt.Errorf("error processing partition arg '%s': %w", arg, err) + } + + res = append(res, parquet.PartitionPattern{Table: tablePattern, Partition: partitionPattern}) + } + + return res, nil +} + func setExitCodeForCompactError(err error) { // set exit code only if an error occurred and no exit code is already set if exitCode != 0 || err == nil { diff --git a/cmd/connect.go b/cmd/connect.go index 7bfe3455..3b03995f 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -1,33 +1,23 @@ package cmd import ( - "context" "encoding/json" "fmt" - "io" - "log" "os" "path/filepath" "strings" - "time" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/thediveo/enumflag/v2" - "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/pipe-fittings/v2/cmdconfig" "github.com/turbot/pipe-fittings/v2/connection" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/error_helpers" - pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" - "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/parquet" - "golang.org/x/exp/maps" ) // variable used to assign the output mode flag @@ -38,22 +28,15 @@ func connectCmd() *cobra.Command { Use: "connect [flags]", Args: cobra.ArbitraryArgs, Run: runConnectCmd, - Short: "Return a connection string for a database, with a schema determined by the provided parameters", - Long: `Return a connection string for a database, with a schema determined by the provided parameters.`, + Short: "Return a connection string for the ducklake database", + Long: "Return a connection string for the ducklake database.", } - // args `from` and `to` accept: - // - ISO 8601 date (2024-01-01) - // - ISO 8601 datetime (2006-01-02T15:04:05) - // - ISO 8601 datetime with ms (2006-01-02T15:04:05.000) - // - RFC 3339 datetime with timezone (2006-01-02T15:04:05Z07:00) - // - relative time formats (T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) - cmdconfig.OnCmd(cmd). - AddStringFlag(pconstants.ArgFrom, "", "Specify the start time"). - AddStringFlag(pconstants.ArgTo, "", "Specify the end time"). - AddStringSliceFlag(pconstants.ArgIndex, nil, "Specify the index to use"). - AddStringSliceFlag(pconstants.ArgPartition, nil, "Specify the partition to use"). + AddStringFlag(pconstants.ArgFrom, "", "Specify the start time", cmdconfig.FlagOptions.Deprecated("'from' is not supported with ducklake")). + AddStringFlag(pconstants.ArgTo, "", "Specify the end time", cmdconfig.FlagOptions.Deprecated("'to'' is not supported with ducklake")). + AddStringSliceFlag(pconstants.ArgIndex, nil, "Specify the index to use", cmdconfig.FlagOptions.Deprecated("'index' is not supported with ducklake")). + AddStringSliceFlag(pconstants.ArgPartition, nil, "Specify the partition to use", cmdconfig.FlagOptions.Deprecated("'partition' is not supported with ducklake")). AddVarFlag(enumflag.New(&connectOutputMode, pconstants.ArgOutput, constants.ConnectOutputModeIds, enumflag.EnumCaseInsensitive), pconstants.ArgOutput, fmt.Sprintf("Output format; one of: %s", strings.Join(constants.FlagValues(constants.PluginOutputModeIds), ", "))) @@ -62,17 +45,9 @@ func connectCmd() *cobra.Command { } func runConnectCmd(cmd *cobra.Command, _ []string) { - var err error - var databaseFilePath string ctx := cmd.Context() - - defer func() { - if r := recover(); r != nil { - err = helpers.ToError(r) - } - setExitCodeForConnectError(err) - displayOutput(ctx, databaseFilePath, err) - }() + dataPath := config.GlobalWorkspaceProfile.GetDataDir() + dbFilePath := filepath.Join(dataPath, "metadata.sqlite") // if diagnostic mode is set, print out config and return if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { @@ -80,60 +55,18 @@ func runConnectCmd(cmd *cobra.Command, _ []string) { return } - databaseFilePath, err = generateDbFile(ctx) - - // we are done - the defer block will print either the filepath (if successful) or the error (if not) -} - -func generateDbFile(ctx context.Context) (string, error) { - databaseFilePath := generateTempDBFilename(config.GlobalWorkspaceProfile.GetDataDir()) - - // cleanup the old db files if not in use - err := cleanupOldDbFiles() - if err != nil { - return "", err - } - - // first build the filters - filters, err := getFilters() - if err != nil { - return "", fmt.Errorf("error building filters: %w", err) - } - - // if there are no filters, just copy the db file - if len(filters) == 0 { - err = copyDBFile(filepaths.TailpipeDbFilePath(), databaseFilePath) - return databaseFilePath, err - } - - // Open a DuckDB connection (creates the file if it doesn't exist) - db, err := database.NewDuckDb(database.WithDbFile(databaseFilePath)) - - if err != nil { - return "", fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - - err = database.AddTableViews(ctx, db, filters...) - return databaseFilePath, err -} - -func displayOutput(ctx context.Context, databaseFilePath string, err error) { switch viper.GetString(pconstants.ArgOutput) { case pconstants.OutputFormatText: - if err == nil { - // output the filepath - fmt.Println(databaseFilePath) //nolint:forbidigo // ui output - } else { - error_helpers.ShowError(ctx, err) - } + // output the filepath + connectionString := backend.GetDucklakeConnectionString(dbFilePath, dataPath) + fmt.Println(connectionString) //nolint:forbidigo // ui output + case pconstants.OutputFormatJSON: res := connection.TailpipeConnectResponse{ - DatabaseFilepath: databaseFilePath, - } - if err != nil { - res.Error = err.Error() + DatabaseFilepath: dbFilePath, + DataPath: dataPath, } + b, err := json.Marshal(res) if err == nil { fmt.Println(string(b)) //nolint:forbidigo // ui output @@ -146,276 +79,3 @@ func displayOutput(ctx context.Context, databaseFilePath string, err error) { error_helpers.ShowError(ctx, fmt.Errorf("unsupported output format %q", viper.GetString(pconstants.ArgOutput))) } } - -func getFilters() ([]string, error) { - var result []string - if viper.IsSet(pconstants.ArgFrom) { - from := viper.GetString(pconstants.ArgFrom) - // parse the string as time.Time - // arg `from` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), - // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) - t, err := parse.ParseTime(from, time.Now()) - if err != nil { - return nil, fmt.Errorf("invalid date format for 'from': %s", from) - } - // format as SQL timestamp - fromDate := t.Format(time.DateOnly) - fromTimestamp := t.Format(time.DateTime) - result = append(result, fmt.Sprintf("tp_date >= date '%s' and tp_timestamp >= timestamp '%s'", fromDate, fromTimestamp)) - } - if viper.IsSet(pconstants.ArgTo) { - to := viper.GetString(pconstants.ArgTo) - // parse the string as time.Time - // arg `to` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), - // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) - t, err := parse.ParseTime(to, time.Now()) - if err != nil { - return nil, fmt.Errorf("invalid date format for 'to': %s", to) - } - // format as SQL timestamp - toDate := t.Format(time.DateOnly) - toTimestamp := t.Format(time.DateTime) - result = append(result, fmt.Sprintf("tp_date <= date '%s' and tp_timestamp <= timestamp '%s'", toDate, toTimestamp)) - } - if viper.IsSet(pconstants.ArgPartition) { - // we have loaded tailpipe config by this time - availablePartitions := config.GlobalConfig.Partitions - partitionArgs := viper.GetStringSlice(pconstants.ArgPartition) - // get the SQL filters from the provided partition - sqlFilters, err := getPartitionSqlFilters(partitionArgs, maps.Keys(availablePartitions)) - if err != nil { - return nil, err - } - result = append(result, sqlFilters) - } - if viper.IsSet(pconstants.ArgIndex) { - indexArgs := viper.GetStringSlice(pconstants.ArgIndex) - // get the SQL filters from the provided index - sqlFilters, err := getIndexSqlFilters(indexArgs) - if err != nil { - return nil, err - } - result = append(result, sqlFilters) - } - return result, nil -} - -// generateTempDBFilename generates a temporary filename with a timestamp -func generateTempDBFilename(dataDir string) string { - timestamp := time.Now().Format("20060102150405") // e.g., 20241031103000 - return filepath.Join(dataDir, fmt.Sprintf("tailpipe_%s.db", timestamp)) -} - -func setExitCodeForConnectError(err error) { - // if exit code already set, leave as is - // NOTE: DO NOT set exit code if the output format is JSON - if exitCode != 0 || err == nil || viper.GetString(pconstants.ArgOutput) == pconstants.OutputFormatJSON { - return - } - - exitCode = 1 -} - -// copyDBFile copies the source database file to the destination -func copyDBFile(src, dst string) error { - sourceFile, err := os.Open(src) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(dst) - if err != nil { - return err - } - defer destFile.Close() - - _, err = io.Copy(destFile, sourceFile) - return err -} - -// cleanupOldDbFiles deletes old db files(older than a day) that are not in use -func cleanupOldDbFiles() error { - baseDir := pfilepaths.GetDataDir() - log.Printf("[INFO] Cleaning up old db files in %s\n", baseDir) - cutoffTime := time.Now().Add(-constants.DbFileMaxAge) // Files older than 1 day - - // The baseDir ("$TAILPIPE_INSTALL_DIR/data") is expected to have subdirectories for different workspace - // profiles(default, work etc). Each subdirectory may contain multiple .db files. - // Example structure: - // data/ - // ├── default/ - // │ ├── tailpipe_20250115182129.db - // │ ├── tailpipe_20250115193816.db - // │ ├── tailpipe.db - // │ └── ... - // ├── work/ - // │ ├── tailpipe_20250115182129.db - // │ ├── tailpipe_20250115193816.db - // │ ├── tailpipe.db - // │ └── ... - // So we traverse all these subdirectories for each workspace and process the relevant files. - err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error accessing path %s: %v", path, err) - } - - // skip directories and non-`.db` files - if info.IsDir() || !strings.HasSuffix(info.Name(), ".db") { - return nil - } - - // skip `tailpipe.db` file - if info.Name() == "tailpipe.db" { - return nil - } - - // only process `tailpipe_*.db` files - if !strings.HasPrefix(info.Name(), "tailpipe_") { - return nil - } - - // check if the file is older than the cutoff time - if info.ModTime().After(cutoffTime) { - log.Printf("[DEBUG] Skipping deleting file %s(%s) as it is not older than %s\n", path, info.ModTime().String(), cutoffTime) - return nil - } - - // check for a lock on the file - db, err := database.NewDuckDb(database.WithDbFile(path)) - if err != nil { - log.Printf("[INFO] Skipping deletion of file %s due to error: %v\n", path, err) - return nil - } - defer db.Close() - - // if no lock, delete the file - err = os.Remove(path) - if err != nil { - log.Printf("[INFO] Failed to delete db file %s: %v", path, err) - } else { - log.Printf("[DEBUG] Cleaned up old unused db file: %s\n", path) - } - - return nil - }) - - if err != nil { - return err - } - return nil -} - -func getPartitionSqlFilters(partitionArgs []string, availablePartitions []string) (string, error) { - // Get table and partition patterns using getPartitionPatterns - patterns, err := getPartitionPatterns(partitionArgs, availablePartitions) - if err != nil { - return "", fmt.Errorf("error processing partition args: %w", err) - } - - // Handle the case when patterns are empty - if len(patterns) == 0 { - return "", nil - } - - // Replace wildcards from '*' to '%' for SQL compatibility - sqlPatterns := replaceWildcards(patterns) - - var conditions []string - - for i := 0; i < len(sqlPatterns); i++ { - table := sqlPatterns[i].Table - partition := sqlPatterns[i].Partition - - var tableCondition, partitionCondition string - - // If there is no wildcard, use '=' instead of like - if table == "%" { - // Skip table condition if full wildcard - tableCondition = "" - } else if strings.Contains(table, "%") { - tableCondition = fmt.Sprintf("tp_table like '%s'", table) - } else { - tableCondition = fmt.Sprintf("tp_table = '%s'", table) - } - - if partition == "%" { - // Skip partition condition if full wildcard - partitionCondition = "" - } else if strings.Contains(partition, "%") { - partitionCondition = fmt.Sprintf("tp_partition like '%s'", partition) - } else { - partitionCondition = fmt.Sprintf("tp_partition = '%s'", partition) - } - - // Remove empty conditions and combine valid ones - if tableCondition != "" && partitionCondition != "" { - conditions = append(conditions, fmt.Sprintf("(%s and %s)", tableCondition, partitionCondition)) - } else if tableCondition != "" { - conditions = append(conditions, tableCondition) - } else if partitionCondition != "" { - conditions = append(conditions, partitionCondition) - } - } - - // Combine all conditions with OR - sqlFilters := strings.Join(conditions, " OR ") - - return sqlFilters, nil -} - -func getIndexSqlFilters(indexArgs []string) (string, error) { - // Return empty if no indexes provided - if len(indexArgs) == 0 { - return "", nil - } - - // Build SQL filter based on whether wildcards are present - var conditions []string - for _, index := range indexArgs { - if index == "*" { - // Skip index condition if full wildcard - conditions = append(conditions, "") - } else if strings.Contains(index, "*") { - // Replace '*' wildcard with '%' for SQL like compatibility - index = strings.ReplaceAll(index, "*", "%") - conditions = append(conditions, fmt.Sprintf("cast(tp_index as varchar) like '%s'", index)) - } else { - // Exact match using '=' - conditions = append(conditions, fmt.Sprintf("tp_index = '%s'", index)) - } - } - - // Combine all conditions with OR - sqlFilter := strings.Join(conditions, " OR ") - - return sqlFilter, nil -} - -// getPartitionPatterns returns the table and partition patterns for the given partition args -func getPartitionPatterns(partitionArgs []string, partitions []string) ([]parquet.PartitionPattern, error) { - var res []parquet.PartitionPattern - for _, arg := range partitionArgs { - tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) - if err != nil { - return nil, fmt.Errorf("error processing partition arg '%s': %w", arg, err) - } - - res = append(res, parquet.PartitionPattern{Table: tablePattern, Partition: partitionPattern}) - } - - return res, nil -} - -// convert partition patterns with '*' wildcards to SQL '%' wildcards -func replaceWildcards(patterns []parquet.PartitionPattern) []parquet.PartitionPattern { - updatedPatterns := make([]parquet.PartitionPattern, len(patterns)) - - for i, p := range patterns { - updatedPatterns[i] = parquet.PartitionPattern{ - Table: strings.ReplaceAll(p.Table, "*", "%"), - Partition: strings.ReplaceAll(p.Partition, "*", "%")} - } - return updatedPatterns - -} diff --git a/cmd/connect_test.go b/cmd/connect_test.go deleted file mode 100644 index 62fb47ec..00000000 --- a/cmd/connect_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package cmd - -import ( - "testing" -) - -func Test_getPartitionSqlFilters(t *testing.T) { - tests := []struct { - name string - partitions []string - args []string - wantFilters string - wantErr bool - }{ - { - name: "Basic partition filters with wildcard", - partitions: []string{ - "aws_cloudtrail_log.p1", - "aws_cloudtrail_log.p2", - "github_audit_log.p1", - }, - args: []string{"aws_cloudtrail_log.*", "github_audit_log.p1"}, - wantFilters: "tp_table = 'aws_cloudtrail_log' OR " + - "(tp_table = 'github_audit_log' and tp_partition = 'p1')", - wantErr: false, - }, - { - name: "Wildcard in table and exact partition", - partitions: []string{ - "aws_cloudtrail_log.p1", - "sys_logs.p2", - }, - args: []string{"aws*.p1", "sys_logs.*"}, - wantFilters: "(tp_table like 'aws%' and tp_partition = 'p1') OR " + - "tp_table = 'sys_logs'", - wantErr: false, - }, - { - name: "Exact table and partition", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"aws_cloudtrail_log.p1"}, - wantFilters: "(tp_table = 'aws_cloudtrail_log' and tp_partition = 'p1')", - wantErr: false, - }, - { - name: "Partition with full wildcard", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"aws_cloudtrail_log.*"}, - wantFilters: "tp_table = 'aws_cloudtrail_log'", - wantErr: false, - }, - { - name: "Table with full wildcard", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"*.p1"}, - wantFilters: "tp_partition = 'p1'", - wantErr: false, - }, - { - name: "Both table and partition with full wildcards", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"*.*"}, - wantFilters: "", - wantErr: false, - }, - { - name: "Empty input", - partitions: []string{"aws_cloudtrail_log.p1"}, - args: []string{}, - wantFilters: "", - wantErr: false, - }, - { - name: "Multiple wildcards in table and partition", - partitions: []string{ - "aws_cloudtrail_log.p1", - "sys_logs.p2", - }, - args: []string{"aws*log.p*"}, - wantFilters: "(tp_table like 'aws%log' and tp_partition like 'p%')", - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotFilters, err := getPartitionSqlFilters(tt.args, tt.partitions) - if (err != nil) != tt.wantErr { - t.Errorf("getPartitionSqlFilters() name = %s error = %v, wantErr %v", tt.name, err, tt.wantErr) - return - } - if gotFilters != tt.wantFilters { - t.Errorf("getPartitionSqlFilters() name = %s got = %v, want %v", tt.name, gotFilters, tt.wantFilters) - } - }) - } -} - -func Test_getIndexSqlFilters(t *testing.T) { - tests := []struct { - name string - indexArgs []string - wantFilters string - wantErr bool - }{ - { - name: "Multiple indexes with wildcards and exact values", - indexArgs: []string{"1234*", "456789012345", "98*76"}, - wantFilters: "cast(tp_index as varchar) like '1234%' OR " + - "tp_index = '456789012345' OR " + - "cast(tp_index as varchar) like '98%76'", - wantErr: false, - }, - { - name: "Single index with wildcard", - indexArgs: []string{"12345678*"}, - wantFilters: "cast(tp_index as varchar) like '12345678%'", - wantErr: false, - }, - { - name: "No input provided", - indexArgs: []string{}, - wantFilters: "", - wantErr: false, - }, - { - name: "Fully wildcarded index", - indexArgs: []string{"*"}, - wantFilters: "", - wantErr: false, - }, - { - name: "Exact numeric index", - indexArgs: []string{"123456789012"}, - wantFilters: "tp_index = '123456789012'", - wantErr: false, - }, - { - name: "Mixed patterns", - indexArgs: []string{"12*", "3456789", "9*76"}, - wantFilters: "cast(tp_index as varchar) like '12%' OR " + - "tp_index = '3456789' OR " + - "cast(tp_index as varchar) like '9%76'", - wantErr: false, - }, - { - name: "Multiple exact values", - indexArgs: []string{"123456789012", "987654321098"}, - wantFilters: "tp_index = '123456789012' OR tp_index = '987654321098'", - wantErr: false, - }, - { - name: "Leading and trailing spaces in exact value", - indexArgs: []string{" 123456789012 "}, - wantFilters: "tp_index = ' 123456789012 '", // Spaces preserved - wantErr: false, - }, - { - name: "Combination of wildcards and exact values", - indexArgs: []string{"*456*", "1234", "98*76"}, - wantFilters: "cast(tp_index as varchar) like '%456%' OR " + - "tp_index = '1234' OR " + - "cast(tp_index as varchar) like '98%76'", - wantErr: false, - }, - { - name: "Empty string as index", - indexArgs: []string{""}, - wantFilters: "tp_index = ''", - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotFilters, err := getIndexSqlFilters(tt.indexArgs) - if (err != nil) != tt.wantErr { - t.Errorf("getIndexSqlFilters() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotFilters != tt.wantFilters { - t.Errorf("getIndexSqlFilters() got = %v, want %v", gotFilters, tt.wantFilters) - } - }) - } -} diff --git a/cmd/partition.go b/cmd/partition.go index 44a3cb0f..3c3a4485 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -17,10 +17,12 @@ import ( "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" + "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" "github.com/turbot/tailpipe/internal/filepaths" "github.com/turbot/tailpipe/internal/parquet" @@ -92,8 +94,12 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { return } + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - resources, err := display.ListPartitionResources(ctx) + resources, err := display.ListPartitionResources(ctx, db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resources...) @@ -148,9 +154,24 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { return } + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - partitionName := args[0] - resource, err := display.GetPartitionResource(partitionName) + + partitions, err := getPartitions(args) + error_helpers.FailOnError(err) + // if no partitions are found, return an error + if len(partitions) == 0 { + error_helpers.FailOnError(fmt.Errorf("no partitions found matching %s", args[0])) + } + // if more than one partition is found, return an error + if len(partitions) > 1 { + error_helpers.FailOnError(fmt.Errorf("multiple partitions found matching %s, please specify a more specific partition name", args[0])) + } + + resource, err := display.GetPartitionResource(cmd.Context(), partitions[0], db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resource) @@ -184,6 +205,7 @@ func partitionDeleteCmd() *cobra.Command { cmdconfig.OnCmd(cmd). AddStringFlag(pconstants.ArgFrom, "", "Specify the start time"). + AddStringFlag(pconstants.ArgTo, "", "Specify the end time"). AddBoolFlag(pconstants.ArgForce, false, "Force delete without confirmation") return cmd @@ -204,18 +226,35 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { localcmdconfig.DisplayConfig() return } - - // arg `fromTime` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), - // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) + // args `fromTime` and `ToTime` accepts: + // - ISO 8601 date(2024-01-01) + // - ISO 8601 datetime(2006-01-02T15:04:05) + // - ISO 8601 datetime with ms(2006-01-02T15:04:05.000) + // - RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) + // - relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) var fromTime time.Time - var fromStr string + // toTime defaults to now, but can be set to a specific time + toTime := time.Now() + // confirm deletion + var fromStr, toStr string + if viper.IsSet(pconstants.ArgFrom) { var err error fromTime, err = parseFromToTime(viper.GetString(pconstants.ArgFrom)) - error_helpers.FailOnError(err) + error_helpers.FailOnErrorWithMessage(err, "invalid from time") fromStr = fmt.Sprintf(" from %s", fromTime.Format(time.DateOnly)) } + if viper.IsSet(pconstants.ArgTo) { + var err error + toTime, err = parseFromToTime(viper.GetString(pconstants.ArgTo)) + error_helpers.FailOnErrorWithMessage(err, "invalid to time") + } + toStr = fmt.Sprintf(" to %s", toTime.Format(time.DateOnly)) + if toTime.Before(fromTime) { + error_helpers.FailOnError(fmt.Errorf("to time %s cannot be before from time %s", toTime.Format(time.RFC3339), fromTime.Format(time.RFC3339))) + } + // retrieve the partition partitionName := args[0] partition, ok := config.GlobalConfig.Partitions[partitionName] if !ok { @@ -223,15 +262,22 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { } if !viper.GetBool(pconstants.ArgForce) { - // confirm deletion - msg := fmt.Sprintf("Are you sure you want to delete partition %s%s?", partitionName, fromStr) + msg := fmt.Sprintf("Are you sure you want to delete partition %s%s%s?", partitionName, fromStr, toStr) if !utils.UserConfirmationWithDefault(msg, true) { fmt.Println("Deletion cancelled") //nolint:forbidigo//expected output return } } - - filesDeleted, err := parquet.DeleteParquetFiles(partition, fromTime) + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + error_helpers.FailOnError(err) + defer db.Close() + + // show spinner while deleting the partition + spinner := statushooks.NewStatusSpinnerHook() + spinner.SetStatus(fmt.Sprintf("Deleting partition %s", partition.TableName)) + spinner.Show() + rowsDeleted, err := parquet.DeletePartition(ctx, partition, fromTime, toTime, db) + spinner.Hide() error_helpers.FailOnError(err) // build the collection state path @@ -255,17 +301,17 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { // now prune the collection folders err = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetCollectionDir()) if err != nil { - slog.Warn("DeleteParquetFiles failed to prune empty collection folders", "error", err) + slog.Warn("DeletePartition failed to prune empty collection folders", "error", err) } - msg := buildStatusMessage(filesDeleted, partitionName, fromStr) + msg := buildStatusMessage(rowsDeleted, partitionName, fromStr) fmt.Println(msg) //nolint:forbidigo//expected output } -func buildStatusMessage(filesDeleted int, partition string, fromStr string) interface{} { - var deletedStr = " (no parquet files deleted)" - if filesDeleted > 0 { - deletedStr = fmt.Sprintf(" (deleted %d parquet %s)", filesDeleted, utils.Pluralize("file", filesDeleted)) +func buildStatusMessage(rowsDeleted int, partition string, fromStr string) interface{} { + var deletedStr = " (nothing deleted)" + if rowsDeleted > 0 { + deletedStr = fmt.Sprintf(" (deleted %d %s)", rowsDeleted, utils.Pluralize("rows", rowsDeleted)) } return fmt.Sprintf("\nDeleted partition '%s'%s%s.\n", partition, fromStr, deletedStr) diff --git a/cmd/query.go b/cmd/query.go index 8bb0b3ea..21cf7b4c 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "fmt" "os" "strings" @@ -84,9 +83,8 @@ func runQueryCmd(cmd *cobra.Command, args []string) { return } - // get a connection to the database - var db *database.DuckDb - db, err = openDatabaseConnection(ctx) + // get a connection to the database, with DuckLake enabled + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) if err != nil { return } @@ -107,17 +105,6 @@ func runQueryCmd(cmd *cobra.Command, args []string) { // if there were any errors, they would have been shown already from `RunBatchSession` - just set the exit code exitCode = pconstants.ExitCodeQueryExecutionFailed } - -} - -// generate a db file - this will respect any time/index filters specified in the command args -func openDatabaseConnection(ctx context.Context) (*database.DuckDb, error) { - dbFilePath, err := generateDbFile(ctx) - if err != nil { - return nil, err - } - // Open a DuckDB connection - return database.NewDuckDb(database.WithDbFile(dbFilePath)) } func setExitCodeForQueryError(err error) { @@ -126,6 +113,6 @@ func setExitCodeForQueryError(err error) { return } - // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/106 + // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/496 exitCode = 1 } diff --git a/cmd/table.go b/cmd/table.go index 0ff868ff..265e323e 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -17,6 +17,7 @@ import ( "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" ) @@ -85,8 +86,12 @@ func runTableListCmd(cmd *cobra.Command, args []string) { return } + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - resources, err := display.ListTableResources(ctx) + resources, err := display.ListTableResources(ctx, db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resources...) @@ -141,8 +146,12 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { return } + db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - resource, err := display.GetTableResource(ctx, args[0]) + resource, err := display.GetTableResource(ctx, args[0], db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resource) diff --git a/go.mod b/go.mod index 753eba49..7fc2a9a4 100644 --- a/go.mod +++ b/go.mod @@ -6,24 +6,23 @@ toolchain go1.24.0 replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 -//github.com/turbot/pipe-fittings/v2 => ../pipe-fittings -//github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core -//github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk + github.com/turbot/pipe-fittings/v2 => ../pipe-fittings + //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core + github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/hashicorp/hcl/v2 v2.20.1 github.com/mattn/go-isatty v0.0.20 - github.com/spf13/cobra v1.8.1 + github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 github.com/turbot/pipe-fittings/v2 v2.6.0 github.com/turbot/tailpipe-plugin-sdk v0.9.2 github.com/zclconf/go-cty v1.14.4 - golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c - + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 ) require ( @@ -33,40 +32,45 @@ require ( github.com/charmbracelet/bubbletea v1.2.4 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/dustin/go-humanize v1.0.1 - github.com/fsnotify/fsnotify v1.8.0 + github.com/fsnotify/fsnotify v1.9.0 github.com/gosuri/uiprogress v0.0.1 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.6.1 github.com/hashicorp/go-version v1.7.0 github.com/jedib0t/go-pretty/v6 v6.5.9 - github.com/marcboeker/go-duckdb/v2 v2.1.0 + github.com/marcboeker/go-duckdb/v2 v2.3.5 github.com/thediveo/enumflag/v2 v2.0.5 github.com/turbot/tailpipe-plugin-core v0.2.10 - golang.org/x/sync v0.12.0 - golang.org/x/text v0.23.0 - google.golang.org/grpc v1.69.2 - google.golang.org/protobuf v1.36.1 + golang.org/x/sync v0.16.0 + golang.org/x/text v0.27.0 + google.golang.org/grpc v1.73.0 + google.golang.org/protobuf v1.36.6 ) require ( github.com/goccy/go-json v0.10.5 // indirect - github.com/google/flatbuffers v25.1.24+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect ) require ( - cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.7.2 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.1.10 // indirect - cloud.google.com/go/storage v1.42.0 // indirect + cel.dev/expr v0.23.0 // indirect + cloud.google.com/go v0.121.0 // indirect + cloud.google.com/go/auth v0.16.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/iam v1.5.0 // indirect + cloud.google.com/go/monitoring v1.24.0 // indirect + cloud.google.com/go/storage v1.52.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/apache/arrow-go/v18 v18.1.0 // indirect + github.com/apache/arrow-go/v18 v18.4.0 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go v1.44.183 // indirect @@ -92,6 +96,7 @@ require ( github.com/charmbracelet/lipgloss v1.0.0 // indirect github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect github.com/containerd/containerd v1.7.27 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -100,40 +105,43 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/duckdb/duckdb-go-bindings v0.1.13 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.8 // indirect + github.com/duckdb/duckdb-go-bindings v0.1.17 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 // indirect github.com/elastic/go-grok v0.3.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/fatih/color v1.17.0 // indirect + github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gertd/go-pluralize v0.2.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.0 // indirect github.com/go-git/go-git/v5 v5.13.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect - github.com/go-viper/mapstructure/v2 v2.3.0 // indirect - github.com/goccy/go-yaml v1.11.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-yaml v1.17.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/gosuri/uilive v0.0.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.7.5 // indirect + github.com/hashicorp/go-getter v1.7.9 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -161,8 +169,8 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/marcboeker/go-duckdb/arrowmapping v0.0.6 // indirect - github.com/marcboeker/go-duckdb/mapping v0.0.6 // indirect + github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 // indirect + github.com/marcboeker/go-duckdb/mapping v0.0.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -184,6 +192,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.1.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.5.0 // indirect @@ -197,7 +206,8 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stevenle/topsort v0.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -209,27 +219,31 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zclconf/go-cty-yaml v1.0.3 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.29.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/oauth2 v0.29.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.35.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/api v0.189.0 // indirect - google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect + google.golang.org/api v0.230.0 // indirect + google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 50eee3dc..97175990 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,11 @@ +cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss= +cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,6 +18,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -26,32 +30,96 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg= +cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= -cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= -cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= -cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= -cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/auth v0.16.0 h1:Pd8P1s9WkcrBE2n/PhAwKsdrR35V3Sg2II9B+ndM3CU= +cloud.google.com/go/auth v0.16.0/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -59,12 +127,44 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -72,127 +172,464 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v1.1.10 h1:ZSAr64oEhQSClwBL670MsJAW5/RLiC6kfw3Bqmd5ZDI= -cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.5.0 h1:QlLcVMhbLGOjRcGe6VTGGTyQib8dRLK2B/kYNV0+2xs= +cloud.google.com/go/iam v1.5.0/go.mod h1:U+DOtKQltF/LxPEtcDLoobcsZMilSRwR7mgNL7knOpo= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= -cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.6.6 h1:XJNDo5MUfMM05xK3ewpbSdmt7R2Zw+aQEMbdQR65Rbw= +cloud.google.com/go/longrunning v0.6.6/go.mod h1:hyeGJUrPHcx0u2Uu1UFSoYZLn4lkMrccJig0t4FI7yw= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM= +cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.42.0 h1:4QtGpplCVt1wz6g5o1ifXd656P5z+yNgzdw1tVfp0cU= -cloud.google.com/go/storage v1.42.0/go.mod h1:HjMXRFq65pGKFn6hxj6x3HCyR41uSB72Z0SO/Vn6JFQ= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.52.0 h1:ROpzMW/IwipKtatA69ikxibdzQSiXJrY9f6IgBa9AlA= +cloud.google.com/go/storage v1.52.0/go.mod h1:4wrBAbAYUvYkbrf19ahGm4I5kDQhESSqN3CGEkMGvOY= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.11.3 h1:c+I4YFjxRQjvAhRmSsmjpASUKq88chOX854ied0K/pE= +cloud.google.com/go/trace v1.11.3/go.mod h1:pt7zCYiDSQjC9Y2oqCsh9jF4GStB/hmjrYLsxRR27q8= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -204,15 +641,23 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= -github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= -github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= -github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/apache/arrow-go/v18 v18.4.0 h1:/RvkGqH517iY8bZKc4FD5/kkdwXJGjxf28JIXbJ/oB0= +github.com/apache/arrow-go/v18 v18.4.0/go.mod h1:Aawvwhj8x2jURIzD9Moy72cF0FyJXOpkYpdmGRHcw14= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-dump v0.0.0-20190214190832-042adf3cf4a0 h1:MzVXffFUye+ZcSR6opIgz9Co7WcDx6ZcY+RjfFHoA0I= @@ -256,6 +701,8 @@ github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To= @@ -263,8 +710,11 @@ github.com/btubbs/datetime v0.1.1/go.mod h1:n2BZ/2ltnRzNiz27aE3wUb2onNttQdC+WFxA github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv3KnQRNCE= @@ -284,11 +734,17 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= @@ -301,7 +757,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= @@ -316,18 +773,20 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/duckdb/duckdb-go-bindings v0.1.13 h1:3Ec0SjMBuzt7wExde5ZoMXd1Nk91LJmpopq2Ee6g9Pw= -github.com/duckdb/duckdb-go-bindings v0.1.13/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.8 h1:n4RNMqiUPao53YKmlh36zGEr49CnUXGVKOtOMCEhwFE= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.8/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.8 h1:3ZBS6wETlZp9UDmaWJ4O4k7ZSjqQjyhMW5aZZBXThqM= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.8/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.8 h1:KCUI9KSAUKbYasNlTcjky30nbDtF18S6s6R3usXWLqk= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.8/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.8 h1:QgKzpNG7EMPq3ayYcr0LzGfC+dCzGA/Gm6Y7ndbrXHg= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.8/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.8 h1:lmseSULUmuVycRBJ6DVH86eFOQhHz32hN8mfxF7z+0w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.8/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/duckdb/duckdb-go-bindings v0.1.17 h1:SjpRwrJ7v0vqnIvLeVFHlhuS72+Lp8xxQ5jIER2LZP4= +github.com/duckdb/duckdb-go-bindings v0.1.17/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 h1:8CLBnsq9YDhi2Gmt3sjSUeXxMzyMQAKefjqUy9zVPFk= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 h1:wjO4I0GhMh2xIpiUgRpzuyOT4KxXLoUS/rjU7UUVvCE= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 h1:HzKQi2C+1jzmwANsPuYH6x9Sfw62SQTjNAEq3OySKFI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 h1:YGSR7AFLw2gJ7IbgLE6DkKYmgKv1LaRSd/ZKF1yh2oE= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 h1:2aduW6fnFnT2Q45PlIgHbatsPOxV9WSZ5B2HzFfxaxA= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= @@ -343,24 +802,44 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA= github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8= @@ -372,6 +851,10 @@ github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkv github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -379,6 +862,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -391,15 +876,19 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= -github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= -github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= +github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= +github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -431,15 +920,18 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= -github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -455,13 +947,15 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -473,6 +967,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -481,8 +976,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -490,8 +985,10 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -501,9 +998,12 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= -github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= @@ -511,13 +1011,15 @@ github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tf github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.5 h1:dT58k9hQ/vbxNMwoI5+xFYAJuv6152UNvdHokfI5wE4= -github.com/hashicorp/go-getter v1.7.5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.9 h1:G9gcjrDixz7glqJ+ll5IWvggSBR+R0B54DSRt4qfdC4= +github.com/hashicorp/go-getter v1.7.9/go.mod h1:dyFCmT1AQkDfOIt9NH8pw9XBDqNrIKJT5ylbpi7zPNE= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -545,6 +1047,7 @@ github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -584,17 +1087,25 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/gows v0.3.0 h1:/FGSuBiJMUqNOJPsAdLvHFg7RnkFoWBS8USpdco5ONQ= github.com/karrick/gows v0.3.0/go.mod h1:kdZ/jfdo8yqKYn+BMjBkhP+/oRKUABR1abaomzRi/n8= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -607,16 +1118,19 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.6 h1:FaNX2JP4pKw7Xh2rMBCCvqWIafhX3nSXrUffexNRB68= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.6/go.mod h1:WjLM334CLZux/OtAeF0DT2n9LyNqquqT3EhCHQcflNk= -github.com/marcboeker/go-duckdb/mapping v0.0.6 h1:Y+nHQDHXqo78i8MM4UP7qVmFgTAofbdvpUdRdxJXjSk= -github.com/marcboeker/go-duckdb/mapping v0.0.6/go.mod h1:k1lwBZvSza+RSpuA1kcMS/vxlNuqqFynoDef/clDD2M= -github.com/marcboeker/go-duckdb/v2 v2.1.0 h1:mhAEwy+Ut9Iji+QvyjkB86HhhC/r/H0RRKpkwfANu88= -github.com/marcboeker/go-duckdb/v2 v2.1.0/go.mod h1:W76KqN7EWTm8kpU2irA0V4f1R+6QEt3uLUVZ3wAtZ7M= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 h1:G1W+GVnUefR8uy7jHdNO+CRMsmFG5mFPIHVAespfFCA= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.10/go.mod h1:jccUb8TYD0p5TsEEeN4SXuslNJHo23QaKOqKD+U6uFU= +github.com/marcboeker/go-duckdb/mapping v0.0.11 h1:fusN1b1l7Myxafifp596I6dNLNhN5Uv/rw31qAqBwqw= +github.com/marcboeker/go-duckdb/mapping v0.0.11/go.mod h1:aYBjFLgfKO0aJIbDtXPiaL5/avRQISveX/j9tMf9JhU= +github.com/marcboeker/go-duckdb/v2 v2.3.5 h1:dpLZdPppUPdwd37/kDEE025iVgQoRw2Q4qXFtXroNIo= +github.com/marcboeker/go-duckdb/v2 v2.3.5/go.mod h1:8adNrftF4Ye29XMrpIl5NYNosTVsZu1mz3C82WdVvrk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -624,7 +1138,6 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -639,6 +1152,7 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= @@ -681,28 +1195,45 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -724,22 +1255,28 @@ github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stevenle/topsort v0.2.0 h1:LLWgtp34HPX6/RBDRS0kElVxGOTzGBLI1lSAa5Lb46k= github.com/stevenle/topsort v0.2.0/go.mod h1:ck2WG2/ZrOr6dLApQ/5Xrqy5wv3T0qhKYWE7r9tkibc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -749,6 +1286,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -769,14 +1307,10 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.6.0 h1:RhCHble2MB7W0l9lE5QQLRQKuMD8xlM7AKuhoFOwqy4= -github.com/turbot/pipe-fittings/v2 v2.6.0/go.mod h1:wcKckD5UUCVWSQkdW6J27cumy5GkACV/wn7FJOajYHE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= github.com/turbot/tailpipe-plugin-core v0.2.10/go.mod h1:dHzPUR1p5GksSvDqqEeZEvvJX6wTEwK/ZDev//9nSLw= -github.com/turbot/tailpipe-plugin-sdk v0.9.2 h1:bsivlduG4BSYlyjYIKGCiFHiYsrLhtuZbimjv1TnUOQ= -github.com/turbot/tailpipe-plugin-sdk v0.9.2/go.mod h1:Egojp0j7+th/4Bh6muMuF6aZa5iE3MuiJ4pzBo0J2mg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= @@ -788,6 +1322,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -799,6 +1334,8 @@ github.com/zclconf/go-cty-yaml v1.0.3 h1:og/eOQ7lvA/WWhHGFETVWNduJM7Rjsv2RRpx1sd github.com/zclconf/go-cty-yaml v1.0.3/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -810,21 +1347,29 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -832,25 +1377,48 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -874,9 +1442,17 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -907,11 +1483,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -922,10 +1501,23 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -949,10 +1541,14 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -966,9 +1562,15 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1009,11 +1611,14 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1022,9 +1627,12 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1043,17 +1651,39 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1063,16 +1693,31 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1085,6 +1730,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1113,18 +1759,26 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1135,8 +1789,16 @@ golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNq golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= -gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1184,9 +1846,18 @@ google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaE google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= -google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.230.0 h1:2u1hni3E+UXAXrONrrkfWpi/V6cyKVAbfGVeGtC3OxM= +google.golang.org/api v0.230.0/go.mod h1:aqvtoMk7YkiXx+6U12arQFExiRV9D/ekvMCwCd/TksQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1229,7 +1900,9 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1262,6 +1935,7 @@ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= @@ -1294,13 +1968,41 @@ google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53B google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= +google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e h1:UdXH7Kzbj+Vzastr5nVfccbmFsmYNygVLSPk1pEfDoY= +google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e/go.mod h1:085qFyf2+XaZlRdCgKNCIZ3afY2p4HHZdoIRpId8F4A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1:29cjnHVylHwTzH66WfFZqgSQgnxzvWE+jvBwpZCLRxY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1327,6 +2029,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1336,8 +2039,13 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= -google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1354,8 +2062,11 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -1381,9 +2092,45 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/internal/cmdconfig/cmd_hooks.go b/internal/cmdconfig/cmd_hooks.go index 2d7ecbce..503435b2 100644 --- a/internal/cmdconfig/cmd_hooks.go +++ b/internal/cmdconfig/cmd_hooks.go @@ -21,7 +21,6 @@ import ( "github.com/turbot/pipe-fittings/v2/workspace_profile" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/logger" "github.com/turbot/tailpipe/internal/parse" "github.com/turbot/tailpipe/internal/plugin" @@ -47,7 +46,7 @@ func preRunHook(cmd *cobra.Command, args []string) error { ew := initGlobalConfig(ctx) // display any warnings ew.ShowWarnings() - // TODO #errors sort exit code https://github.com/turbot/tailpipe/issues/106 + // TODO #errors sort exit code https://github.com/turbot/tailpipe/issues/496 // check for error error_helpers.FailOnError(ew.Error) @@ -155,12 +154,6 @@ func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { return error_helpers.NewErrorsAndWarning(err) } - // ensure we have a database file for this workspace - err = database.EnsureDatabaseFile(ctx) - if err != nil { - return error_helpers.NewErrorsAndWarning(err) - } - var cmd = viper.Get(pconstants.ConfigKeyActiveCommand).(*cobra.Command) // set-up viper with defaults from the env and default workspace profile @@ -185,27 +178,11 @@ func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { // load the connection config and HCL options (passing plugin versions tailpipeConfig, loadConfigErrorsAndWarnings := parse.LoadTailpipeConfig(pluginVersionFile) - if loadConfigErrorsAndWarnings.Error != nil { - return loadConfigErrorsAndWarnings - } + if loadConfigErrorsAndWarnings.Error == nil { + // store global config + config.GlobalConfig = tailpipeConfig - if loadConfigErrorsAndWarnings.Warnings != nil { - for _, warning := range loadConfigErrorsAndWarnings.Warnings { - error_helpers.ShowWarning(warning) - } } - // store global config - config.GlobalConfig = tailpipeConfig - - // now validate all config values have appropriate values - return validateConfig() -} - -// now validate config values have appropriate values -func validateConfig() error_helpers.ErrorAndWarnings { - var res = error_helpers.ErrorAndWarnings{} - - // TODO #config validate - return res + return loadConfigErrorsAndWarnings } diff --git a/internal/collector/collector.go b/internal/collector/collector.go index c3630f44..90120b43 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -13,6 +13,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/spf13/viper" pconstants "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/tailpipe-plugin-sdk/events" sdkfilepaths "github.com/turbot/tailpipe-plugin-sdk/filepaths" "github.com/turbot/tailpipe-plugin-sdk/row_source" @@ -51,6 +52,9 @@ type Collector struct { // the path to the JSONL files - the plugin will write to this path sourcePath string + // database connection + db *database.DuckDb + // bubble tea app app *tea.Program cancel context.CancelFunc @@ -84,6 +88,18 @@ func New(pluginManager *plugin.PluginManager, partition *config.Partition, cance } c.sourcePath = sourcePath + // create the DuckDB connection + // load json and inet extension in addition to the DuckLake extension - the convertor will need them + db, err := database.NewDuckDb( + database.WithDuckDbExtensions(pconstants.DuckDbExtensions), + database.WithDuckLakeEnabled(true), + ) + + if err != nil { + return nil, fmt.Errorf("failed to create DuckDB connection: %w", err) + } + c.db = db + return c, nil } @@ -95,20 +111,16 @@ func New(pluginManager *plugin.PluginManager, partition *config.Partition, cance func (c *Collector) Close() { close(c.Events) - if c.parquetConvertor != nil { - c.parquetConvertor.Close() - } - - // close the tea app - if c.app != nil { - c.app.Quit() - } - // if inbox path is empty, remove it (ignore errors) _ = os.Remove(c.sourcePath) // delete the collection temp dir _ = os.RemoveAll(c.collectionTempDir) + + // close the tea app + if c.app != nil { + c.app.Quit() + } } // Collect asynchronously starts the collection process @@ -118,7 +130,7 @@ func (c *Collector) Close() { // - starts the collection UI // - creates a parquet writer, which will process the JSONL files as they are written // - starts listening to plugin events -func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, recollect bool) (err error) { +func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, overwrite bool) (err error) { if c.execution != nil { return errors.New("collection already in progress") } @@ -132,22 +144,20 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, rec } }() - // create the execution - // NOTE: create _before_ calling the plugin to ensure it is ready to receive the started event - c.execution = newExecution(c.partition) - - // tell plugin to start collecting - collectResponse, err := c.pluginManager.Collect(ctx, c.partition, fromTime, toTime, recollect, c.collectionTempDir) - if err != nil { - return err + var collectResponse *plugin.CollectResponse + // is this is a synthetic partition? + if c.partition.SyntheticMetadata != nil { + if collectResponse, err = c.doCollectSynthetic(ctx, fromTime, toTime, overwrite); err != nil { + return err + } + } else { + if collectResponse, err = c.doCollect(ctx, fromTime, toTime, overwrite); err != nil { + return err + } } - // _now_ set the execution id - c.execution.id = collectResponse.ExecutionId - // validate the schema returned by the plugin - err = collectResponse.Schema.Validate() - if err != nil { + if err = collectResponse.Schema.Validate(); err != nil { err := fmt.Errorf("table '%s' returned invalid schema: %w", c.partition.TableName, err) // set execution to error c.execution.done(err) @@ -157,6 +167,22 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, rec // determine the time to start collecting from resolvedFromTime := collectResponse.FromTime + // if we are overwriting, we need to delete any existing data in the partition + if overwrite { + // show spinner while deleting the partition + spinner := statushooks.NewStatusSpinnerHook() + spinner.SetStatus(fmt.Sprintf("Deleting partition %s", c.partition.TableName)) + spinner.Show() + err := c.deletePartitionData(ctx, resolvedFromTime.Time, toTime) + spinner.Hide() + if err != nil { + // set execution to error + c.execution.done(err) + // and return error + return fmt.Errorf("failed to delete partition data: %w", err) + } + } + // display the progress UI err = c.showCollectionStatus(resolvedFromTime, toTime) if err != nil { @@ -167,7 +193,7 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, rec c.addTimeRangeFilters(resolvedFromTime, toTime) // create a parquet writer - parquetConvertor, err := parquet.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount) + parquetConvertor, err := parquet.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount, c.db) if err != nil { return fmt.Errorf("failed to create parquet writer: %w", err) } @@ -179,6 +205,22 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, rec return nil } +func (c *Collector) doCollect(ctx context.Context, fromTime time.Time, toTime time.Time, overwrite bool) (*plugin.CollectResponse, error) { + // create the execution + // NOTE: create _before_ calling the plugin to ensure it is ready to receive the started event + c.execution = newExecution(c.partition) + + // tell plugin to start collecting + collectResponse, err := c.pluginManager.Collect(ctx, c.partition, fromTime, toTime, overwrite, c.collectionTempDir) + if err != nil { + return nil, err + } + + // _now_ set the execution id + c.execution.id = collectResponse.ExecutionId + return collectResponse, nil +} + // addTimeRangeFilters adds filters to the partition based on the from and to time func (c *Collector) addTimeRangeFilters(resolvedFromTime *row_source.ResolvedFromTime, toTime time.Time) { // if there is a from time, add a filter to the partition - this will be used by the parquet writer @@ -219,14 +261,16 @@ func (c *Collector) Compact(ctx context.Context) error { c.updateApp(AwaitingCompactionMsg{}) - updateAppCompactionFunc := func(compactionStatus parquet.CompactionStatus) { + updateAppCompactionFunc := func(status parquet.CompactionStatus) { c.statusLock.Lock() defer c.statusLock.Unlock() - c.status.UpdateCompactionStatus(&compactionStatus) + c.status.compactionStatus = &status c.updateApp(CollectionStatusUpdateMsg{status: c.status}) } partitionPattern := parquet.NewPartitionPattern(c.partition) - err := parquet.CompactDataFiles(ctx, updateAppCompactionFunc, partitionPattern) + + err := parquet.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, partitionPattern) + if err != nil { return fmt.Errorf("failed to compact data files: %w", err) } @@ -246,85 +290,16 @@ func (c *Collector) Completed() { } } -// handlePluginEvent handles an event from a plugin -func (c *Collector) handlePluginEvent(ctx context.Context, e events.Event) { - // handlePluginEvent the event - // switch based on the struct of the event - switch ev := e.(type) { - case *events.Started: - slog.Info("Started event", "execution", ev.ExecutionId) - c.execution.state = ExecutionState_STARTED - case *events.Status: - c.statusLock.Lock() - defer c.statusLock.Unlock() - c.status.UpdateWithPluginStatus(ev) - c.updateApp(CollectionStatusUpdateMsg{status: c.status}) - case *events.Chunk: - - executionId := ev.ExecutionId - chunkNumber := ev.ChunkNumber - - // log every 100 chunks - if ev.ChunkNumber%100 == 0 { - slog.Debug("Chunk event", "execution", ev.ExecutionId, "chunk", ev.ChunkNumber) - } - - err := c.parquetConvertor.AddChunk(executionId, chunkNumber) - if err != nil { - slog.Error("failed to add chunk to parquet writer", "error", err) - c.execution.done(err) - } - case *events.Complete: - slog.Info("Complete event", "execution", ev.ExecutionId) - - // was there an error? - if ev.Err != nil { - slog.Error("execution error", "execution", ev.ExecutionId, "error", ev.Err) - // update the execution - c.execution.done(ev.Err) - return - } - // this event means all JSON files have been written - we need to wait for all to be converted to parquet - // we then combine the parquet files into a single file - - // start thread waiting for conversion to complete - // - this will wait for all parquet files to be written, and will then combine these into a single parquet file - slog.Info("handlePluginEvent - waiting for conversions to complete") - go func() { - err := c.waitForConversions(ctx, ev) - if err != nil { - slog.Error("error waiting for execution to complete", "error", err) - c.execution.done(err) - } else { - slog.Info("handlePluginEvent - conversions all complete") - } - }() - - case *events.Error: - // TODO #errors error events are deprecated an will only be sent for plugins not using sdk > v0.2.0 - // TODO #errors decide what (if anything) we should do with error events from old plugins https://github.com/turbot/tailpipe/issues/297 - //ev := e.GetErrorEvent() - //// for now just store errors and display at end - ////c.execution.state = ExecutionState_ERROR - ////c.execution.error = fmt.Errorf("plugin error: %s", ev.Error) - //slog.Warn("plugin error", "execution", ev.ExecutionId, "error", ev.Error) - } -} - -func (c *Collector) createTableView(ctx context.Context) error { - // so we are done writing chunks - now update the db to add a view to this data - // Open a DuckDB connection - db, err := database.NewDuckDb(database.WithDbFile(filepaths.TailpipeDbFilePath())) +// deletePartitionData deletes all parquet files in the partition between the fromTime and toTime +func (c *Collector) deletePartitionData(ctx context.Context, fromTime, toTime time.Time) error { + slog.Info("Deleting parquet files after the from time", "partition", c.partition.Name, "from", fromTime) + _, err := parquet.DeletePartition(ctx, c.partition, fromTime, toTime, c.db) if err != nil { - return err - } - defer db.Close() + slog.Warn("Failed to delete parquet files after the from time", "partition", c.partition.Name, "from", fromTime, "error", err) - err = database.AddTableView(ctx, c.execution.table, db) - if err != nil { - return err } - return nil + slog.Info("Completed deleting parquet files after the from time", "partition", c.partition.Name, "from", fromTime) + return err } func (c *Collector) showCollectionStatus(resolvedFromTime *row_source.ResolvedFromTime, toTime time.Time) error { @@ -409,17 +384,7 @@ func (c *Collector) waitForConversions(ctx context.Context, ce *events.Complete) } // wait for the conversions to complete - c.parquetConvertor.WaitForConversions(ctx) - - // create or update the table view for ths table being collected - if err := c.createTableView(ctx); err != nil { - slog.Error("error creating table view", "error", err) - return err - } - - slog.Info("handlePluginEvent - conversions all complete") - - return nil + return c.parquetConvertor.WaitForConversions(ctx) } // listenToEvents listens to the events channel and handles events @@ -428,9 +393,65 @@ func (c *Collector) listenToEvents(ctx context.Context) { select { case <-ctx.Done(): return - case event := <-c.Events: - c.handlePluginEvent(ctx, event) + case e := <-c.Events: + c.handlePluginEvent(ctx, e) + } + } +} + +// handlePluginEvent handles an event from a plugin +func (c *Collector) handlePluginEvent(ctx context.Context, e events.Event) { + // handlePluginEvent the event + // switch based on the struct of the event + switch ev := e.(type) { + case *events.Started: + slog.Info("Started event", "execution", ev.ExecutionId) + c.execution.state = ExecutionState_STARTED + case *events.Status: + c.statusLock.Lock() + defer c.statusLock.Unlock() + c.status.UpdateWithPluginStatus(ev) + c.updateApp(CollectionStatusUpdateMsg{status: c.status}) + case *events.Chunk: + + executionId := ev.ExecutionId + chunkNumber := ev.ChunkNumber + + // log every 100 chunks + if ev.ChunkNumber%100 == 0 { + slog.Debug("Chunk event", "execution", ev.ExecutionId, "chunk", ev.ChunkNumber) + } + + err := c.parquetConvertor.AddChunk(executionId, chunkNumber) + if err != nil { + slog.Error("failed to add chunk to parquet writer", "error", err) + c.execution.done(err) + } + case *events.Complete: + slog.Info("Complete event", "execution", ev.ExecutionId) + + // was there an error? + if ev.Err != nil { + slog.Error("execution error", "execution", ev.ExecutionId, "error", ev.Err) + // update the execution + c.execution.done(ev.Err) + return } + // this event means all JSON files have been written - we need to wait for all to be converted to parquet + // we then combine the parquet files into a single file + + // start thread waiting for conversion to complete + // - this will wait for all parquet files to be written, and will then combine these into a single parquet file + slog.Info("handlePluginEvent - waiting for conversions to complete") + go func() { + err := c.waitForConversions(ctx, ev) + if err != nil { + slog.Error("error waiting for execution to complete", "error", err) + c.execution.done(err) + } else { + slog.Info("all conversions complete") + } + }() } } diff --git a/internal/collector/collector_synthetic.go b/internal/collector/collector_synthetic.go new file mode 100644 index 00000000..b6435deb --- /dev/null +++ b/internal/collector/collector_synthetic.go @@ -0,0 +1,584 @@ +package collector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "math" + "os" + "path/filepath" + "strings" + "time" + + "bufio" + "runtime" + "sync" + + "github.com/turbot/tailpipe-plugin-sdk/events" + "github.com/turbot/tailpipe-plugin-sdk/row_source" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" + "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/plugin" +) + +func (c *Collector) doCollectSynthetic(ctx context.Context, fromTime time.Time, toTime time.Time, overwrite bool) (*plugin.CollectResponse, error) { + // create the execution + // NOTE: create _before_ calling the plugin to ensure it is ready to receive the started event + c.execution = &execution{ + id: "synthetic", + partition: c.partition.UnqualifiedName, + table: c.partition.TableName, + plugin: "synthetic", + state: ExecutionState_PENDING, + completionChan: make(chan error, 1), + } + + schema := buildsyntheticchema(c.partition.SyntheticMetadata.Columns) + // start a thread to fake the collection process + go c.collectSynthetic(ctx, schema, fromTime, toTime) + + // build a collect response + collectResponse := &plugin.CollectResponse{ + ExecutionId: c.execution.id, + Schema: schema, + FromTime: &row_source.ResolvedFromTime{ + Time: fromTime, + Source: "synthetic", + }, + } + // _now_ set the execution id + c.execution.id = collectResponse.ExecutionId + return collectResponse, nil +} + +// syntheticColumnTypes defines the available column types for synthetic data generation +var syntheticColumnTypes = []struct { + Name string + SQLType string + StructFields []*schema.ColumnSchema +}{ + {"string_col", "VARCHAR", nil}, + {"int_col", "INTEGER", nil}, + {"float_col", "DOUBLE", nil}, + {"bool_col", "BOOLEAN", nil}, + {"json_col", "JSON", nil}, + {"timestamp_col", "TIMESTAMP", nil}, + {"array_col", "JSON", nil}, + {"nested_json_col", "JSON", nil}, + {"uuid_col", "VARCHAR", nil}, + {"simple_struct_col", "STRUCT", []*schema.ColumnSchema{ + { + SourceName: "id", + ColumnName: "id", + Type: "INTEGER", + Description: "Simple struct ID field", + }, + { + SourceName: "name", + ColumnName: "name", + Type: "VARCHAR", + Description: "Simple struct name field", + }, + { + SourceName: "active", + ColumnName: "active", + Type: "BOOLEAN", + Description: "Simple struct active field", + }, + }}, + {"nested_struct_col", "STRUCT", []*schema.ColumnSchema{ + { + SourceName: "metadata", + ColumnName: "metadata", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "created_at", + ColumnName: "created_at", + Type: "VARCHAR", + Description: "Creation timestamp", + }, + { + SourceName: "version", + ColumnName: "version", + Type: "VARCHAR", + Description: "Version string", + }, + }, + Description: "Metadata information", + }, + { + SourceName: "data", + ColumnName: "data", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "field1", + ColumnName: "field1", + Type: "INTEGER", + Description: "Numeric field 1", + }, + { + SourceName: "field2", + ColumnName: "field2", + Type: "VARCHAR", + Description: "String field 2", + }, + { + SourceName: "field3", + ColumnName: "field3", + Type: "BOOLEAN", + Description: "Boolean field 3", + }, + }, + Description: "Data fields", + }, + }}, + {"complex_struct_col", "STRUCT", []*schema.ColumnSchema{ + { + SourceName: "user", + ColumnName: "user", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "id", + ColumnName: "id", + Type: "INTEGER", + Description: "User ID", + }, + { + SourceName: "name", + ColumnName: "name", + Type: "VARCHAR", + Description: "User name", + }, + { + SourceName: "profile", + ColumnName: "profile", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "age", + ColumnName: "age", + Type: "INTEGER", + Description: "User age", + }, + { + SourceName: "email", + ColumnName: "email", + Type: "VARCHAR", + Description: "User email", + }, + { + SourceName: "verified", + ColumnName: "verified", + Type: "BOOLEAN", + Description: "Email verified", + }, + }, + Description: "User profile information", + }, + }, + Description: "User information", + }, + { + SourceName: "settings", + ColumnName: "settings", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "theme", + ColumnName: "theme", + Type: "VARCHAR", + Description: "UI theme", + }, + { + SourceName: "notifications", + ColumnName: "notifications", + Type: "BOOLEAN", + Description: "Notifications enabled", + }, + }, + Description: "User settings", + }, + }}, +} + +// ConcurrentDataGenerator handles concurrent data generation and marshaling +type ConcurrentDataGenerator struct { + numWorkers int + rowChan chan []byte + errorChan chan error + doneChan chan bool +} + +// NewConcurrentDataGenerator creates a new concurrent data generator +func NewConcurrentDataGenerator(numWorkers int) *ConcurrentDataGenerator { + return &ConcurrentDataGenerator{ + numWorkers: numWorkers, + rowChan: make(chan []byte, numWorkers*100), // Buffer for generated rows + errorChan: make(chan error, 1), + doneChan: make(chan bool, 1), + } +} + +// generateRowData generates a single row's JSON data +func generateRowData(rowIndex int, partition *config.Partition, tableSchema *schema.TableSchema, fromTime time.Time, timestampInterval time.Duration) ([]byte, error) { + // Create row map + rowMap := make(map[string]any, len(tableSchema.Columns)) + timestamp := fromTime.Add(time.Duration(rowIndex) * timestampInterval).Format("2006-01-02 15:04:05") + + // Populate row map (skip tp_index and tp_date) + for _, column := range tableSchema.Columns { + if column.ColumnName == "tp_index" || column.ColumnName == "tp_date" { + continue + } + + switch column.ColumnName { + case "tp_timestamp": + rowMap[column.ColumnName] = timestamp + case "tp_partition": + rowMap[column.ColumnName] = partition.ShortName + case "tp_table": + rowMap[column.ColumnName] = partition.TableName + default: + // Generate synthetic data for other columns + rowMap[column.ColumnName] = generateSyntheticValue(column, rowIndex) + } + } + + // Marshal to JSON + data, err := json.Marshal(rowMap) + if err != nil { + return nil, fmt.Errorf("failed to marshal row %d: %w", rowIndex, err) + } + + // Add newline + data = append(data, '\n') + return data, nil +} + +// worker generates data for a range of rows +func (cdg *ConcurrentDataGenerator) worker(startRow, endRow int, partition *config.Partition, tableSchema *schema.TableSchema, fromTime time.Time, timestampInterval time.Duration) { + for rowIndex := startRow; rowIndex < endRow; rowIndex++ { + data, err := generateRowData(rowIndex, partition, tableSchema, fromTime, timestampInterval) + if err != nil { + select { + case cdg.errorChan <- err: + default: + } + return + } + + select { + case cdg.rowChan <- data: + case <-cdg.doneChan: + return + } + } +} + +// writeOptimizedChunkToJSONLConcurrent uses multiple goroutines for data generation +func writeOptimizedChunkToJSONLConcurrent(filepath string, tableSchema *schema.TableSchema, rows int, startRowIndex int, partition *config.Partition, fromTime time.Time, timestampInterval time.Duration) error { + file, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filepath, err) + } + defer file.Close() + + // Use buffered writer for better I/O performance + bufWriter := bufio.NewWriter(file) + defer bufWriter.Flush() + + // Determine number of workers (use CPU cores, but cap at reasonable number) + numWorkers := runtime.NumCPU() + if numWorkers > 8 { + numWorkers = 8 // Cap at 8 to avoid too much overhead + } + if numWorkers > rows { + numWorkers = rows // Don't create more workers than rows + } + + // Create concurrent data generator + cdg := NewConcurrentDataGenerator(numWorkers) + + // Calculate rows per worker + rowsPerWorker := rows / numWorkers + remainder := rows % numWorkers + + // Start workers + var wg sync.WaitGroup + startRow := startRowIndex + for i := 0; i < numWorkers; i++ { + endRow := startRow + rowsPerWorker + if i < remainder { + endRow++ // Distribute remainder rows + } + + wg.Add(1) + go func(start, end int) { + defer wg.Done() + cdg.worker(start, end, partition, tableSchema, fromTime, timestampInterval) + }(startRow, endRow) + + startRow = endRow + } + + // Start a goroutine to close the row channel when all workers are done + go func() { + wg.Wait() + close(cdg.rowChan) + }() + + // Write rows from channel to file + rowsWritten := 0 + for data := range cdg.rowChan { + if _, err := bufWriter.Write(data); err != nil { + close(cdg.doneChan) // Signal workers to stop + return fmt.Errorf("failed to write row %d: %w", rowsWritten, err) + } + rowsWritten++ + } + + // Check for errors + select { + case err := <-cdg.errorChan: + return fmt.Errorf("worker error: %w", err) + default: + } + + if rowsWritten != rows { + return fmt.Errorf("expected %d rows, but wrote %d", rows, rowsWritten) + } + + return nil +} + +func buildsyntheticchema(columns int) *schema.TableSchema { + // Create a basic schema with the required number of columns + // Start with required tp_ fields + s := &schema.TableSchema{ + Columns: make([]*schema.ColumnSchema, 0, columns+5), // +5 for tp_ fields (including tp_index and tp_date) + } + + // Add required tp_ fields first + tpFields := []struct { + name string + columnType string + description string + }{ + {"tp_timestamp", "TIMESTAMP", "Timestamp when the record was collected"}, + {"tp_partition", "VARCHAR", "Partition identifier"}, + {"tp_table", "VARCHAR", "Table identifier"}, + {"tp_index", "VARCHAR", "Index identifier"}, + {"tp_date", "VARCHAR", "Date identifier"}, + } + + for _, tpField := range tpFields { + column := &schema.ColumnSchema{ + SourceName: tpField.name, + ColumnName: tpField.name, + Type: tpField.columnType, + StructFields: nil, + Description: tpField.description, + Required: true, // tp_ fields are always required + NullIf: "", + Transform: "", + } + s.Columns = append(s.Columns, column) + } + + // Add the specified number of synthetic columns by cycling through the column types + for i := 0; i < columns; i++ { + // Cycle through the column types + typeIndex := i % len(syntheticColumnTypes) + baseType := syntheticColumnTypes[typeIndex] + + // Create a unique column name + columnName := fmt.Sprintf("%s_%d", baseType.Name, i) + + column := &schema.ColumnSchema{ + SourceName: columnName, + ColumnName: columnName, + Type: baseType.SQLType, + StructFields: baseType.StructFields, + Description: fmt.Sprintf("Synthetic column of type %s", baseType.SQLType), + Required: false, + NullIf: "", + Transform: "", + } + + s.Columns = append(s.Columns, column) + } + + return s +} + +func (c *Collector) collectSynthetic(ctx context.Context, tableSchema *schema.TableSchema, fromTime time.Time, toTime time.Time) { + metadata := c.partition.SyntheticMetadata + + // set the execution state to started + c.execution.state = ExecutionState_STARTED + + if err := c.Notify(ctx, &events.Started{ExecutionId: c.execution.id}); err != nil { + slog.Error("failed to notify started event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify started event: %w", err) + return + } + + var chunkIdx int32 = 0 + var totalRowsProcessed int64 = 0 + + // Calculate timestamp interval based on fromTime, toTime, and total rows + var timestampInterval time.Duration + if metadata.Rows > 1 { + timestampInterval = toTime.Sub(fromTime) / time.Duration(metadata.Rows-1) + } else { + timestampInterval = 0 + } + + for rowCount := 0; rowCount < metadata.Rows; rowCount += metadata.ChunkSize { + t := time.Now() + // Check if context is cancelled + select { + case <-ctx.Done(): + c.execution.completionChan <- ctx.Err() + return + default: + } + + rows := int(math.Min(float64(metadata.Rows-rowCount), float64(metadata.ChunkSize))) + + // write optimized chunk to JSONL file + filename := table.ExecutionIdToJsonlFileName(c.execution.id, chunkIdx) + filepath := filepath.Join(c.sourcePath, filename) + + // write the chunk to JSONL file using optimized approach + if err := writeOptimizedChunkToJSONLConcurrent(filepath, tableSchema, rows, rowCount, c.partition, fromTime, timestampInterval); err != nil { + c.execution.completionChan <- fmt.Errorf("error writing chunk to JSONL file: %w", err) + return + } + + dur := time.Since(t) + // if this is less that deliver interval, wait for the remaining time + if metadata.DeliveryIntervalMs > 0 && dur < time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond { + slog.Debug("Waiting for delivery interval", "duration", dur, "expected", time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond) + select { + case <-time.After(time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond - dur): + case <-ctx.Done(): + c.execution.completionChan <- ctx.Err() + return + } + } + // send chunk event to the plugin + chunkEvent := &events.Chunk{ExecutionId: c.execution.id, ChunkNumber: chunkIdx} + if err := c.Notify(ctx, chunkEvent); err != nil { + slog.Error("failed to notify chunk event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify chunk event: %w", err) + return + } + + totalRowsProcessed += int64(rows) + statusEvent := &events.Status{ExecutionId: c.execution.id, RowsReceived: totalRowsProcessed, RowsEnriched: totalRowsProcessed} + if err := c.Notify(ctx, statusEvent); err != nil { + slog.Error("failed to notify status event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify status event: %w", err) + return + } + + chunkIdx++ + } + + // Send completion event + if err := c.Notify(ctx, events.NewCompletedEvent(c.execution.id, int64(metadata.Rows), chunkIdx, nil)); err != nil { + slog.Error("failed to notify completed event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify completed event: %w", err) + return + } + + // Signal completion + c.execution.completionChan <- nil +} + +func generateSyntheticValue(column *schema.ColumnSchema, rowIndex int) any { + // Use the column's Type field directly instead of fuzzy matching on name + columnType := column.Type + + // Generate value based on exact type match (case-insensitive) + switch strings.ToUpper(columnType) { + case "VARCHAR": + return fmt.Sprintf("%s_val%d", column.ColumnName, rowIndex%100000) + case "INTEGER": + return (rowIndex % 100000) + 1 + case "DOUBLE": + return float64(rowIndex%100000) * 0.1 + case "BOOLEAN": + return rowIndex%2 == 0 + case "JSON": + return generateJSONValue(column, rowIndex) + case "TIMESTAMP": + return time.Now().AddDate(0, 0, -rowIndex%30).Format("2006-01-02 15:04:05") + default: + // Handle struct types and complex types + if strings.Contains(strings.ToUpper(columnType), "STRUCT") { + return generateStructValue(column, rowIndex) + } + // For any other unrecognized type, throw an error + panic(fmt.Sprintf("Unsupported column type '%s' for column '%s'", columnType, column.ColumnName)) + } +} + +func generateJSONValue(column *schema.ColumnSchema, rowIndex int) any { + // Generate different JSON structures based on column name + if strings.Contains(column.ColumnName, "nested_json") { + return map[string]any{ + "metadata": map[string]any{ + "created_at": time.Now().AddDate(0, 0, -rowIndex%30).Format("2006-01-02"), + "version": fmt.Sprintf("v%d.%d", rowIndex%10, rowIndex%5), + }, + "data": map[string]any{ + "field1": rowIndex % 100000, + "field2": fmt.Sprintf("field_%d", rowIndex%100000), + "field3": rowIndex%2 == 0, + }, + } + } else if strings.Contains(column.ColumnName, "array") { + return []any{ + fmt.Sprintf("item_%d", rowIndex%100000), + rowIndex % 100000, + rowIndex%2 == 0, + float64(rowIndex%100000) * 0.1, + } + } else { + // Default JSON object + return map[string]any{ + "id": rowIndex % 100000, + "name": fmt.Sprintf("item_%d", rowIndex%100000), + "value": (rowIndex % 100000) + 1, + "tags": []string{"tag1", "tag2", "tag3"}, + } + } +} + +func generateStructValue(column *schema.ColumnSchema, rowIndex int) any { + if column.StructFields == nil { + return map[string]any{ + "id": rowIndex % 100000, + "name": fmt.Sprintf("struct_%d", rowIndex%100000), + } + } + + result := make(map[string]any) + for _, field := range column.StructFields { + if field.StructFields != nil { + // Nested struct + result[field.ColumnName] = generateStructValue(field, rowIndex) + } else { + // Simple field + result[field.ColumnName] = generateSyntheticValue(field, rowIndex) + } + } + return result +} diff --git a/internal/collector/status.go b/internal/collector/status.go index 9f672220..25a0f33e 100644 --- a/internal/collector/status.go +++ b/internal/collector/status.go @@ -56,19 +56,6 @@ func (s *status) UpdateConversionStatus(rowsConverted, failedRows int64, errors } } -// UpdateCompactionStatus updates the status with the values from the compaction status event -func (s *status) UpdateCompactionStatus(compactionStatus *parquet.CompactionStatus) { - if compactionStatus == nil { - return - } - - if s.compactionStatus == nil { - s.compactionStatus = parquet.NewCompactionStatus() - } - - s.compactionStatus.Update(*compactionStatus) -} - // CollectionHeader returns a string to display at the top of the collection status for app or alone for non-progress display func (s *status) CollectionHeader() string { // wrap the source in parentheses if it exists @@ -220,14 +207,11 @@ func (s *status) displayFilesSection() string { var out strings.Builder out.WriteString("Files:\n") - if s.compactionStatus.Source == 0 && s.compactionStatus.Uncompacted == 0 { + if s.compactionStatus.InitialFiles == 0 { // no counts available, display status text out.WriteString(fmt.Sprintf(" %s\n", statusText)) } else { - // display counts source => dest - l := int64(s.compactionStatus.Source + s.compactionStatus.Uncompacted) - r := int64(s.compactionStatus.Dest + s.compactionStatus.Uncompacted) - out.WriteString(fmt.Sprintf(" Compacted: %s => %s\n", humanize.Comma(l), humanize.Comma(r))) + out.WriteString(fmt.Sprintf(" %s\n", s.compactionStatus.String())) } out.WriteString("\n") @@ -287,14 +271,22 @@ func (s *status) displayErrorsSection() string { // displayTimingSection returns a string representation of the timing section of the status (time elapsed since start of collection) func (s *status) displayTimingSection() string { duration := time.Since(s.started) - timeLabel := "Time:" // if we're complete, change the time label to show this if s.complete { - timeLabel = "Completed:" + if s.compactionStatus != nil && s.compactionStatus.Duration > 0 { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("Collection: %s\n", utils.HumanizeDuration(duration))) + sb.WriteString(fmt.Sprintf("Compaction: %s\n", utils.HumanizeDuration(s.compactionStatus.Duration))) + sb.WriteString(fmt.Sprintf("Total: %s\n", utils.HumanizeDuration(duration+s.compactionStatus.Duration))) + return sb.String() + } + return fmt.Sprintf("Completed: %s\n", utils.HumanizeDuration(duration)) + } else { + // if not complete, show elapsed time + return fmt.Sprintf("Time: %s\n", utils.HumanizeDuration(duration)) } - return fmt.Sprintf("%s %s\n", timeLabel, utils.HumanizeDuration(duration)) } // writeCountLine returns a formatted string for a count line in the status display, used for alignment and readability diff --git a/internal/config/connection.go b/internal/config/connection.go index e82416fa..698dd3cf 100644 --- a/internal/config/connection.go +++ b/internal/config/connection.go @@ -37,12 +37,10 @@ func (c *TailpipeConnection) GetSubType() string { func (c *TailpipeConnection) ToProto() *proto.ConfigData { return &proto.ConfigData{ - //Target: c.Name(), - // TODO fix connection parsing to populate name + // Target is of form `connection.` Target: "connection." + c.Plugin, - - Hcl: c.Hcl, - Range: proto.RangeToProto(c.DeclRange), + Hcl: c.Hcl, + Range: proto.RangeToProto(c.DeclRange), } } diff --git a/internal/config/partition.go b/internal/config/partition.go index 27a7c24c..e9a0c655 100644 --- a/internal/config/partition.go +++ b/internal/config/partition.go @@ -21,6 +21,13 @@ func init() { registerResourceWithSubType(schema.BlockTypePartition) } +type SyntheticMetadata struct { + Columns int + Rows int + ChunkSize int + DeliveryIntervalMs int +} + type Partition struct { modconfig.HclResourceImpl // required to allow partial decoding @@ -46,6 +53,9 @@ type Partition struct { Filter string `cty:"filter"` // the sql column to use for the tp_index TpIndexColumn string `cty:"tp_index_column"` + + // if this is a synthetic partition for testing, this will be non-null + SyntheticMetadata *SyntheticMetadata } func NewPartition(block *hcl.Block, fullName string) (modconfig.HclResource, hcl.Diagnostics) { diff --git a/internal/constants/database.go b/internal/constants/database.go deleted file mode 100644 index f7667e5f..00000000 --- a/internal/constants/database.go +++ /dev/null @@ -1,8 +0,0 @@ -package constants - -import "time" - -const ( - TailpipeDbName = "tailpipe.db" - DbFileMaxAge = 24 * time.Hour -) diff --git a/internal/constants/duckdb_extensions.go b/internal/constants/duckdb_extensions.go deleted file mode 100644 index e7d02979..00000000 --- a/internal/constants/duckdb_extensions.go +++ /dev/null @@ -1,3 +0,0 @@ -package constants - -var DuckDbExtensions = []string{"json", "inet"} diff --git a/internal/constants/metaquery_commands.go b/internal/constants/metaquery_commands.go index 8f8ee7b5..978e51ad 100644 --- a/internal/constants/metaquery_commands.go +++ b/internal/constants/metaquery_commands.go @@ -3,9 +3,7 @@ package constants // Metaquery commands const ( - //CmdTableList = ".tables" // List all tables - CmdOutput = ".output" // Set output mode - //CmdTiming = ".timing" // Toggle query timer + CmdOutput = ".output" // Set output mode CmdHeaders = ".header" // Toggle headers output CmdSeparator = ".separator" // Set the column separator CmdExit = ".exit" // Exit the interactive prompt @@ -15,5 +13,4 @@ const ( CmdClear = ".clear" // clear the console CmdHelp = ".help" // list all meta commands CmdAutoComplete = ".autocomplete" // enable or disable auto complete - TpPrefix = "tp_" // tailpipe prefix for tailpipe specific columns ) diff --git a/internal/database/create.go b/internal/database/create.go deleted file mode 100644 index 9c237152..00000000 --- a/internal/database/create.go +++ /dev/null @@ -1,27 +0,0 @@ -package database - -import ( - "context" - _ "github.com/marcboeker/go-duckdb/v2" - filehelpers "github.com/turbot/go-kit/files" - _ "github.com/turbot/go-kit/helpers" - _ "github.com/turbot/pipe-fittings/v2/utils" - "github.com/turbot/tailpipe/internal/filepaths" -) - -func EnsureDatabaseFile(ctx context.Context) error { - databaseFilePath := filepaths.TailpipeDbFilePath() - if filehelpers.FileExists(databaseFilePath) { - return nil - } - - // - // Open a DuckDB connection (creates the file if it doesn't exist) - db, err := NewDuckDb(WithDbFile(databaseFilePath)) - if err != nil { - return err - } - defer db.Close() - - return AddTableViews(ctx, db) -} diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index adf2eb42..e0460ce6 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -4,10 +4,13 @@ import ( "context" "database/sql" "fmt" + "log/slog" "os" + "github.com/turbot/pipe-fittings/v2/backend" + pconstants "github.com/turbot/pipe-fittings/v2/constants" pf "github.com/turbot/pipe-fittings/v2/filepaths" - "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/filepaths" ) @@ -18,17 +21,30 @@ import ( type DuckDb struct { // duckDb connection *sql.DB - extensions []string - dataSourceName string - tempDir string - maxMemoryMb int + extensions []string + dataSourceName string + tempDir string + maxMemoryMb int + ducklakeEnabled bool } -func NewDuckDb(opts ...DuckDbOpt) (*DuckDb, error) { +func NewDuckDb(opts ...DuckDbOpt) (_ *DuckDb, err error) { + slog.Info("Initializing DuckDB connection") + w := &DuckDb{} for _, opt := range opts { opt(w) } + defer func() { + if err != nil { + // If an error occurs during initialization, close the DB connection if it was opened + if w.DB != nil { + _ = w.DB.Close() + } + w.DB = nil // ensure DB is nil to avoid further operations on a closed connection + } + }() + // Connect to DuckDB db, err := sql.Open("duckdb", w.dataSourceName) if err != nil { @@ -36,12 +52,34 @@ func NewDuckDb(opts ...DuckDbOpt) (*DuckDb, error) { } w.DB = db + // for duckdb, limit connections to 1 - DuckDB is designed for single-connection usage + w.DB.SetMaxOpenConns(1) + if len(w.extensions) > 0 { // install and load the JSON extension if err := w.installAndLoadExtensions(); err != nil { return nil, fmt.Errorf(": %w", err) } } + if w.ducklakeEnabled { + dataDir := config.GlobalWorkspaceProfile.GetDataDir() + // TODO #DL tactical - for now check env for data dir override + // remove this for prod release https://github.com/turbot/tailpipe/issues/520 + if envDir := os.Getenv("TAILPIPE_DATA_DIR"); envDir != "" { + dataDir = envDir + } + + ducklakeDb := config.GlobalWorkspaceProfile.GetDucklakeDbPath() + + if err := backend.ConnectDucklake(context.Background(), db, ducklakeDb, dataDir); err != nil { + return nil, fmt.Errorf("failed to connect to DuckLake: %w", err) + } + + // Set the default catalog to tailpipe_ducklake to avoid catalog context issues + if _, err := db.Exec(`use "tailpipe_ducklake"`); err != nil { + return nil, fmt.Errorf("failed to set default catalog: %w", err) + } + } // Configure DuckDB's temp directory: // - If WithTempDir option was provided, use that directory @@ -70,6 +108,7 @@ func NewDuckDb(opts ...DuckDbOpt) (*DuckDb, error) { return nil, fmt.Errorf("failed to set max_memory: %w", err) } } + return w, nil } @@ -133,7 +172,7 @@ func (d *DuckDb) installAndLoadExtensions() error { } // install and load the extensions - for _, extension := range constants.DuckDbExtensions { + for _, extension := range pconstants.DuckDbExtensions { if _, err := d.DB.Exec(fmt.Sprintf("INSTALL '%s'; LOAD '%s';", extension, extension)); err != nil { return fmt.Errorf("failed to install and load extension %s: %s", extension, err.Error()) } diff --git a/internal/database/duck_db_error.go b/internal/database/duck_db_error.go index d03e0a80..693c48a5 100644 --- a/internal/database/duck_db_error.go +++ b/internal/database/duck_db_error.go @@ -7,6 +7,7 @@ import ( "os" "regexp" "sort" + "strconv" "strings" "time" @@ -81,7 +82,6 @@ func handleDuckDbError(err error) error { return newInvalidParquetError(updatedFilename) } // so we have no filename - //TODO handle Invalid Error: TProtocolException: Invalid data } return err @@ -162,21 +162,34 @@ func newInvalidParquetError(parquetFilePath string) error { parquetFilePath: parquetFilePath, } + var year, month int + // Extract table, partition and date from path components parts := strings.Split(parquetFilePath, "/") for _, part := range parts { - if strings.HasPrefix(part, "tp_table=") { + switch { + case strings.HasPrefix(part, "tp_table="): err.table = strings.TrimPrefix(part, "tp_table=") - } else if strings.HasPrefix(part, "tp_partition=") { + case strings.HasPrefix(part, "tp_partition="): err.partition = strings.TrimPrefix(part, "tp_partition=") - } else if strings.HasPrefix(part, "tp_date=") { - dateString := strings.TrimPrefix(part, "tp_date=") - date, parseErr := time.Parse("2006-01-02", dateString) + case strings.HasPrefix(part, "year="): + yearString := strings.TrimPrefix(part, "year=") + y, parseErr := strconv.Atoi(yearString) + if parseErr == nil { + year = y + } + case strings.HasPrefix(part, "month="): + monthString := strings.TrimPrefix(part, "month=") + m, parseErr := strconv.Atoi(monthString) if parseErr == nil { - err.date = date + month = m } } } + // if we have a year and month, set the error date + if year > 0 && month > 0 { + err.date = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) + } return err } diff --git a/internal/database/duck_db_options.go b/internal/database/duck_db_options.go index 1f39c46c..ad5d3f1a 100644 --- a/internal/database/duck_db_options.go +++ b/internal/database/duck_db_options.go @@ -38,3 +38,10 @@ func WithMaxMemoryMb(maxMemoryMb int) DuckDbOpt { d.maxMemoryMb = maxMemoryMb } } + +// WithDuckLakeEnabled enables the DuckLake extension for DuckDB. +func WithDuckLakeEnabled(enabled bool) DuckDbOpt { + return func(d *DuckDb) { + d.ducklakeEnabled = enabled + } +} diff --git a/internal/database/duck_db_test.go b/internal/database/duck_db_test.go index 9d7cdbc6..36c15bb2 100644 --- a/internal/database/duck_db_test.go +++ b/internal/database/duck_db_test.go @@ -110,8 +110,8 @@ func Test_executeWithParquetErrorRetry(t *testing.T) { // Helper function to create a test file with proper path structure mkTestFile := func(attempt int) string { - // Create a path that matches the expected format: tp_table=aws_cloudtrail/tp_partition=cloudtrail/tp_date=2024-03-20/test.parquet.N - path := filepath.Join(tmpDir, "tp_table=aws_cloudtrail", "tp_partition=cloudtrail", "tp_date=2024-03-20") + // Create a path that matches the expected format: tp_table=aws_cloudtrail/tp_partition=cloudtrail/year=2024/month=03/test.parquet + path := filepath.Join(tmpDir, "tp_table=aws_cloudtrail", "tp_partition=cloudtrail", "year=2024", "month=03") if err := os.MkdirAll(path, 0755); err != nil { t.Fatalf("failed to create test directory: %v", err) } @@ -206,6 +206,9 @@ func Test_executeWithParquetErrorRetry(t *testing.T) { } func TestDuckDb_WrapperMethods(t *testing.T) { + // TODO fix me + t.Skip("Skipping this test due to CI issues") + // Create a temporary directory for testing tmpDir := t.TempDir() @@ -217,7 +220,9 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test Query t.Run("Query", func(t *testing.T) { - rows, err := db.Query("select 1") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + rows, err := db.QueryContext(ctx, "select 1") if err != nil { t.Errorf("Query failed: %v", err) } @@ -228,7 +233,8 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test QueryContext t.Run("QueryContext", func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() rows, err := db.QueryContext(ctx, "select 1") if err != nil { t.Errorf("QueryContext failed: %v", err) @@ -240,7 +246,9 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test QueryRow t.Run("QueryRow", func(t *testing.T) { - row := db.QueryRow("select 1") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + row := db.QueryRowContext(ctx, "select 1") if row == nil { t.Error("QueryRow returned nil") } @@ -248,7 +256,8 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test QueryRowContext t.Run("QueryRowContext", func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() row := db.QueryRowContext(ctx, "select 1") if row == nil { t.Error("QueryRowContext returned nil") @@ -257,7 +266,9 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test Exec t.Run("Exec", func(t *testing.T) { - result, err := db.Exec("select 1") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := db.ExecContext(ctx, "select 1") if err != nil { t.Errorf("Exec failed: %v", err) } @@ -268,7 +279,8 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test ExecContext t.Run("ExecContext", func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() result, err := db.ExecContext(ctx, "select 1") if err != nil { t.Errorf("ExecContext failed: %v", err) diff --git a/internal/database/partitions.go b/internal/database/partitions.go deleted file mode 100644 index 9d6f25c4..00000000 --- a/internal/database/partitions.go +++ /dev/null @@ -1,52 +0,0 @@ -package database - -import ( - "context" - "fmt" - - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/filepaths" -) - -// ListPartitions uses DuckDB to build a list of all partitions for all tables -func ListPartitions(ctx context.Context) ([]string, error) { - // Hive format is table, partition, index, date - - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - if dataDir == "" { - return nil, fmt.Errorf("data directory is not set") - } - // TODO KAI handle no partitions - - // Build DuckDB query to get the names of all partitions underneath data dir - parquetPath := filepaths.GetParquetFileGlobForTable(dataDir, "*", "") - query := `select distinct tp_table || '.' || tp_partition from read_parquet('` + parquetPath + `', hive_partitioning=true)` - - // Open DuckDB in-memory database - db, err := NewDuckDb() - if err != nil { - return nil, fmt.Errorf("failed to open DuckDB: %v", err) - } - defer db.Close() - - rows, err := db.QueryContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to execute query: %v", err) - } - defer rows.Close() - - var partitions []string - for rows.Next() { - var partition string - if err := rows.Scan(&partition); err != nil { - return nil, fmt.Errorf("failed to scan row: %v", err) - } - partitions = append(partitions, partition) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating rows: %v", err) - } - - return partitions, nil -} diff --git a/internal/database/tables.go b/internal/database/tables.go index e52826a7..a3aafb03 100644 --- a/internal/database/tables.go +++ b/internal/database/tables.go @@ -3,194 +3,17 @@ package database import ( "context" "fmt" - "log/slog" - "os" - "regexp" "strings" - "github.com/turbot/pipe-fittings/v2/error_helpers" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/helpers" + "github.com/turbot/pipe-fittings/v2/constants" ) -// AddTableViews creates a view for each table in the data directory, applying the provided duck db filters to the view query -func AddTableViews(ctx context.Context, db *DuckDb, filters ...string) error { - tables, err := getDirNames(config.GlobalWorkspaceProfile.GetDataDir()) - if err != nil { - return fmt.Errorf("failed to get tables: %w", err) - } - - // optimisation - it seems the first time DuckDB creates a view which inspects the file system it is slow - // creating and empty view first and then dropping it seems to speed up the process - createAndDropEmptyView(ctx, db) - - //create a view for each table - for _, tableFolder := range tables { - // create a view for the table - // the tab;le folder is a hive partition folder so will have the format tp_table=table_name - table := strings.TrimPrefix(tableFolder, "tp_table=") - err = AddTableView(ctx, table, db, filters...) - if err != nil { - return err - } - } - return nil -} - -// NOTE: tactical optimisation - it seems the first time DuckDB creates a view which inspects the file system it is slow -// creating and empty view first and then dropping it seems to speed up the process -func createAndDropEmptyView(ctx context.Context, db *DuckDb) { - _ = AddTableView(ctx, "empty", db) - // drop again - _, _ = db.ExecContext(ctx, "DROP VIEW empty") -} - -func AddTableView(ctx context.Context, tableName string, db *DuckDb, filters ...string) error { - slog.Info("creating view", "table", tableName, "filters", filters) - - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - // Path to the Parquet directory - // hive structure is /tp_table=/tp_partition=/tp_index=/tp_date=.parquet - parquetPath := filepaths.GetParquetFileGlobForTable(dataDir, tableName, "") - - // Step 1: Query the first Parquet file to infer columns - columns, err := getColumnNames(ctx, parquetPath, db) - if err != nil { - // if this is because no parquet files match, suppress the error - if strings.Contains(err.Error(), "IO Error: No files found that match the pattern") || error_helpers.IsCancelledError(err) { - return nil - } - return err - } - - // Step 2: Build the select clause - cast tp_index as string - // (this is necessary as duckdb infers the type from the partition column name - // if the index looks like a number, it will infer the column as an int) - var typeOverrides = map[string]string{ - "tp_partition": "varchar", - "tp_index": "varchar", - "tp_date": "date", - } - var selectClauses []string - for _, col := range columns { - wrappedCol := fmt.Sprintf(`"%s"`, col) - if overrideType, ok := typeOverrides[col]; ok { - // Apply the override with casting - selectClauses = append(selectClauses, fmt.Sprintf("cast(%s as %s) as %s", col, overrideType, wrappedCol)) - } else { - // Add the column as-is - selectClauses = append(selectClauses, wrappedCol) - } - } - selectClause := strings.Join(selectClauses, ", ") - - // Step 3: Build the where clause - filterString := "" - if len(filters) > 0 { - filterString = fmt.Sprintf(" where %s", strings.Join(filters, " and ")) - } - - // Step 4: Construct the final query - query := fmt.Sprintf( - "create or replace view %s as select %s from '%s'%s", - tableName, selectClause, parquetPath, filterString, - ) - - // Execute the query - _, err = db.ExecContext(ctx, query) - if err != nil { - slog.Warn("failed to create view", "table", tableName, "error", err) - return fmt.Errorf("failed to create view: %w", err) - } - slog.Info("created view", "table", tableName) - return nil -} - -// query the provided parquet path to get the columns -func getColumnNames(ctx context.Context, parquetPath string, db *DuckDb) ([]string, error) { - columnQuery := fmt.Sprintf("select * from '%s' limit 0", parquetPath) - rows, err := db.QueryContext(ctx, columnQuery) - if err != nil { - return nil, err - } - defer rows.Close() - - // Retrieve column names - columns, err := rows.Columns() - if err != nil { - return nil, err - } +func GetTables(ctx context.Context, db *DuckDb) ([]string, error) { - // Sort column names alphabetically but with tp_ fields on the end - columns = helpers.SortColumnsAlphabetically(columns) - - return columns, nil -} - -func getDirNames(folderPath string) ([]string, error) { - var dirNames []string - - // Read the directory contents - files, err := os.ReadDir(folderPath) - if err != nil { - return nil, err - } - - // Loop through the contents and add directories to dirNames - for _, file := range files { - if file.IsDir() { - dirNames = append(dirNames, file.Name()) - } - } - - return dirNames, nil -} - -func GetRowCount(ctx context.Context, tableName string, partitionName *string) (int64, error) { - // Open a DuckDB connection - db, err := NewDuckDb(WithDbFile(filepaths.TailpipeDbFilePath())) - if err != nil { - return 0, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - - var tableNameRegex = regexp.MustCompile(`^[a-zA-Z0-9_]+$`) - if !tableNameRegex.MatchString(tableName) { - return 0, fmt.Errorf("invalid table name") - } - query := fmt.Sprintf("select count(*) from %s", tableName) // #nosec G201 // this is a controlled query tableName must match a regex - if partitionName != nil { - query = fmt.Sprintf("select count(*) from %s where tp_partition = '%s'", tableName, *partitionName) // #nosec G201 // this is a controlled query tableName must match a regex - } + query := fmt.Sprintf("select table_name from %s.ducklake_table", constants.DuckLakeMetadataCatalog) rows, err := db.QueryContext(ctx, query) if err != nil { - return 0, fmt.Errorf("failed to get row count: %w", err) - } - defer rows.Close() - - var count int64 - if rows.Next() { - err = rows.Scan(&count) - if err != nil { - return 0, fmt.Errorf("failed to scan row count: %w", err) - } - } - return count, nil -} - -func GetTableViews(ctx context.Context) ([]string, error) { - // Open a DuckDB connection - db, err := NewDuckDb(WithDbFile(filepaths.TailpipeDbFilePath())) - if err != nil { - return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - - query := "select table_name from information_schema.tables where table_type='VIEW';" - rows, err := db.QueryContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to get table views: %w", err) + return nil, fmt.Errorf("failed to get tables: %w", err) } defer rows.Close() @@ -206,19 +29,15 @@ func GetTableViews(ctx context.Context) ([]string, error) { return tableViews, nil } -func GetTableViewSchema(ctx context.Context, viewName string) (map[string]string, error) { - // Open a DuckDB connection - db, err := NewDuckDb(WithDbFile(filepaths.TailpipeDbFilePath())) - if err != nil { - return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() +func GetTableSchema(ctx context.Context, viewName string, db *DuckDb) (map[string]string, error) { + + query := fmt.Sprintf(`select c.column_name, c.column_type +from %s.ducklake_table t +join %s.ducklake_column c + on t.table_id = c.table_id +where t.table_name = ? +order by c.column_name;`, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) - query := ` - select column_name, data_type - from information_schema.columns - where table_name = ? ORDER BY columns.column_name; - ` rows, err := db.QueryContext(ctx, query, viewName) if err != nil { return nil, fmt.Errorf("failed to get view schema for %s: %w", viewName, err) diff --git a/internal/display/partition.go b/internal/display/partition.go index 2249cfc4..935aefeb 100644 --- a/internal/display/partition.go +++ b/internal/display/partition.go @@ -3,12 +3,10 @@ package display import ( "context" "fmt" - "strings" - "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" + "github.com/turbot/tailpipe/internal/parquet" ) // PartitionResource represents a partition resource and is used for list/show commands @@ -17,6 +15,18 @@ type PartitionResource struct { Description *string `json:"description,omitempty"` Plugin string `json:"plugin"` Local TableResourceFiles `json:"local,omitempty"` + table string + partition string +} + +func NewPartitionResource(p *config.Partition) *PartitionResource { + return &PartitionResource{ + Name: p.UnqualifiedName, + Description: p.Description, + Plugin: p.Plugin.Alias, + table: p.TableName, + partition: p.ShortName, + } } // GetShowData implements the printers.Showable interface @@ -43,10 +53,11 @@ func (r *PartitionResource) GetListData() *printers.RowData { return res } -func ListPartitionResources(ctx context.Context) ([]*PartitionResource, error) { +func ListPartitionResources(ctx context.Context, db *database.DuckDb) ([]*PartitionResource, error) { var res []*PartitionResource - // TODO Add in unconfigured partitions to list output + // TODO Add in unconfigured partitions which exist in database but not configt to list output + // https://github.com/turbot/tailpipe/issues/254 // load all partition names from the data //partitionNames, err := database.ListPartitions(ctx) //if err != nil { @@ -56,14 +67,10 @@ func ListPartitionResources(ctx context.Context) ([]*PartitionResource, error) { partitions := config.GlobalConfig.Partitions for _, p := range partitions { - name := fmt.Sprintf("%s.%s", p.TableName, p.ShortName) - partition := &PartitionResource{ - Name: name, - Description: p.Description, - Plugin: p.Plugin.Alias, - } + partition := NewPartitionResource(p) - err := partition.setFileInformation() + // populate the partition resource with local file information + err := partition.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("error setting file information: %w", err) } @@ -74,18 +81,10 @@ func ListPartitionResources(ctx context.Context) ([]*PartitionResource, error) { return res, nil } -func GetPartitionResource(partitionName string) (*PartitionResource, error) { - p, ok := config.GlobalConfig.Partitions[partitionName] - if !ok { - return nil, fmt.Errorf("no partitions found") - } - partition := &PartitionResource{ - Name: partitionName, - Description: p.Description, - Plugin: p.Plugin.Alias, - } +func GetPartitionResource(ctx context.Context, p *config.Partition, db *database.DuckDb) (*PartitionResource, error) { + partition := NewPartitionResource(p) - err := partition.setFileInformation() + err := partition.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("error setting file information: %w", err) } @@ -93,27 +92,17 @@ func GetPartitionResource(partitionName string) (*PartitionResource, error) { return partition, nil } -func (r *PartitionResource) setFileInformation() error { - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - - nameParts := strings.Split(r.Name, ".") +func (r *PartitionResource) setFileInformation(ctx context.Context, db *database.DuckDb) error { - partitionDir := filepaths.GetParquetPartitionPath(dataDir, nameParts[0], nameParts[1]) - metadata, err := getFileMetadata(partitionDir) + // Get file metadata using shared function + metadata, err := parquet.GetPartitionFileMetadata(ctx, r.table, r.partition, db) if err != nil { - return err + return fmt.Errorf("unable to obtain file metadata: %w", err) } - r.Local.FileMetadata = metadata - - if metadata.FileCount > 0 { - var rc int64 - rc, err = database.GetRowCount(context.Background(), nameParts[0], &nameParts[1]) - if err != nil { - return fmt.Errorf("unable to obtain row count: %w", err) - } - r.Local.RowCount = rc - } + r.Local.FileSize = metadata.FileSize + r.Local.FileCount = metadata.FileCount + r.Local.RowCount = metadata.RowCount return nil } diff --git a/internal/display/shared.go b/internal/display/shared.go index 1f1631ed..9bcba86c 100644 --- a/internal/display/shared.go +++ b/internal/display/shared.go @@ -1,18 +1,10 @@ package display import ( - "math" - "os" - "path/filepath" - "github.com/dustin/go-humanize" + "math" ) -type FileMetadata struct { - FileSize int64 `json:"file_size"` - FileCount int64 `json:"file_count"` -} - func humanizeBytes(bytes int64) string { if bytes == 0 { return "-" @@ -26,30 +18,3 @@ func humanizeCount(count int64) string { } return humanize.Comma(count) } - -func getFileMetadata(basePath string) (FileMetadata, error) { - var metadata FileMetadata - - // if basePath doesn't exist - nothing collected so short-circuit - if _, err := os.Stat(basePath); os.IsNotExist(err) { - return metadata, nil - } - - // Get File Information - err := filepath.Walk(basePath, func(filePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - return nil - } - - metadata.FileCount++ - metadata.FileSize += info.Size() - - return nil - }) - - return metadata, err -} diff --git a/internal/display/table.go b/internal/display/table.go index bb925787..4ac948fb 100644 --- a/internal/display/table.go +++ b/internal/display/table.go @@ -3,18 +3,18 @@ package display import ( "context" "fmt" - "path" "slices" "strings" "github.com/turbot/go-kit/types" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/sanitize" - sdkconstants "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/helpers" "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" ) @@ -29,7 +29,7 @@ type TableResource struct { } // tableResourceFromConfigTable creates a TableResource (display item) from a config.Table (custom table) -func tableResourceFromConfigTable(tableName string, configTable *config.Table) (*TableResource, error) { +func tableResourceFromConfigTable(ctx context.Context, tableName string, configTable *config.Table, db *database.DuckDb) (*TableResource, error) { cols := make([]TableColumnResource, len(configTable.Columns)) for i, c := range configTable.Columns { cols[i] = TableColumnResource{ @@ -47,7 +47,7 @@ func tableResourceFromConfigTable(tableName string, configTable *config.Table) ( } table.setPartitions() - err := table.setFileInformation() + err := table.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("failed to set file information for table '%s': %w", tableName, err) } @@ -56,7 +56,7 @@ func tableResourceFromConfigTable(tableName string, configTable *config.Table) ( } // tableResourceFromSchemaTable creates a TableResource (display item) from a schema.TableSchema (defined table) -func tableResourceFromSchemaTable(tableName string, pluginName string, schemaTable *schema.TableSchema) (*TableResource, error) { +func tableResourceFromSchemaTable(ctx context.Context, tableName string, pluginName string, schemaTable *schema.TableSchema, db *database.DuckDb) (*TableResource, error) { cols := make([]TableColumnResource, len(schemaTable.Columns)) for i, c := range schemaTable.Columns { cols[i] = TableColumnResource{ @@ -74,7 +74,7 @@ func tableResourceFromSchemaTable(tableName string, pluginName string, schemaTab } table.setPartitions() - err := table.setFileInformation() + err := table.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("failed to set file information for table '%s': %w", tableName, err) } @@ -91,8 +91,9 @@ type TableColumnResource struct { // TableResourceFiles represents the file information and a row count for a table resource type TableResourceFiles struct { - FileMetadata - RowCount int64 `json:"row_count,omitempty"` + FileSize int64 `json:"file_size"` + FileCount int64 `json:"file_count"` + RowCount int64 `json:"row_count,omitempty"` } // GetShowData implements the printers.Showable interface @@ -123,7 +124,7 @@ func (r *TableResource) GetListData() *printers.RowData { return res } -func ListTableResources(ctx context.Context) ([]*TableResource, error) { +func ListTableResources(ctx context.Context, db *database.DuckDb) ([]*TableResource, error) { var res []*TableResource tables := make(map[string]*TableResource) @@ -136,25 +137,25 @@ func ListTableResources(ctx context.Context) ([]*TableResource, error) { return nil, fmt.Errorf("unable to obtain plugin list: %w", err) } - for _, p := range plugins { - desc, err := pluginManager.Describe(ctx, p.Name) + for _, partition := range plugins { + desc, err := pluginManager.Describe(ctx, partition.Name) if err != nil { return nil, fmt.Errorf("unable to obtain plugin details: %w", err) } - for t, s := range desc.Schemas { - table, err := tableResourceFromSchemaTable(t, p.Name, s) + for tableName, schema := range desc.Schemas { + table, err := tableResourceFromSchemaTable(ctx, tableName, partition.Name, schema, db) if err != nil { return nil, err } - tables[t] = table + tables[tableName] = table } } // custom tables - these take precedence over plugin defined tables, so overwrite any duplicates in map for tableName, tableDef := range config.GlobalConfig.CustomTables { - table, err := tableResourceFromConfigTable(tableName, tableDef) + table, err := tableResourceFromConfigTable(ctx, tableName, tableDef, db) if err != nil { return nil, err } @@ -170,10 +171,10 @@ func ListTableResources(ctx context.Context) ([]*TableResource, error) { return res, nil } -func GetTableResource(ctx context.Context, tableName string) (*TableResource, error) { +func GetTableResource(ctx context.Context, tableName string, db *database.DuckDb) (*TableResource, error) { // custom table takes precedence over plugin defined table, check there first if customTable, ok := config.GlobalConfig.CustomTables[tableName]; ok { - table, err := tableResourceFromConfigTable(tableName, customTable) + table, err := tableResourceFromConfigTable(ctx, tableName, customTable, db) return table, err } @@ -194,7 +195,7 @@ func GetTableResource(ctx context.Context, tableName string) (*TableResource, er } if tableSchema, ok := desc.Schemas[tableName]; ok { - return tableResourceFromSchemaTable(tableName, pluginName, tableSchema) + return tableResourceFromSchemaTable(ctx, tableName, pluginName, tableSchema, db) } else { return nil, fmt.Errorf("table %s not found", tableName) } @@ -210,22 +211,16 @@ func (r *TableResource) setPartitions() { slices.Sort(r.Partitions) } -func (r *TableResource) setFileInformation() error { - metadata, err := getFileMetadata(path.Join(config.GlobalWorkspaceProfile.GetDataDir(), fmt.Sprintf("%s=%s", sdkconstants.TpTable, r.Name))) +func (r *TableResource) setFileInformation(ctx context.Context, db *database.DuckDb) error { + // Get file metadata using shared function + metadata, err := parquet.GetTableFileMetadata(ctx, r.Name, db) if err != nil { return fmt.Errorf("unable to obtain file metadata: %w", err) } - r.Local.FileMetadata = metadata - - if metadata.FileCount > 0 { - var rc int64 - rc, err = database.GetRowCount(context.Background(), r.Name, nil) - if err != nil { - return fmt.Errorf("unable to obtain row count: %w", err) - } - r.Local.RowCount = rc - } + r.Local.FileSize = metadata.FileSize + r.Local.FileCount = metadata.FileCount + r.Local.RowCount = metadata.RowCount return nil } @@ -235,24 +230,21 @@ func (r *TableResource) getColumnsRenderFunc() printers.RenderFunc { var lines []string lines = append(lines, "") // blank line before column details - cols := r.Columns - // TODO: #graza we utilize similar behaviour in the view creation but only on string, can we combine these into a single func? - tpPrefix := "tp_" - slices.SortFunc(cols, func(a, b TableColumnResource) int { - isPrefixedA, isPrefixedB := strings.HasPrefix(a.ColumnName, tpPrefix), strings.HasPrefix(b.ColumnName, tpPrefix) - switch { - case isPrefixedA && !isPrefixedB: - return 1 // a > b - case !isPrefixedA && isPrefixedB: - return -1 // a < b - default: - return strings.Compare(a.ColumnName, b.ColumnName) // standard alphabetical sort - } - }) + // Extract column names and build map in a single loop + columnNames := make([]string, len(r.Columns)) + columnMap := make(map[string]TableColumnResource) + for i, col := range r.Columns { + columnNames[i] = col.ColumnName + columnMap[col.ColumnName] = col + } + // sort column names alphabetically, with tp fields at the end + sortedColumnNames := helpers.SortColumnsAlphabetically(columnNames) - for _, c := range r.Columns { + // Build lines in sorted order + for _, colName := range sortedColumnNames { + col := columnMap[colName] // type is forced to lowercase, this should be the case for our tables/plugins but this provides consistency for custom tables, etc - line := fmt.Sprintf(" %s: %s", c.ColumnName, strings.ToLower(c.Type)) + line := fmt.Sprintf(" %s: %s", col.ColumnName, strings.ToLower(col.Type)) lines = append(lines, line) } diff --git a/internal/filepaths/database.go b/internal/filepaths/database.go deleted file mode 100644 index c11b9cbf..00000000 --- a/internal/filepaths/database.go +++ /dev/null @@ -1,13 +0,0 @@ -package filepaths - -import ( - "path/filepath" - - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/constants" -) - -func TailpipeDbFilePath() string { - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - return filepath.Join(dataDir, constants.TailpipeDbName) -} diff --git a/internal/filepaths/parquet.go b/internal/filepaths/parquet.go deleted file mode 100644 index 0e4cd22a..00000000 --- a/internal/filepaths/parquet.go +++ /dev/null @@ -1,37 +0,0 @@ -package filepaths - -import ( - "fmt" - - "path/filepath" - - pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" -) - -const TempParquetExtension = ".parquet.tmp" - -func GetParquetFileGlobForTable(dataDir, tableName, fileRoot string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/*/*/*/%s*.parquet", tableName, fileRoot)) -} - -func GetParquetFileGlobForPartition(dataDir, tableName, partitionName, fileRoot string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s/*/*/%s*.parquet", tableName, partitionName, fileRoot)) -} - -func GetTempParquetFileGlobForPartition(dataDir, tableName, partitionName, fileRoot string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s/*/*/%s*%s", tableName, partitionName, fileRoot, TempParquetExtension)) -} - -// GetTempAndInvalidParquetFileGlobForPartition returns a glob pattern for invalid and temporary parquet files for a partition -func GetTempAndInvalidParquetFileGlobForPartition(dataDir, tableName, partitionName string) string { - base := filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s", tableName, partitionName)) - return filepath.Join(base, "*.parquet.*") -} - -func GetParquetPartitionPath(dataDir, tableName, partitionName string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s", tableName, partitionName)) -} - -func InvalidParquetFilePath() string { - return filepath.Join(pfilepaths.EnsureInternalDir(), "invalid_parquet.json") -} diff --git a/internal/filepaths/partition_fields.go b/internal/filepaths/partition_fields.go deleted file mode 100644 index c9e9663a..00000000 --- a/internal/filepaths/partition_fields.go +++ /dev/null @@ -1,72 +0,0 @@ -package filepaths - -import ( - "fmt" - "strconv" - "strings" - "time" -) - -// PartitionFields represents the components of a parquet file path -type PartitionFields struct { - Table string - Partition string - Date time.Time - Index int -} - -// ExtractPartitionFields parses a parquet file path and returns its components. -// Expected path format: -// -// /path/to/dir/tp_table=/tp_partition=/tp_date=/tp_index=/file.parquet -// -// Rules: -// - Fields can appear in any order -// - It is an error for the same field to appear with different values -// - Date must be in YYYY-MM-DD format -// - Missing fields are allowed (will have zero values) -func ExtractPartitionFields(parquetFilePath string) (PartitionFields, error) { - fields := PartitionFields{} - - parts := strings.Split(parquetFilePath, "/") - for _, part := range parts { - switch { - case strings.HasPrefix(part, "tp_table="): - value := strings.TrimPrefix(part, "tp_table=") - if fields.Table != "" && fields.Table != value { - return PartitionFields{}, fmt.Errorf("conflicting table values: %s and %s", fields.Table, value) - } - fields.Table = value - case strings.HasPrefix(part, "tp_partition="): - value := strings.TrimPrefix(part, "tp_partition=") - if fields.Partition != "" && fields.Partition != value { - return PartitionFields{}, fmt.Errorf("conflicting partition values: %s and %s", fields.Partition, value) - } - fields.Partition = value - case strings.HasPrefix(part, "tp_date="): - value := strings.TrimPrefix(part, "tp_date=") - date, err := time.Parse("2006-01-02", value) - if err == nil { - if !fields.Date.IsZero() && !fields.Date.Equal(date) { - return PartitionFields{}, fmt.Errorf("conflicting date values: %s and %s", fields.Date.Format("2006-01-02"), value) - } - fields.Date = date - } - case strings.HasPrefix(part, "tp_index="): - value := strings.TrimPrefix(part, "tp_index=") - if fields.Index != 0 { - if index, err := strconv.Atoi(value); err == nil { - if fields.Index != index { - return PartitionFields{}, fmt.Errorf("conflicting index values: %d and %s", fields.Index, value) - } - } - } else { - if index, err := strconv.Atoi(value); err == nil { - fields.Index = index - } - } - } - } - - return fields, nil -} diff --git a/internal/filepaths/partition_fields_test.go b/internal/filepaths/partition_fields_test.go deleted file mode 100644 index a95118de..00000000 --- a/internal/filepaths/partition_fields_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package filepaths - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestExtractPartitionFields(t *testing.T) { - tests := []struct { - name string - path string - expected PartitionFields - expectError bool - }{ - { - name: "complete path", - path: "/some/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_index=1/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 1, - }, - expectError: false, - }, - { - name: "missing index", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 0, - }, - expectError: false, - }, - { - name: "invalid date", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=invalid/tp_index=1/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Time{}, - Index: 1, - }, - expectError: false, - }, - { - name: "invalid index", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_index=invalid/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 0, - }, - expectError: false, - }, - { - name: "empty path", - path: "", - expected: PartitionFields{}, - expectError: false, - }, - { - name: "duplicate table field with different values", - path: "/path/tp_table=aws_account/tp_table=aws_iam/tp_partition=123456789/tp_date=2024-03-15/tp_index=1/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate partition field with different values", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_partition=987654321/tp_date=2024-03-15/tp_index=1/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate date field with different values", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_date=2024-03-16/tp_index=1/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate index field with different values", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_index=1/tp_index=2/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate fields with same values should not error", - path: "/path/tp_table=aws_account/tp_table=aws_account/tp_partition=123456789/tp_partition=123456789/tp_date=2024-03-15/tp_date=2024-03-15/tp_index=1/tp_index=1/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 1, - }, - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := ExtractPartitionFields(tt.path) - if tt.expectError { - assert.Error(t, err) - assert.Empty(t, result) - return - } - assert.NoError(t, err) - assert.Equal(t, tt.expected, result) - }) - } -} diff --git a/internal/helpers/sort.go b/internal/helpers/sort.go deleted file mode 100644 index d1b2dbe3..00000000 --- a/internal/helpers/sort.go +++ /dev/null @@ -1,24 +0,0 @@ -package helpers - -import ( - "slices" - "strings" - - "github.com/turbot/tailpipe/internal/constants" -) - -// SortColumnsAlphabetically sorts the column names alphabetically but with tp_ fields on the end -func SortColumnsAlphabetically(columns []string) []string { - slices.SortFunc(columns, func(a, b string) int { - isPrefixedA, isPrefixedB := strings.HasPrefix(a, constants.TpPrefix), strings.HasPrefix(b, constants.TpPrefix) - switch { - case isPrefixedA && !isPrefixedB: - return 1 // a > b - case !isPrefixedA && isPrefixedB: - return -1 // a < b - default: - return strings.Compare(a, b) // normal alphabetical comparison - } - }) - return columns -} diff --git a/internal/interactive/interactive_client.go b/internal/interactive/interactive_client.go index 533166ba..753f0b15 100644 --- a/internal/interactive/interactive_client.go +++ b/internal/interactive/interactive_client.go @@ -48,7 +48,7 @@ type InteractiveClient struct { executionLock sync.Mutex // the schema metadata - this is loaded asynchronously during init //schemaMetadata *db_common.SchemaMetadata - tableViews []string + tables []string highlighter *Highlighter // hidePrompt is used to render a blank as the prompt prefix hidePrompt bool @@ -79,12 +79,12 @@ func newInteractiveClient(ctx context.Context, db *database.DuckDb) (*Interactiv db: db, } - // initialise the table views for autocomplete - tv, err := database.GetTableViews(ctx) + // initialise the table list for autocomplete + tv, err := database.GetTables(ctx, db) if err != nil { return nil, err } - c.tableViews = tv + c.tables = tv // initialise autocomplete suggestions err = c.initialiseSuggestions(ctx) @@ -434,6 +434,7 @@ func (c *InteractiveClient) executeMetaquery(ctx context.Context, query string) Query: query, Prompt: c.interactivePrompt, ClosePrompt: func() { c.afterClose = AfterPromptCloseExit }, + Db: c.db, }) } @@ -478,17 +479,17 @@ func (c *InteractiveClient) queryCompleter(d prompt.Document) []prompt.Suggest { suggestions := c.getFirstWordSuggestions(text) s = append(s, suggestions...) case isDuckDbMetaQuery(text): - tableSuggestions := c.getTableSuggestions(lastWord(text)) + tableSuggestions := c.getTableSuggestions() s = append(s, tableSuggestions...) case metaquery.IsMetaQuery(text): suggestions := metaquery.Complete(&metaquery.CompleterInput{ Query: text, - ViewSuggestions: c.getTableSuggestions(lastWord(text)), + ViewSuggestions: c.getTableSuggestions(), }) s = append(s, suggestions...) default: if queryInfo := getQueryInfo(text); queryInfo.EditingTable { - tableSuggestions := c.getTableSuggestions(lastWord(text)) + tableSuggestions := c.getTableSuggestions() s = append(s, tableSuggestions...) } } @@ -514,24 +515,16 @@ func (c *InteractiveClient) getFirstWordSuggestions(word string) []prompt.Sugges return s } -func (c *InteractiveClient) getTableSuggestions(word string) []prompt.Suggest { +func (c *InteractiveClient) getTableSuggestions() []prompt.Suggest { var s []prompt.Suggest - for _, tv := range c.tableViews { - s = append(s, prompt.Suggest{Text: tv, Output: tv}) + for _, tableName := range c.tables { + s = append(s, prompt.Suggest{Text: tableName, Output: tableName}) } return s } -// -//func (c *InteractiveClient) newSuggestion(itemType string, description string, name string) prompt.Suggest { -// if description != "" { -// itemType += fmt.Sprintf(": %s", description) -// } -// return prompt.Suggest{Text: name, Output: name, Description: itemType} -//} - func (c *InteractiveClient) startCancelHandler() chan bool { sigIntChannel := make(chan os.Signal, 1) quitChannel := make(chan bool, 1) diff --git a/internal/interactive/interactive_client_autocomplete.go b/internal/interactive/interactive_client_autocomplete.go index bab662cb..5f2b86c1 100644 --- a/internal/interactive/interactive_client_autocomplete.go +++ b/internal/interactive/interactive_client_autocomplete.go @@ -2,12 +2,9 @@ package interactive import ( "context" - "log" ) func (c *InteractiveClient) initialiseSuggestions(ctx context.Context) error { - log.Printf("[TRACE] initialiseSuggestions") - // reset suggestions c.suggestions = newAutocompleteSuggestions() c.suggestions.sort() diff --git a/internal/interactive/interactive_helpers.go b/internal/interactive/interactive_helpers.go index 2c867b11..34932e7a 100644 --- a/internal/interactive/interactive_helpers.go +++ b/internal/interactive/interactive_helpers.go @@ -71,11 +71,6 @@ func isFirstWord(text string) bool { return strings.LastIndex(text, " ") == -1 } -// split the string by spaces and return the last segment -func lastWord(text string) string { - return text[strings.LastIndex(text, " "):] -} - // isDuckDbMetaQuery returns true if the input string equals 'describe', 'show', or 'summarize' func isDuckDbMetaQuery(s string) bool { ts := strings.ToLower(strings.TrimSpace(s)) diff --git a/internal/metaquery/handler_input.go b/internal/metaquery/handler_input.go index 53fea612..11af2f65 100644 --- a/internal/metaquery/handler_input.go +++ b/internal/metaquery/handler_input.go @@ -14,20 +14,21 @@ type HandlerInput struct { ClosePrompt func() Query string - views *[]string + tables *[]string + Db *database.DuckDb } func (h *HandlerInput) args() []string { return getArguments(h.Query) } -func (h *HandlerInput) GetViews() ([]string, error) { - if h.views == nil { - views, err := database.GetTableViews(context.Background()) +func (h *HandlerInput) GetTables(ctx context.Context) ([]string, error) { + if h.tables == nil { + tables, err := database.GetTables(ctx, h.Db) if err != nil { return nil, err } - h.views = &views + h.tables = &tables } - return *h.views, nil + return *h.tables, nil } diff --git a/internal/metaquery/handler_inspect.go b/internal/metaquery/handler_inspect.go index 1c1fc631..fe4c09a0 100644 --- a/internal/metaquery/handler_inspect.go +++ b/internal/metaquery/handler_inspect.go @@ -6,39 +6,39 @@ import ( "slices" "strings" - "github.com/turbot/tailpipe/internal/helpers" - "github.com/turbot/tailpipe/internal/plugin" - + "github.com/turbot/tailpipe-plugin-sdk/helpers" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/plugin" ) // inspect func inspect(ctx context.Context, input *HandlerInput) error { - views, err := input.GetViews() + tables, err := input.GetTables(ctx) if err != nil { return fmt.Errorf("failed to get tables: %w", err) } if len(input.args()) == 0 { - return listViews(ctx, input, views) + return listTables(ctx, input, tables) } - viewName := input.args()[0] - if slices.Contains(views, viewName) { - return listViewSchema(ctx, input, viewName) + tableName := input.args()[0] + if slices.Contains(tables, tableName) { + return getTableSchema(ctx, input, tableName) } - return fmt.Errorf("could not find a view named '%s'", viewName) + return fmt.Errorf("could not find a view named '%s'", tableName) } -func listViews(ctx context.Context, input *HandlerInput, views []string) error { +func listTables(ctx context.Context, input *HandlerInput, views []string) error { var rows [][]string rows = append(rows, []string{"Table", "Plugin"}) // Header for _, view := range views { // TODO look at using config.GetPluginForTable(ctx, view) instead of this - or perhaps add function + // https://github.com/turbot/tailpipe/issues/500 // GetPluginAndVersionForTable? // getPluginForTable looks at plugin binaries (slower but mre reliable) p, _ := getPluginForTable(ctx, view) @@ -49,10 +49,10 @@ func listViews(ctx context.Context, input *HandlerInput, views []string) error { return nil } -func listViewSchema(ctx context.Context, input *HandlerInput, viewName string) error { - schema, err := database.GetTableViewSchema(ctx, viewName) +func getTableSchema(ctx context.Context, input *HandlerInput, tableName string) error { + schema, err := database.GetTableSchema(ctx, tableName, input.Db) if err != nil { - return fmt.Errorf("failed to get view schema: %w", err) + return fmt.Errorf("failed to get table schema: %w", err) } var rows [][]string diff --git a/internal/parquet/compact.go b/internal/parquet/compact.go index e4fe2bd4..7bea1227 100644 --- a/internal/parquet/compact.go +++ b/internal/parquet/compact.go @@ -2,152 +2,350 @@ package parquet import ( "context" + "database/sql" "fmt" "log/slog" - "os" - "path/filepath" - "strings" "time" - "github.com/spf13/viper" - "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/tailpipe/internal/database" ) -func CompactDataFiles(ctx context.Context, updateFunc func(CompactionStatus), patterns ...PartitionPattern) error { - // get the root data directory - baseDir := config.GlobalWorkspaceProfile.GetDataDir() +const ( + // maxCompactionRowsPerChunk is the maximum number of rows to compact in a single insert operation + maxCompactionRowsPerChunk = 5_000_000 +) + +func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), patterns ...PartitionPattern) error { + slog.Info("Compacting DuckLake data files") + + t := time.Now() - // open a duckdb connection - db, err := database.NewDuckDb() + // get a list of partition key combinations which match any of the patterns + partitionKeys, err := getPartitionKeysMatchingPattern(ctx, db, patterns) if err != nil { - return fmt.Errorf("failed to open duckdb connection: %w", err) + return fmt.Errorf("failed to get partition keys requiring compaction: %w", err) } - defer db.Close() - // if the flag was provided, migrate the tp_index files - if viper.GetBool(constants.ArgReindex) { - // traverse the directory and migrate files - if err := migrateTpIndex(ctx, db, baseDir, updateFunc, patterns); err != nil { - return err - } + if len(partitionKeys) == 0 { + slog.Info("No matching partitions found for compaction") + return nil } - // traverse the directory and compact files - if err := traverseAndCompact(ctx, db, baseDir, updateFunc, patterns); err != nil { + status, err := orderDataFiles(ctx, db, updateFunc, partitionKeys) + if err != nil { + slog.Error("Failed to compact DuckLake parquet files", "error", err) return err } - // now delete any invalid parquet files that match the patterns - invalidDeleteErr := deleteInvalidParquetFiles(config.GlobalWorkspaceProfile.GetDataDir(), patterns) - if invalidDeleteErr != nil { - slog.Warn("Failed to delete invalid parquet files", "error", invalidDeleteErr) + slog.Info("Expiring old DuckLake snapshots") + // now expire unused snapshots + if err := expirePrevSnapshots(ctx, db); err != nil { + slog.Error("Failed to expire previous DuckLake snapshots", "error", err) + return err } - return nil -} -func traverseAndCompact(ctx context.Context, db *database.DuckDb, dirPath string, updateFunc func(CompactionStatus), patterns []PartitionPattern) error { - // if this is the partition folder, check if it matches the patterns before descending further - if table, partition, ok := getPartitionFromPath(dirPath); ok { - if !PartitionMatchesPatterns(table, partition, patterns) { - return nil - } + slog.Info("[SKIPPING] Merging adjacent DuckLake parquet files") + // TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 + // so we should now have multiple, time ordered parquet files + // now merge the the parquet files in the duckdb database + // the will minimise the parquet file count to the optimum + // if err := mergeParquetFiles(ctx, db); err != nil { + // slog.Error("Failed to merge DuckLake parquet files", "error", err) + // return nil, err + // } + + slog.Info("Cleaning up expired files in DuckLake") + // delete unused files + if err := cleanupExpiredFiles(ctx, db); err != nil { + slog.Error("Failed to cleanup expired files", "error", err) + return err } - entries, err := os.ReadDir(dirPath) + // get the file count after merging and cleanup + finalFileCount, err := getFileCountForPartitionKeys(ctx, db, partitionKeys) if err != nil { - return fmt.Errorf("failed to read directory %s: %w", dirPath, err) + return err } + // update status + status.FinalFiles = finalFileCount + // set the compaction time + status.Duration = time.Since(t) - var parquetFiles []string + // call final update + updateFunc(*status) - // process directory entries - for _, entry := range entries { - if entry.IsDir() { - // recursively process subdirectories - subDirPath := filepath.Join(dirPath, entry.Name()) - err := traverseAndCompact(ctx, db, subDirPath, updateFunc, patterns) - if err != nil { - return err - } - } else if strings.HasSuffix(entry.Name(), ".parquet") { - // collect parquet file paths - parquetFiles = append(parquetFiles, filepath.Join(dirPath, entry.Name())) - } - } - numFiles := len(parquetFiles) - if numFiles < 2 { - // nothing to compact - update the totals anyway so we include uncompacted files in the overall total - updateFunc(CompactionStatus{Uncompacted: numFiles}) - return nil - } + slog.Info("DuckLake compaction complete", "source_file_count", status.InitialFiles, "destination_file_count", status.FinalFiles) + return nil +} - err = compactParquetFiles(ctx, db, parquetFiles, dirPath) - if err != nil { +//nolint: unused // TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 +// mergeParquetFiles combines adjacent parquet files in the DuckDB database. +func mergeParquetFiles(ctx context.Context, db *database.DuckDb) error { + if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { if ctx.Err() != nil { return err } - return fmt.Errorf("failed to compact parquet files in %s: %w", dirPath, err) + return fmt.Errorf("failed to merge parquet files: %w", err) } - - // update the totals - updateFunc(CompactionStatus{Source: numFiles, Dest: 1}) - return nil } -// compactParquetFiles compacts the given parquet files into a single file in the specified inputPath. -func compactParquetFiles(ctx context.Context, db *database.DuckDb, parquetFiles []string, inputPath string) (err error) { - now := time.Now() - compactedFileName := fmt.Sprintf("snap_%s_%06d.parquet", now.Format("20060102150405"), now.Nanosecond()/1000) +// we order data files as follows: +// - get list of partition keys matching patterns. For each key: +// - analyze file fragmentation to identify overlapping time ranges +// - for each overlapping time range, reorder all data in that range +// - delete original unordered entries for that time range +func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), partitionKeys []*partitionKey) (*CompactionStatus, error) { + slog.Info("Ordering DuckLake data files") - if !filepath.IsAbs(inputPath) { - return fmt.Errorf("inputPath must be an absolute path") + status := NewCompactionStatus() + // get total file and row count for status - iterating over partition keys + for _, pk := range partitionKeys { + status.InitialFiles += pk.fileCount + status.TotalRows += pk.stats.rowCount } - // define temp and output file paths - tempOutputFile := filepath.Join(inputPath, compactedFileName+".tmp") - outputFile := filepath.Join(inputPath, compactedFileName) - defer func() { + // Process each partition + for _, pk := range partitionKeys { + // determine which files are not time ordered + unorderedRanges, err := getUnorderedRangesForPartitionKey(ctx, db, pk) if err != nil { - if ctx.Err() == nil { - slog.Error("Compaction failed", "inputPath", inputPath, "error", err) + slog.Error("failed to get unorderedRanges", "partition", pk, "error", err) + return nil, err + } + // if no files out of order, nothing to do + if len(unorderedRanges) == 0 { + slog.Debug("Partition key is not out of order - skipping reordering", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + ) + continue + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + // This is a system failure - stop everything + return nil, fmt.Errorf("failed to begin transaction for partition %v: %w", pk, err) + } + + slog.Info("Compacting partition entries", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + "unorderedRanges",len(unorderedRanges), + ) + + // func to update status with number of rows compacted for this partition key + // - passed to compactAndOrderPartitionKeyEntries + updateRowsFunc := func(rowsCompacted int64) { + status.RowsCompacted += rowsCompacted + if status.TotalRows > 0 { + status.ProgressPercent = (float64(status.RowsCompacted) / float64(status.TotalRows)) * 100 } - // delete temp file if it exists - _ = os.Remove(tempOutputFile) + updateFunc(*status) } - }() - // compact files using duckdb - query := fmt.Sprintf(` - copy ( - select * from read_parquet('%s/*.parquet') - ) to '%s' (format parquet, overwrite true); - `, inputPath, tempOutputFile) + if err := compactAndOrderPartitionKeyEntries(ctx, tx, pk, unorderedRanges, updateRowsFunc); err != nil { + slog.Error("failed to compact partition", "partition", pk, "error", err) + txErr := tx.Rollback() + if txErr != nil { + slog.Error("failed to rollback transaction after compaction", "partition", pk, "error", txErr) + } + return nil, err + } - if _, err := db.ExecContext(ctx, query); err != nil { - if ctx.Err() != nil { - return err + if err := tx.Commit(); err != nil { + slog.Error("failed to commit transaction after compaction", "partition", pk, "error", err) + txErr := tx.Rollback() + if txErr != nil { + slog.Error("failed to rollback transaction after compaction", "partition", pk, "error", txErr) + } + return nil, err } - return fmt.Errorf("failed to compact parquet files: %w", err) + + slog.Info("Compacted and ordered all partition entries", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + "input_files", pk.fileCount, + ) + } - // rename all parquet files to add a .compacted extension - renamedSourceFiles, err := addExtensionToFiles(parquetFiles, ".compacted") + slog.Info("Finished ordering DuckLake data file") + return status, nil +} + +// compactAndOrderPartitionKeyEntries processes overlapping time ranges for a partition key: +// - iterates over each unordered time range +// - reorders all data within each time range (potentially in chunks for large ranges) +// - deletes original unordered entries for that time range +func compactAndOrderPartitionKeyEntries(ctx context.Context, tx *sql.Tx, pk *partitionKey, unorderedRanges []unorderedDataTimeRange, updateRowsCompactedFunc func(int64)) error { + + slog.Debug("partition statistics", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + "row_count", pk.stats.rowCount, + "total file_count", pk.fileCount, + "min_timestamp", pk.stats.minTimestamp, + "max_timestamp", pk.stats.maxTimestamp, + "total_ranges", len(unorderedRanges), + ) + + // Process each overlapping time range + for i, timeRange := range unorderedRanges { + slog.Debug("processing overlapping time range", + "range_index", i+1, + "start_time", timeRange.StartTime, + "end_time", timeRange.EndTime, + "row_count", timeRange.RowCount) + + // Use the pre-calculated time range and row count from the struct + minTime := timeRange.StartTime + maxTime := timeRange.EndTime + rowCount := timeRange.RowCount + + // Determine chunking strategy for this time range + chunks, intervalDuration := determineChunkingInterval(minTime, maxTime, rowCount) + + slog.Debug("processing time range in chunks", + "range_index", i+1, + "row_count", rowCount, + "chunks", chunks, + "interval_duration", intervalDuration.String()) + + // Process this time range in chunks + currentStart := minTime + for i := 1; currentStart.Before(maxTime); i++ { + currentEnd := currentStart.Add(intervalDuration) + if currentEnd.After(maxTime) { + currentEnd = maxTime + } + + // For the final chunk, make it inclusive to catch the last row + isFinalChunk := currentEnd.Equal(maxTime) + + rowsInserted, err := insertOrderedDataForTimeRange(ctx, tx, pk, currentStart, currentEnd, isFinalChunk) + if err != nil { + return fmt.Errorf("failed to insert ordered data for time range %s to %s: %w", + currentStart.Format("2006-01-02 15:04:05"), + currentEnd.Format("2006-01-02 15:04:05"), err) + } + updateRowsCompactedFunc(rowsInserted) + slog.Debug(fmt.Sprintf("processed chunk %d/%d for range %d", i, chunks, i+1)) + + // Ensure next chunk starts exactly where this one ended to prevent gaps + currentStart = currentEnd + } + + // Delete original unordered entries for this time range + err := deleteUnorderedEntriesForTimeRange(ctx, tx, pk, minTime, maxTime) + if err != nil { + return fmt.Errorf("failed to delete unordered entries for time range: %w", err) + } + + slog.Debug("completed time range", + "range_index", i+1) + } + + return nil +} + +// insertOrderedDataForTimeRange inserts ordered data for a specific time range within a partition key +func insertOrderedDataForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitionKey, startTime, endTime time.Time, isFinalChunk bool) (int64, error) { + // For the final chunk, use inclusive end time to catch the last row + timeEndOperator := "<" + if isFinalChunk { + timeEndOperator = "<=" + } + + // For overlapping files, we need to reorder ALL rows in the overlapping time range + // Since files overlap, we can't distinguish which specific rows came from which files + // So we reorder all rows in the time range for this partition + args := []interface{}{startTime, endTime, pk.tpPartition, pk.tpIndex} + + tableName, err := backend.SanitizeDuckDBIdentifier(pk.tpTable) if err != nil { - // delete the temp file - _ = os.Remove(tempOutputFile) - return err + return 0, err } + //nolint: gosec // sanitized + insertQuery := fmt.Sprintf(`insert into %s + select * from %s + where tp_timestamp >= ? + and tp_timestamp %s ? + and tp_partition = ? + and tp_index = ? + order by tp_timestamp`, + tableName, + tableName, + timeEndOperator) - // rename temp file to final output file - if err := os.Rename(tempOutputFile, outputFile); err != nil { - return fmt.Errorf("failed to rename temp file %s to %s: %w", tempOutputFile, outputFile, err) + result, err := tx.ExecContext(ctx, insertQuery, args...) + if err != nil { + return 0, fmt.Errorf("failed to insert ordered data for time range: %w", err) + } + rowsInserted, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get rows affected count: %w", err) + } + return rowsInserted, nil +} + +// deleteUnorderedEntriesForTimeRange deletes the original unordered entries for a specific time range within a partition key +func deleteUnorderedEntriesForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitionKey, startTime, endTime time.Time) error { + // Delete all rows in the time range for this partition key (we're re-inserting them in order) + tableName, err := backend.SanitizeDuckDBIdentifier(pk.tpTable) + if err != nil { + return err } + //nolint: gosec // sanitized + deleteQuery := fmt.Sprintf(`delete from %s + where tp_partition = ? + and tp_index = ? + and tp_timestamp >= ? + and tp_timestamp <= ? + and rowid <= ?`, + tableName) - // finally, delete renamed source parquet files - err = deleteFilesConcurrently(ctx, renamedSourceFiles, config.GlobalWorkspaceProfile.GetDataDir()) + args := []interface{}{pk.tpPartition, pk.tpIndex, startTime, endTime, pk.stats.maxRowId} + + _, err = tx.ExecContext(ctx, deleteQuery, args...) + if err != nil { + return fmt.Errorf("failed to delete unordered entries for time range: %w", err) + } return nil } + +// determineChunkingInterval calculates the optimal chunking strategy for a time range based on row count. +// It returns the number of chunks and the duration of each chunk interval. +// For large datasets, it splits the time range into multiple chunks to stay within maxCompactionRowsPerChunk. +// Ensures minimum chunk interval is at least 1 hour to avoid excessive fragmentation. +func determineChunkingInterval(startTime, endTime time.Time, rowCount int64) (chunks int, intervalDuration time.Duration) { + intervalDuration = endTime.Sub(startTime) + chunks = 1 + + // If row count is greater than maxCompactionRowsPerChunk, calculate appropriate chunk interval + if rowCount > maxCompactionRowsPerChunk { + chunks = int((rowCount + maxCompactionRowsPerChunk - 1) / maxCompactionRowsPerChunk) + intervalDuration = intervalDuration / time.Duration(chunks) + + // Ensure minimum interval is at least 1 hour + if intervalDuration < time.Hour { + intervalDuration = time.Hour + } + } + + return chunks, intervalDuration +} diff --git a/internal/parquet/compaction_status.go b/internal/parquet/compaction_status.go index 65b8dfee..a24d36ac 100644 --- a/internal/parquet/compaction_status.go +++ b/internal/parquet/compaction_status.go @@ -2,37 +2,28 @@ package parquet import ( "fmt" + "github.com/dustin/go-humanize" "github.com/turbot/pipe-fittings/v2/utils" - "golang.org/x/exp/maps" + "time" ) type CompactionStatus struct { - Source int - Dest int - Uncompacted int + InitialFiles int + FinalFiles int + RowsCompacted int64 + TotalRows int64 + ProgressPercent float64 MigrateSource int // number of source files migrated MigrateDest int // number of destination files after migration PartitionIndexExpressions map[string]string // the index expression used for migration for each partition + Duration time.Duration // duration of the compaction process } func NewCompactionStatus() *CompactionStatus { return &CompactionStatus{ PartitionIndexExpressions: make(map[string]string), } - -} - -func (s *CompactionStatus) Update(other CompactionStatus) { - s.Source += other.Source - s.Dest += other.Dest - s.Uncompacted += other.Uncompacted - s.MigrateSource += other.MigrateSource - s.MigrateDest += other.MigrateDest - if s.PartitionIndexExpressions == nil { - s.PartitionIndexExpressions = make(map[string]string) - } - maps.Copy(s.PartitionIndexExpressions, other.PartitionIndexExpressions) } func (s *CompactionStatus) VerboseString() string { @@ -44,7 +35,6 @@ func (s *CompactionStatus) VerboseString() string { utils.Pluralize("partition", len(s.PartitionIndexExpressions)), ) if s.MigrateSource != s.MigrateDest { - migratedString += fmt.Sprintf(" (%d %s migrated to %d %s)", s.MigrateSource, utils.Pluralize("file", s.MigrateSource), @@ -54,38 +44,53 @@ func (s *CompactionStatus) VerboseString() string { migratedString += ".\n" } - var uncompactedString, compactedString string - if s.Source == 0 && s.Dest == 0 && s.Uncompacted == 0 { - compactedString = "\nNo files to compact." + var compactedString string + if s.RowsCompacted == 0 { + compactedString = "\nNo files required compaction." } else { - - if s.Uncompacted > 0 { - uncompactedString = fmt.Sprintf("%d files did not need compaction.", s.Uncompacted) - } - - if s.Source > 0 { - if len(uncompactedString) > 0 { - uncompactedString = fmt.Sprintf(" (%s)", uncompactedString) - } - compactedString = fmt.Sprintf("Compacted %d files into %d files.%s\n", s.Source, s.Dest, uncompactedString) + // if the file count is the same, we must have just ordered + if s.InitialFiles == s.FinalFiles { + compactedString = fmt.Sprintf("Ordered %s rows in %s files (%s).\n", s.TotalRowsString(), s.InitialFilesString(), s.DurationString()) } else { - // Nothing compacted; show only uncompacted note if present - compactedString = uncompactedString + "\n\n" + compactedString = fmt.Sprintf("Compacted and ordered %s rows in %s files into %s files in (%s).\n", s.TotalRowsString(), s.InitialFilesString(), s.FinalFilesString(), s.DurationString()) } } return migratedString + compactedString } -func (s *CompactionStatus) BriefString() string { - if s.Source == 0 { - return "" +func (s *CompactionStatus) String() string { + var migratedString string + var compactedString string + if s.RowsCompacted == 0 { + compactedString = "No files required compaction." + } else { + // if the file count is the same, we must have just ordered + if s.InitialFiles == s.FinalFiles { + compactedString = fmt.Sprintf("Ordered %s rows in %s files in %s.\n", s.TotalRowsString(), s.InitialFilesString(), s.Duration.String()) + } else { + compactedString = fmt.Sprintf("Compacted and ordered %s rows in %s files into %s files in %s.\n", s.TotalRowsString(), s.InitialFilesString(), s.FinalFilesString(), s.Duration.String()) + } } - uncompactedString := "" - if s.Uncompacted > 0 { - uncompactedString = fmt.Sprintf(" (%d files did not need compaction.)", s.Uncompacted) - } + return migratedString + compactedString +} - return fmt.Sprintf("Compacted %d files into %d files.%s\n", s.Source, s.Dest, uncompactedString) +func (s *CompactionStatus) TotalRowsString() any { + return humanize.Comma(s.TotalRows) +} +func (s *CompactionStatus) InitialFilesString() any { + return humanize.Comma(int64(s.InitialFiles)) +} +func (s *CompactionStatus) FinalFilesString() any { + return humanize.Comma(int64(s.FinalFiles)) +} +func (s *CompactionStatus) DurationString() string { + return utils.HumanizeDuration(s.Duration) +} +func (s *CompactionStatus) RowsCompactedString() any { + return humanize.Comma(s.RowsCompacted) +} +func (s *CompactionStatus) ProgressPercentString() string { + return fmt.Sprintf("%.1f%%", s.ProgressPercent) } diff --git a/internal/parquet/compaction_types.go b/internal/parquet/compaction_types.go new file mode 100644 index 00000000..4af4cda4 --- /dev/null +++ b/internal/parquet/compaction_types.go @@ -0,0 +1,136 @@ +package parquet + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/turbot/tailpipe/internal/database" +) + +// getUnorderedRangesForPartitionKey analyzes file fragmentation and creates disorder metrics for a partition key. +// It queries DuckLake metadata to get all files for the partition, their timestamp ranges, and row counts. +// Then it identifies groups of files with overlapping time ranges that need compaction. +// Returns metrics including total file count and overlapping file sets with their metadata. +func getUnorderedRangesForPartitionKey(ctx context.Context, db *database.DuckDb, pk *partitionKey) ([]unorderedDataTimeRange, error) { + // Single query to get files and their timestamp ranges and row counts for this partition key + query := `select + df.path, + cast(fcs.min_value as timestamp) as min_timestamp, + cast(fcs.max_value as timestamp) as max_timestamp, + df.record_count + from __ducklake_metadata_tailpipe_ducklake.ducklake_data_file df + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv1 + on df.data_file_id = fpv1.data_file_id and fpv1.partition_key_index = 0 + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv2 + on df.data_file_id = fpv2.data_file_id and fpv2.partition_key_index = 1 + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv3 + on df.data_file_id = fpv3.data_file_id and fpv3.partition_key_index = 2 + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv4 + on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 + join __ducklake_metadata_tailpipe_ducklake.ducklake_table t + on df.table_id = t.table_id + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_column_statistics fcs + on df.data_file_id = fcs.data_file_id + join __ducklake_metadata_tailpipe_ducklake.ducklake_column c + on fcs.column_id = c.column_id + where t.table_name = ? + and fpv1.partition_value = ? + and fpv2.partition_value = ? + and fpv3.partition_value = ? + and fpv4.partition_value = ? + and c.column_name = 'tp_timestamp' + and df.end_snapshot is null + and c.end_snapshot is null + order by df.data_file_id` + + rows, err := db.QueryContext(ctx, query, pk.tpTable, pk.tpPartition, pk.tpIndex, pk.year, pk.month) + if err != nil { + return nil, fmt.Errorf("failed to get file timestamp ranges: %w", err) + } + defer rows.Close() + + var fileRanges []fileTimeRange + for rows.Next() { + var path string + var minTime, maxTime time.Time + var rowCount int64 + if err := rows.Scan(&path, &minTime, &maxTime, &rowCount); err != nil { + return nil, fmt.Errorf("failed to scan file range: %w", err) + } + fileRanges = append(fileRanges, fileTimeRange{path: path, min: minTime, max: maxTime, rowCount: rowCount}) + } + + totalFiles := len(fileRanges) + if totalFiles <= 1 { + return nil, nil + } + + // build string for the ranges + var rangesStr strings.Builder + for i, file := range fileRanges { + rangesStr.WriteString(fmt.Sprintf("start: %s, end: %s", file.min.String(), file.max.String())) + if i < len(fileRanges)-1 { + rangesStr.WriteString(", ") + } + } + // Build unordered time ranges + unorderedRanges, err := pk.findOverlappingFileRanges(fileRanges) + if err != nil { + return nil, fmt.Errorf("failed to build unordered time ranges: %w", err) + } + + return unorderedRanges, nil +} + +type fileTimeRange struct { + path string + min time.Time + max time.Time + rowCount int64 +} + +// unorderedDataTimeRange represents a time range containing unordered data that needs reordering +type unorderedDataTimeRange struct { + StartTime time.Time // start of the time range containing unordered data + EndTime time.Time // end of the time range containing unordered data + RowCount int64 // total row count in this time range +} + +// newUnorderedDataTimeRange creates a single unorderedDataTimeRange from overlapping files +func newUnorderedDataTimeRange(overlappingFiles []fileTimeRange) (unorderedDataTimeRange, error) { + var rowCount int64 + var startTime, endTime time.Time + + // Single loop to sum row counts and calculate time range + for i, file := range overlappingFiles { + rowCount += file.rowCount + + // Calculate time range + if i == 0 { + startTime = file.min + endTime = file.max + } else { + if file.min.Before(startTime) { + startTime = file.min + } + if file.max.After(endTime) { + endTime = file.max + } + } + } + + return unorderedDataTimeRange{ + StartTime: startTime, + EndTime: endTime, + RowCount: rowCount, + }, nil +} + +// rangesOverlap checks if two timestamp ranges overlap (excluding contiguous ranges) +func rangesOverlap(r1, r2 fileTimeRange) bool { + // Two ranges overlap if one starts before the other ends AND they're not just touching + // Contiguous ranges (where one ends exactly when the other starts) are NOT considered overlapping + return r1.min.Before(r2.max) && r2.min.Before(r1.max) +} diff --git a/internal/parquet/conversion_error.go b/internal/parquet/conversion_error.go index 16725f85..2e8e50a4 100644 --- a/internal/parquet/conversion_error.go +++ b/internal/parquet/conversion_error.go @@ -12,16 +12,17 @@ import ( // handleConversionError attempts to handle conversion errors by counting the number of lines in the file. // if we fail, just return the raw error. -func handleConversionError(err error, path string) error { +// TODO we need to pass an error prefix into here so we know the context https://github.com/turbot/tailpipe/issues/477 +func handleConversionError(err error, paths ...string) error { logArgs := []any{ "error", err, "path", - path, + paths, } // try to count the number of rows in the file - rows, countErr := countLines(path) + rows, countErr := countLinesForFiles(paths...) if countErr == nil { logArgs = append(logArgs, "rows_affected", rows) } @@ -33,9 +34,19 @@ func handleConversionError(err error, path string) error { } // return wrapped error - return NewConversionError(err, rows, path) + return NewConversionError(err, rows, paths...) +} +func countLinesForFiles(filenames ...string) (int64, error) { + total := 0 + for _, filename := range filenames { + count, err := countLines(filename) + if err != nil { + return 0, fmt.Errorf("failed to count lines in %s: %w", filename, err) + } + total += int(count) + } + return int64(total), nil } - func countLines(filename string) (int64, error) { file, err := os.Open(filename) if err != nil { @@ -61,15 +72,19 @@ func countLines(filename string) (int64, error) { } type ConversionError struct { - SourceFile string + SourceFiles []string BaseError error RowsAffected int64 displayError string } -func NewConversionError(err error, rowsAffected int64, path string) *ConversionError { +func NewConversionError(err error, rowsAffected int64, paths ...string) *ConversionError { + sourceFiles := make([]string, len(paths)) + for i, path := range paths { + sourceFiles[i] = filepath.Base(path) + } return &ConversionError{ - SourceFile: filepath.Base(path), + SourceFiles: sourceFiles, BaseError: err, RowsAffected: rowsAffected, displayError: strings.Split(err.Error(), "\n")[0], @@ -77,7 +92,7 @@ func NewConversionError(err error, rowsAffected int64, path string) *ConversionE } func (c *ConversionError) Error() string { - return fmt.Sprintf("%s: %s", c.SourceFile, c.displayError) + return fmt.Sprintf("%s: %s", strings.Join(c.SourceFiles, ", "), c.displayError) } // Merge adds a second error to the conversion error message. diff --git a/internal/parquet/conversion_worker.go b/internal/parquet/conversion_worker.go deleted file mode 100644 index fada8860..00000000 --- a/internal/parquet/conversion_worker.go +++ /dev/null @@ -1,584 +0,0 @@ -package parquet - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - "time" - - "github.com/marcboeker/go-duckdb/v2" - sdkconstants "github.com/turbot/tailpipe-plugin-sdk/constants" - "github.com/turbot/tailpipe-plugin-sdk/table" - "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" -) - -// limit tha max partitions to convert -const maxPartitionsPerConversion = 1000 - -type parquetJob struct { - chunkNumber int32 -} - -// conversionWorker is an implementation of worker that converts JSONL files to Parquet -type conversionWorker struct { - // channel to receive jobs from the writer - jobChan chan *parquetJob - - // the parent converter - converter *Converter - - // source file location - sourceDir string - // dest file location - destDir string - - // helper struct which provides unique filename roots - fileRootProvider *FileRootProvider - db *database.DuckDb - maxMemoryMb int - partitionKeysPerConversion int - // the worker id - a zero based index - used for logging - id int -} - -func newConversionWorker(converter *Converter, maxMemoryMb int, id int) (*conversionWorker, error) { - w := &conversionWorker{ - id: id, - jobChan: converter.jobChan, - sourceDir: converter.sourceDir, - destDir: converter.destDir, - fileRootProvider: converter.fileRootProvider, - converter: converter, - maxMemoryMb: maxMemoryMb, - partitionKeysPerConversion: maxPartitionsPerConversion, - } - - if err := w.validate(); err != nil { - return nil, err - } - if err := w.createDuckDbConnection(); err != nil { - return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - - return w, nil -} - -// validate our params -func (w *conversionWorker) validate() error { - maxAllowedMemoryMB := 256 * 1024 // 256GB in MB - if w.maxMemoryMb < 0 || w.maxMemoryMb > maxAllowedMemoryMB { - return fmt.Errorf("memory must be between 0 and %d MB, got %d", maxAllowedMemoryMB, w.maxMemoryMb) - } - return nil -} - -// this is the worker function run by all workers, which all read from the ParquetJobPool channel -func (w *conversionWorker) start(ctx context.Context) { - slog.Debug("worker start") - // this function runs as long as the worker is running - - // ensure to close on exit - defer w.close() - - // loop until we are closed - for { - select { - case <-ctx.Done(): - // we are done - return - case job := <-w.jobChan: - if job == nil { - // we are done - return - } - slog.Debug("worker got job", "chunk_number", job.chunkNumber) - if err := w.doJSONToParquetConversion(job.chunkNumber); err != nil { - // send the error to the converter - w.converter.addJobErrors(err) - continue - } - // atomically increment the completion count on our converter - w.converter.updateCompletionCount(1) - - } - } -} - -func (w *conversionWorker) close() { - _ = w.db.Close() -} - -// createDuckDbConnection creates a new DuckDB connection, setting the max memory limit -func (w *conversionWorker) createDuckDbConnection() error { - opts := []database.DuckDbOpt{ - database.WithDuckDbExtensions(constants.DuckDbExtensions), - } - // if a memory limit is set, use it - if w.maxMemoryMb > 0 { - opts = append(opts, database.WithMaxMemoryMb(w.maxMemoryMb)) - } - db, err := database.NewDuckDb(opts...) - - if err != nil { - return fmt.Errorf("failed to reopen DuckDB connection: %w", err) - } - w.db = db - return nil -} - -func (w *conversionWorker) forceMemoryRelease() error { - // we need to flush the memory to release it - do this by setting a low memory limit then the full one - // NOTE: do not set the memory to zero as we have temp table data - const minMemoryMb = 64 - - // Set to minimum memory - note the use of ? parameter - if _, err := w.db.Exec("set max_memory = ? || 'MB';", minMemoryMb); err != nil { - return fmt.Errorf("memory flush failed: %w", err) - } - - // Reset to configured memory limit - if _, err := w.db.Exec("set max_memory = ? || 'MB';", w.maxMemoryMb); err != nil { - return fmt.Errorf("memory reset failed: %w", err) - } - return nil - -} - -func (w *conversionWorker) doJSONToParquetConversion(chunkNumber int32) error { - // ensure we signal the converter when we are done - defer w.converter.wg.Done() - startTime := time.Now() - - // build the source filename - jsonFileName := table.ExecutionIdToJsonlFileName(w.converter.id, chunkNumber) - jsonFilePath := filepath.Join(w.sourceDir, jsonFileName) - - // process the ParquetJobPool - err := w.convertFile(jsonFilePath) - - // delete JSON file (configurable?) - if removeErr := os.Remove(jsonFilePath); removeErr != nil { - // log the error but don't fail - slog.Error("failed to delete JSONL file", "file", jsonFilePath, "error", removeErr) - } - activeDuration := time.Since(startTime) - slog.Debug("converted JSONL to Parquet", "file", jsonFilePath, "duration (ms)", activeDuration.Milliseconds()) - // remove the conversion error (if any) - return err -} - -// convert the given jsonl file to parquet -func (w *conversionWorker) convertFile(jsonlFilePath string) (err error) { - // verify the jsonl file has a .jsonl extension - if filepath.Ext(jsonlFilePath) != ".jsonl" { - return NewConversionError(errors.New("invalid file type - conversionWorker only supports .jsonl files"), 0, jsonlFilePath) - } - // verify file exists - if _, err := os.Stat(jsonlFilePath); os.IsNotExist(err) { - return NewConversionError(errors.New("file does not exist"), 0, jsonlFilePath) - } - - // copy the data from the jsonl file to a temp table - if err := w.copyChunkToTempTable(jsonlFilePath); err != nil { - // copyChunkToTempTable will already have called handleSchemaChangeError anf handleConversionError - return err - } - // defer the cleanup of the temp table - defer func() { - // TODO benchmark whether dropping the table actually makes any difference to memory pressure - // or can we rely on the drop if exists? - // validateRows creates the table temp_data - the cleanupQuery drops it - _, tempTableError := w.db.Exec("drop table if exists temp_data;") - if tempTableError != nil { - slog.Error("failed to drop temp table", "error", tempTableError) - // if we do not already have an error return this error - if err == nil { - err = tempTableError - } - } - }() - - // now validate the data - if validateRowsError := w.validateRows(jsonlFilePath); validateRowsError != nil { - // if the error is NOT RowValidationError, just return it - if !errors.Is(validateRowsError, &RowValidationError{}) { - return handleConversionError(validateRowsError, jsonlFilePath) - } - - // so it IS a row validation error - the invalid rows will have been removed from the temp table - // - process the rest of the chunk - // ensure that we return the row validation error, merged with any other error we receive - defer func() { - if err == nil { - err = validateRowsError - } else { - var conversionError *ConversionError - if errors.As(validateRowsError, &conversionError) { - // we have a conversion error - we need to set the row count to 0 - // so we can report the error - conversionError.Merge(err) - } - err = conversionError - } - }() - } - - // ok now we can do the copy query to write the data in the temp table to parquet files - // we limit the number of partitions we create per copy query to avoid excessive memory usage - - partitionsPerConversion := w.partitionKeysPerConversion - - // get row counts for each distinct partition - partitionRowCounts, err := w.getPartitionRowCounts() - if err != nil { - return handleConversionError(err, jsonlFilePath) - } - slog.Debug("found partition combinations", "count", len(partitionRowCounts)) - - // Process partitions in batches using row offsets. - // - // For each batch: - // - Calculate how many partitions to include (up to partitionsPerConversion) - // - Sum the row counts for the selected partitions to determine how many rows to process - // - Export the corresponding rows to Parquet based on rowid range - // - // If an out-of-memory error occurs during export: - // - Reopen the DuckDB connection - // - Halve the number of partitions processed per batch - // - Retry processing - var ( - totalRowCount int64 - rowOffset int64 - ) - - for len(partitionRowCounts) > 0 { - batchSize := partitionsPerConversion - if batchSize > len(partitionRowCounts) { - batchSize = len(partitionRowCounts) - } - - // Calculate total number of rows to process for this batch - var rowsInBatch int64 - for i := 0; i < batchSize; i++ { - rowsInBatch += partitionRowCounts[i] - } - - // Perform conversion for this batch using rowid ranges - rowCount, err := w.doConversionForBatch(jsonlFilePath, rowOffset, rowsInBatch) - if err != nil { - if conversionRanOutOfMemory(err) { - // If out of memory, flush memory, reopen the connection, and retry with fewer partitions - if err := w.forceMemoryRelease(); err != nil { - return err - } - partitionsPerConversion /= 2 - if partitionsPerConversion < 1 { - return fmt.Errorf("failed to convert batch - partition count reduced to 0") - } - slog.Info("JSONL-parquet conversion failed with out of memory - retrying with fewer partitions", "file", jsonlFilePath, "failed partitions", partitionsPerConversion*2, "partitions", partitionsPerConversion, "worker", w.id) - // update partitionKeysPerConversion so the next conversion with this worker uses the new value - w.partitionKeysPerConversion = partitionsPerConversion - continue - } - return err - } - - // Update counters and advance to the next batch - totalRowCount += rowCount - rowOffset += rowsInBatch - partitionRowCounts = partitionRowCounts[batchSize:] - // if we have an error, return it below - // update the row count - w.converter.updateRowCount(rowCount) - - } - - return nil -} - -// conversionRanOutOfMemory checks if the error is an out-of-memory error from DuckDB -func conversionRanOutOfMemory(err error) bool { - var duckDBErr = &duckdb.Error{} - if errors.As(err, &duckDBErr) { - return duckDBErr.Type == duckdb.ErrorTypeOutOfMemory - } - return false -} - -func (w *conversionWorker) copyChunkToTempTable(jsonlFilePath string) error { - var queryBuilder strings.Builder - - // render the read JSON query with the jsonl file path - // - this build a select clause which selects the required data from the JSONL file (with columns types specified) - selectQuery := fmt.Sprintf(w.converter.readJsonQueryFormat, jsonlFilePath) - - // Step: Prepare the temp table from JSONL input - // - // - Drop the temp table if it exists - // - Create a new temp table by reading from the JSONL file - // - Add a row ID (row_number) for stable ordering and chunking - // - Wrap the original select query to allow dot-notation filtering on nested structs later - // - Sort the data by partition key columns (only tp_index, tp_date - there will only be a single table and partition) - // so that full partitions can be selected using only row offsets (because partitions are stored contiguously) - queryBuilder.WriteString(fmt.Sprintf(` -drop table if exists temp_data; - -create temp table temp_data as -select - row_number() over (order by tp_index, tp_date) as rowid, - * -from ( - %s -) -order by - tp_index, tp_date; -`, selectQuery)) - - _, err := w.db.Exec(queryBuilder.String()) - if err != nil { - return w.handleSchemaChangeError(err, jsonlFilePath) - - } - return nil -} - -// getPartitionRowCounts returns a slice of row counts, -// where each count corresponds to a distinct combination of partition key columns -// (tp_table, tp_partition, tp_index, tp_date) in the temp_data table. -// -// The counts are ordered by the partition key columns to allow us to efficiently select -// full partitions based on row offsets without needing additional filtering. -func (w *conversionWorker) getPartitionRowCounts() ([]int64, error) { - // get the distinct partition key combinations - partitionColumns := []string{sdkconstants.TpTable, sdkconstants.TpPartition, sdkconstants.TpIndex, sdkconstants.TpDate} - partitionColumnsString := strings.Join(partitionColumns, ",") - - query := fmt.Sprintf(` - select count(*) as row_count - from temp_data - group by %s - order by %s - `, partitionColumnsString, partitionColumnsString) - - rows, err := w.db.Query(query) - if err != nil { - return nil, err - } - defer rows.Close() - - var result []int64 - for rows.Next() { - var count int64 - if err := rows.Scan(&count); err != nil { - return nil, err - } - result = append(result, count) - } - return result, rows.Err() -} - -// doConversionForBatch writes a batch of rows from the temp_data table to partitioned Parquet files. -// -// It selects rows based on rowid, using the provided startRowId and rowCount to control the range: -// - Rows with rowid > startRowId and rowid <= (startRowId + rowCount) are selected. -// -// This approach ensures that full partitions are processed contiguously and allows efficient batching -// without needing complex WHERE clauses. -// -// Returns the number of rows written and any error encountered. -func (w *conversionWorker) doConversionForBatch(jsonlFilePath string, startRowId int64, rowCount int64) (int64, error) { - // Create a query to write a batch of rows to partitioned Parquet files - - // Get a unique file root - fileRoot := w.fileRootProvider.GetFileRoot() - - // Build select query to pick the correct rows - selectQuery := fmt.Sprintf(` - select * - from temp_data - where rowid > %d and rowid <= %d - `, startRowId, startRowId+rowCount) - - // Build the export query - partitionColumns := []string{sdkconstants.TpTable, sdkconstants.TpPartition, sdkconstants.TpIndex, sdkconstants.TpDate} - exportQuery := fmt.Sprintf(`copy (%s) to '%s' ( - format parquet, - partition_by (%s), - return_files true, - overwrite_or_ignore, - filename_pattern '%s_{i}', - file_extension '%s' -);`, - selectQuery, - w.destDir, - strings.Join(partitionColumns, ","), - fileRoot, - strings.TrimPrefix(filepaths.TempParquetExtension, "."), - ) - - // Execute the export - row := w.db.QueryRow(exportQuery) - var exportedRowCount int64 - var files []interface{} - err := row.Scan(&exportedRowCount, &files) - if err != nil { - return 0, handleConversionError(err, jsonlFilePath) - } - slog.Debug("created parquet files", "count", len(files)) - - // Rename temporary Parquet files - err = w.renameTempParquetFiles(files) - return exportedRowCount, err -} - -// validateRows copies the data from the given select query to a temp table and validates required fields are non null -// it also validates that the schema of the chunk is the same as the inferred schema and if it is not, reports a useful error -// the query count of invalid rows and a list of null fields -func (w *conversionWorker) validateRows(jsonlFilePath string) error { - // build array of required columns to validate - var requiredColumns []string - for _, col := range w.converter.conversionSchema.Columns { - if col.Required { - // if the column is required, add it to the list of columns to validate - requiredColumns = append(requiredColumns, col.ColumnName) - } - } - - // if we have no columns to validate, biuld a validation query to return the number of invalid rows and the columns with nulls - validationQuery := w.buildValidationQuery(requiredColumns) - - row := w.db.QueryRow(validationQuery) - var failedRowCount int64 - var columnsWithNullsInterface []interface{} - - err := row.Scan(&failedRowCount, &columnsWithNullsInterface) - if err != nil { - return w.handleSchemaChangeError(err, jsonlFilePath) - } - - if failedRowCount == 0 { - // no rows with nulls - we are done - return nil - } - - // delete invalid rows from the temp table - if err := w.deleteInvalidRows(requiredColumns); err != nil { - // failed to delete invalid rows - return an error - err := handleConversionError(err, jsonlFilePath) - return err - } - - // Convert the interface slice to string slice - var columnsWithNulls []string - for _, col := range columnsWithNullsInterface { - if col != nil { - columnsWithNulls = append(columnsWithNulls, col.(string)) - } - } - - // we have a failure - return an error with details about which columns had nulls - return NewConversionError(NewRowValidationError(failedRowCount, columnsWithNulls), failedRowCount, jsonlFilePath) -} - -// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema -// infer the schema of this chunk and compare - if they are different, return that in an error -func (w *conversionWorker) handleSchemaChangeError(err error, jsonlFilePath string) error { - schemaChangeErr := w.converter.detectSchemaChange(jsonlFilePath) - if schemaChangeErr != nil { - // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error - var e = &SchemaChangeError{} - if errors.As(schemaChangeErr, &e) { - // update err and fall through to handleConversionError - this wraps the error with additional row count info - err = e - } - } - - // just return the original error, wrapped with the row count - return handleConversionError(err, jsonlFilePath) -} - -// buildValidationQuery builds a query to copy the data from the select query to a temp table -// it then validates that the required columns are not null, removing invalid rows and returning -// the count of invalid rows and the columns with nulls -func (w *conversionWorker) buildValidationQuery(requiredColumns []string) string { - var queryBuilder strings.Builder - - // Build the validation query that: - // - Counts distinct rows that have null values in required columns - // - Lists all required columns that contain null values - queryBuilder.WriteString(`select - count(distinct rowid) as rows_with_required_nulls, -- Count unique rows with nulls in required columns - coalesce(list(distinct col), []) as required_columns_with_nulls -- List required columns that have null values, defaulting to empty list if NULL -from (`) - - // Step 3: For each required column we need to validate: - // - Create a query that selects rows where this column is null - // - Include the column name so we know which column had the null - // - UNION ALL combines all these results (faster than UNION as we don't need to deduplicate) - for i, col := range requiredColumns { - if i > 0 { - queryBuilder.WriteString(" union all\n") - } - // For each required column, create a query that: - // - Selects the rowid (to count distinct rows) - // - Includes the column name (to list which columns had nulls) - // - Only includes rows where this column is null - queryBuilder.WriteString(fmt.Sprintf(" select rowid, '%s' as col from temp_data where %s is null\n", col, col)) - } - - queryBuilder.WriteString(");") - - return queryBuilder.String() -} - -// buildNullCheckQuery builds a WHERE clause to check for null values in the specified columns -func (w *conversionWorker) buildNullCheckQuery(requiredColumns []string) string { - - // build a slice of null check conditions - conditions := make([]string, len(requiredColumns)) - for i, col := range requiredColumns { - conditions[i] = fmt.Sprintf("%s is null", col) - } - return strings.Join(conditions, " or ") -} - -// deleteInvalidRows removes rows with null values in the specified columns from the temp table -func (w *conversionWorker) deleteInvalidRows(requiredColumns []string) error { - whereClause := w.buildNullCheckQuery(requiredColumns) - query := fmt.Sprintf("delete from temp_data where %s;", whereClause) - - _, err := w.db.Exec(query) - return err -} - -// renameTempParquetFiles renames the given list of temporary parquet files to have a .parquet extension. -// note: we receive the list of files as an interface{} as that is what we read back from the db -func (w *conversionWorker) renameTempParquetFiles(files []interface{}) error { - var errList []error - for _, f := range files { - fileName := f.(string) - if strings.HasSuffix(fileName, filepaths.TempParquetExtension) { - newName := strings.TrimSuffix(fileName, filepaths.TempParquetExtension) + ".parquet" - if err := os.Rename(fileName, newName); err != nil { - errList = append(errList, fmt.Errorf("%s: %w", fileName, err)) - } - } - } - - if len(errList) > 0 { - var msg strings.Builder - msg.WriteString(fmt.Sprintf("Failed to rename %d parquet files:\n", len(errList))) - for _, err := range errList { - msg.WriteString(fmt.Sprintf(" - %v\n", err)) - } - return errors.New(msg.String()) - } - - return nil -} diff --git a/internal/parquet/conversion_worker_test.go b/internal/parquet/conversion_worker_test.go deleted file mode 100644 index ce48a9f9..00000000 --- a/internal/parquet/conversion_worker_test.go +++ /dev/null @@ -1,1521 +0,0 @@ -package parquet - -import ( - _ "github.com/marcboeker/go-duckdb/v2" -) - -//var testDb *database.DuckDb -// -//const testDir = "buildViewQuery_test_data" -// -//// we use the same path for all tests -//var jsonlFilePath string -// -//func setup() error { -// var err error -// -// // Create a temporary config directory -// tempConfigDir, err := os.MkdirTemp("", "tailpipe_test_config") -// if err != nil { -// return fmt.Errorf("error creating temp config directory: %w", err) -// } -// -// // Set the config path to our temporary directory -// viper.Set("config_path", tempConfigDir) -// -// // Initialize workspace profile with parse options -// parseOpts := []parse.ParseHclOpt{ -// parse.WithEscapeBackticks(true), -// parse.WithDisableTemplateForProperties(constants.GrokConfigProperties), -// } -// loader, err := pcmdconfig.GetWorkspaceProfileLoader[*workspace_profile.TailpipeWorkspaceProfile](parseOpts...) -// if err != nil { -// return fmt.Errorf("error creating workspace profile loader: %w", err) -// } -// config.GlobalWorkspaceProfile = loader.GetActiveWorkspaceProfile() -// if err := config.GlobalWorkspaceProfile.EnsureWorkspaceDirs(); err != nil { -// return fmt.Errorf("error ensuring workspace dirs: %w", err) -// } -// -// db, err := database.NewDuckDb(database.WithDuckDbExtensions(constants.DuckDbExtensions)) -// if err != nil { -// return fmt.Errorf("error creating duckdb: %w", err) -// } -// testDb = db -// // make tempdata directory in local folder -// // Create the directory -// err = os.MkdirAll(testDir, 0755) -// if err != nil { -// db.Close() -// return fmt.Errorf("error creating temp directory: %w", err) -// } -// -// // resolve the jsonl file path -// jsonlFilePath, err = filepath.Abs(filepath.Join(testDir, "test.jsonl")) -// return err -//} -// -//func teardown() { -// os.RemoveAll("test_data") -// if testDb != nil { -// testDb.Close() -// } -//} - -// // set the version explicitly here since version is set during build time -// // then set the app specific constants needed for the tests -// viper.Set("main.version", "0.0.1") -// cmdconfig.SetAppSpecificConstants() -// -// if err := setup(); err != nil { -// t.Fatalf("error setting up test: %s", err) -// } -// defer teardown() -// -// type args struct { -// schema *schema.ConversionSchema -// json string -// sqlColumn string -// } -// tests := []struct { -// name string -// args args -// wantQuery string -// wantData any -// }{ -// /* -// c.Type = "boolean" -// c.Type = "tinyint" -// c.Type = "smallint" -// c.Type = "integer" -// c.Type = "bigint" -// c.Type = "utinyint" -// c.Type = "usmallint" -// c.Type = "uinteger" -// c.Type = "ubigint" -// c.Type = "float" -// c.Type = "double" -// c.Type = "varchar" -// c.Type = "timestamp" -// -// c.Type = "blob" -// c.Type = "array" -// c.Type = "struct" -// c.Type = "map" -// */ -// { -// name: "struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructField", -// ColumnName: "struct_field", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "bigint"}, -// }, -// }, -// }, -// }, -// }, -// json: `{ "StructField": { "StructStringField": "StructStringVal", "StructIntField": 100 }}`, -// sqlColumn: "struct_field.struct_string_field", -// }, -// wantQuery: `select * from (select -// case -// when "StructField" is null then null -// else struct_pack( -// "struct_string_field" := "StructField"."StructStringField"::varchar, -// "struct_int_field" := "StructField"."StructIntField"::bigint -// ) -// end as "struct_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "StructField": 'struct("StructStringField" varchar, "StructIntField" bigint)' -// } -// ))`, -// wantData: []any{"StructStringVal"}, -// }, -// { -// name: "json", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "JsonField", -// ColumnName: "json_field", -// Type: "json", -// }, -// }, -// }, -// }, -// json: `{ "JsonField": { "string_field": "JsonStringVal", "int_field": 100 }}`, -// sqlColumn: "json_field.string_field", -// }, -// wantQuery: `select * from (select -// json("JsonField") as "json_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "JsonField": 'json' -// } -// ))`, -// wantData: []any{`JsonStringVal`}, -// }, -// { -// name: "struct with keyword names", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "end", -// ColumnName: "end", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "any", ColumnName: "any", Type: "varchar"}, -// }, -// }, -// }, -// }, -// }, -// json: `{ "end": { "any": "StructStringVal" }}`, -// sqlColumn: `"end"."any"`, -// }, -// wantQuery: `select * from (select -// case -// when "end" is null then null -// else struct_pack( -// "any" := "end"."any"::varchar -// ) -// end as "end" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'struct("any" varchar)' -// } -// ))`, -// wantData: []any{"StructStringVal"}, -// }, -// { -// name: "null struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "end", -// ColumnName: "end", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "any", ColumnName: "any", Type: "varchar"}, -// }, -// }, -// }, -// }, -// }, -// json: `{ }`, -// sqlColumn: `"end"."any"`, -// }, -// wantQuery: `select * from (select -// case -// when "end" is null then null -// else struct_pack( -// "any" := "end"."any"::varchar -// ) -// end as "end" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'struct("any" varchar)' -// } -// ))`, -// wantData: []any{nil}, -// }, -// { -// name: "nested struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructField", -// ColumnName: "struct_field", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStruct", -// ColumnName: "nested_struct", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStructStringField", -// ColumnName: "nested_struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// { -// SourceName: "StructStringField", -// ColumnName: "struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// }, -// }, -// }, -// json: `{ "StructField": { "NestedStruct": { "NestedStructStringField": "NestedStructStringVal" }, "StructStringField": "StructStringVal" }}`, -// sqlColumn: "struct_field.nested_struct.nested_struct_string_field", -// }, -// wantQuery: `select * from (select -// case -// when "StructField" is null then null -// else struct_pack( -// "nested_struct" := case -// when "StructField"."NestedStruct" is null then null -// else struct_pack( -// "nested_struct_string_field" := "StructField"."NestedStruct"."NestedStructStringField"::varchar -// ) -// end, -// "struct_string_field" := "StructField"."StructStringField"::varchar -// ) -// end as "struct_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "StructField": 'struct("NestedStruct" struct("NestedStructStringField" varchar), "StructStringField" varchar)' -// } -// ))`, -// wantData: []any{"NestedStructStringVal"}, -// }, -// { -// name: "null nested struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructField", -// ColumnName: "struct_field", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStruct", -// ColumnName: "nested_struct", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStructStringField", -// ColumnName: "nested_struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// { -// SourceName: "StructStringField", -// ColumnName: "struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// }, -// }, -// }, -// json: `{ "StructField": { "NestedStruct": { "NestedStructStringField": "NestedStructStringVal" }, "StructStringField": "StructStringVal" }} -//{ }`, -// sqlColumn: "struct_field.nested_struct.nested_struct_string_field", -// }, -// wantQuery: `select * from (select -// case -// when "StructField" is null then null -// else struct_pack( -// "nested_struct" := case -// when "StructField"."NestedStruct" is null then null -// else struct_pack( -// "nested_struct_string_field" := "StructField"."NestedStruct"."NestedStructStringField"::varchar -// ) -// end, -// "struct_string_field" := "StructField"."StructStringField"::varchar -// ) -// end as "struct_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "StructField": 'struct("NestedStruct" struct("NestedStructStringField" varchar), "StructStringField" varchar)' -// } -// ))`, -// wantData: []any{"NestedStructStringVal", nil}, -// }, -// { -// name: "nested struct with keyword names", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "end", -// ColumnName: "end", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "any", -// ColumnName: "any", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "for", -// ColumnName: "for", -// Type: "varchar", -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// json: `{ "end": { "any": { "for": "NestedStructStringVal" }}}`, -// sqlColumn: `"end"."any"."for"`, -// }, -// wantQuery: `select * from (select -// case -// when "end" is null then null -// else struct_pack( -// "any" := case -// when "end"."any" is null then null -// else struct_pack( -// "for" := "end"."any"."for"::varchar -// ) -// end -// ) -// end as "end" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'struct("any" struct("for" varchar))' -// } -// ))`, -// wantData: []any{"NestedStructStringVal"}, -// }, -// { -// name: "scalar types", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{"BooleanField": true, "TinyIntField": 1, "SmallIntField": 2, "IntegerField": 3, "BigIntField": 4, "UTinyIntField": 5, "USmallIntField": 6, "UIntegerField": 7, "UBigIntField": 8, "FloatField": 1.23, "DoubleField": 4.56, "VarcharField": "StringValue", "TimestampField": "2024-01-01T00:00:00Z"}`, -// sqlColumn: "varchar_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{"StringValue"}, -// }, -// { -// name: "scalar types - reserved names", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "end", ColumnName: "end", Type: "boolean"}, -// {SourceName: "for", ColumnName: "for", Type: "tinyint"}, -// }, -// }, -// }, -// json: `{"end": true, "for": 1}`, -// sqlColumn: `"end"`, -// }, -// wantQuery: `select * from (select -// "end" as "end", -// "for" as "for" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'boolean', -// "for": 'tinyint' -// } -// ))`, -// wantData: []any{true}, -// }, -// { -// name: "scalar types - missing some data", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{"BooleanField": true}`, -// sqlColumn: "boolean_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{true}, -// }, -// { -// name: "scalar types - some rows missing some data", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{"BooleanField": true} -//{"TinyIntField": 1} -//{"TinyIntField": 1, "BooleanField": true}`, -// sqlColumn: "boolean_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{true, nil, true}, -// }, -// { -// name: "scalar types, missing all data", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{}`, -// sqlColumn: "varchar_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{nil}, -// }, -// { -// name: "array types", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanArrayField", ColumnName: "boolean_array_field", Type: "boolean[]"}, -// {SourceName: "TinyIntArrayField", ColumnName: "tinyint_array_field", Type: "tinyint[]"}, -// {SourceName: "SmallIntArrayField", ColumnName: "smallint_array_field", Type: "smallint[]"}, -// {SourceName: "IntegerArrayField", ColumnName: "integer_array_field", Type: "integer[]"}, -// {SourceName: "BigIntArrayField", ColumnName: "bigint_array_field", Type: "bigint[]"}, -// {SourceName: "UTinyIntArrayField", ColumnName: "utinyint_array_field", Type: "utinyint[]"}, -// {SourceName: "USmallIntArrayField", ColumnName: "usmallint_array_field", Type: "usmallint[]"}, -// {SourceName: "UIntegerArrayField", ColumnName: "uinteger_array_field", Type: "uinteger[]"}, -// {SourceName: "UBigIntArrayField", ColumnName: "ubigint_array_field", Type: "ubigint[]"}, -// {SourceName: "FloatArrayField", ColumnName: "float_array_field", Type: "float[]"}, -// {SourceName: "DoubleArrayField", ColumnName: "double_array_field", Type: "double[]"}, -// {SourceName: "VarcharArrayField", ColumnName: "varchar_array_field", Type: "varchar[]"}, -// {SourceName: "TimestampArrayField", ColumnName: "timestamp_array_field", Type: "timestamp[]"}, -// }, -// }, -// }, -// json: `{"BooleanArrayField": [true, false], "TinyIntArrayField": [1, 2], "SmallIntArrayField": [2, 3], "IntegerArrayField": [3, 4], "BigIntArrayField": [4, 5], "UTinyIntArrayField": [5, 6], "USmallIntArrayField": [6, 7], "UIntegerArrayField": [7, 8], "UBigIntArrayField": [8, 9], "FloatArrayField": [1.23, 2.34], "DoubleArrayField": [4.56, 5.67], "VarcharArrayField": ["StringValue1", "StringValue2"], "TimestampArrayField": ["2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"]}`, -// sqlColumn: "boolean_array_field", -// }, -// wantQuery: `select * from (select -// "BooleanArrayField" as "boolean_array_field", -// "TinyIntArrayField" as "tinyint_array_field", -// "SmallIntArrayField" as "smallint_array_field", -// "IntegerArrayField" as "integer_array_field", -// "BigIntArrayField" as "bigint_array_field", -// "UTinyIntArrayField" as "utinyint_array_field", -// "USmallIntArrayField" as "usmallint_array_field", -// "UIntegerArrayField" as "uinteger_array_field", -// "UBigIntArrayField" as "ubigint_array_field", -// "FloatArrayField" as "float_array_field", -// "DoubleArrayField" as "double_array_field", -// "VarcharArrayField" as "varchar_array_field", -// "TimestampArrayField" as "timestamp_array_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanArrayField": 'boolean[]', -// "TinyIntArrayField": 'tinyint[]', -// "SmallIntArrayField": 'smallint[]', -// "IntegerArrayField": 'integer[]', -// "BigIntArrayField": 'bigint[]', -// "UTinyIntArrayField": 'utinyint[]', -// "USmallIntArrayField": 'usmallint[]', -// "UIntegerArrayField": 'uinteger[]', -// "UBigIntArrayField": 'ubigint[]', -// "FloatArrayField": 'float[]', -// "DoubleArrayField": 'double[]', -// "VarcharArrayField": 'varchar[]', -// "TimestampArrayField": 'timestamp[]' -// } -// ))`, -// wantData: []any{[]any{true, false}}, -// }, -// { -// name: "array of simple structs", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}]}`, -// sqlColumn: "struct_array_field[1].struct_string_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{"StringValue1"}, -// }, -// -// // TODO struct arrays are not supported yet -// // in fact one level of struct array field does work, but not nested struct arrays so for -// // now all struct arrays are treated as json -// // { -// // name: "struct with struct array field", -// // args: args{ -// // conversionSchema: &conversionSchema.TableSchema{ -// // Columns: []*conversionSchema.ColumnSchema{ -// // { -// // SourceName: "StructWithArrayField", -// // ColumnName: "struct_with_array_field", -// // Type: "struct", -// // StructFields: []*conversionSchema.ColumnSchema{ -// // {SourceName: "StructArrayField", -// // ColumnName: "struct_array_field", -// // Type: "struct[]", -// // StructFields: []*conversionSchema.ColumnSchema{ -// // {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "VARCHAR"}, -// // {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "INTEGER"}, -// // },}, -// // }, -// // }, -// // }, -// // }, -// // json: `{"StructWithArrayField": {"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}]}}`, -// // sqlColumn: "struct_with_array_field.struct_array_field[1].struct_string_field", -// // }, -// // wantQuery: `WITH raw as ( -// // SELECT -// // row_number() OVER () as rowid, -// // "StructArrayField" as "struct_array_field" -// // FROM -// // read_ndjson( -// // '%s', -// // columns = { -// // "StructArrayField": 'struct("StructStringField" VARCHAR, "StructIntField" INTEGER)[]' -// // } -// // ) -// //), unnest_struct_array_field as ( -// // SELECT -// // rowid, -// // UNNEST(COALESCE("struct_array_field", ARRAY[]::struct("StructStringField" VARCHAR, "StructIntField" INTEGER)[])::struct("StructStringField" VARCHAR, "StructIntField" INTEGER)[]) as struct_array_field -// // FROM -// // raw -// //), rebuild_unnest_struct_array_field as ( -// // SELECT -// // rowid, -// // struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// // struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// // FROM -// // unnest_struct_array_field -// //), grouped_unnest_struct_array_field as ( -// // SELECT -// // rowid, -// // array_agg(struct_pack( -// // struct_string_field := StructArrayField_StructStringField::VARCHAR, -// // struct_int_field := StructArrayField_StructIntField::INTEGER -// // )) as struct_array_field -// // FROM -// // rebuild_unnest_struct_array_field -// // group by -// // rowid -// //) -// //SELECT -// // COALESCE(joined_struct_array_field.struct_array_field, NULL) as struct_array_field -// //FROM -// // raw -// //left join -// // grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// // wantData: []any{"StringValue1"}, -// // }, -// -// { -// name: "array of simple structs plus other fields", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// {SourceName: "IntField", ColumnName: "int_field", Type: "integer"}, -// {SourceName: "StringField", ColumnName: "string_field", Type: "varchar"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// { -// SourceName: "IntArrayField", -// ColumnName: "int_array_field", -// Type: "integer[]", -// }, -// { -// SourceName: "StringArrayField", -// ColumnName: "string_array_field", -// Type: "varchar[]", -// }, -// { -// SourceName: "FloatArrayField", -// ColumnName: "float_array_field", -// Type: "float[]", -// }, -// { -// SourceName: "BooleanArrayField", -// ColumnName: "boolean_array_field", -// Type: "boolean[]", -// }, -// }, -// }, -// }, -// -// json: `{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}], "IntField": 10, "StringField": "SampleString", "FloatField": 10.5, "BooleanField": true, "IntArrayField": [1, 2, 3], "StringArrayField": ["String1", "String2"], "FloatArrayField": [1.1, 2.2, 3.3], "BooleanArrayField": [true, false, true]}`, -// // NOTE: arrays are 1-based -// sqlColumn: "struct_array_field[1].struct_string_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field", -// "IntField" as "int_field", -// "StringField" as "string_field", -// "FloatField" as "float_field", -// "BooleanField" as "boolean_field", -// "IntArrayField" as "int_array_field", -// "StringArrayField" as "string_array_field", -// "FloatArrayField" as "float_array_field", -// "BooleanArrayField" as "boolean_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]', -// "IntField": 'integer', -// "StringField": 'varchar', -// "FloatField": 'float', -// "BooleanField": 'boolean', -// "IntArrayField": 'integer[]', -// "StringArrayField": 'varchar[]', -// "FloatArrayField": 'float[]', -// "BooleanArrayField": 'boolean[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field, -// raw.int_field, -// raw.string_field, -// raw.float_field, -// raw.boolean_field, -// raw.int_array_field, -// raw.string_array_field, -// raw.float_array_field, -// raw.boolean_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{"StringValue1"}, -// }, -// { -// name: "null array of simple structs plus other fields", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// {SourceName: "IntField", ColumnName: "int_field", Type: "integer"}, -// {SourceName: "StringField", ColumnName: "string_field", Type: "varchar"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// { -// SourceName: "IntArrayField", -// ColumnName: "int_array_field", -// Type: "integer[]", -// }, -// { -// SourceName: "StringArrayField", -// ColumnName: "string_array_field", -// Type: "varchar[]", -// }, -// { -// SourceName: "FloatArrayField", -// ColumnName: "float_array_field", -// Type: "float[]", -// }, -// { -// SourceName: "BooleanArrayField", -// ColumnName: "boolean_array_field", -// Type: "boolean[]", -// }, -// }, -// }, -// }, -// -// json: `{"StructArrayField": null, "IntField": 10, "StringField": "SampleString", "FloatField": 10.5, "BooleanField": true, "IntArrayField": [1, 2, 3], "StringArrayField": ["String1", "String2"], "FloatArrayField": [1.1, 2.2, 3.3], "BooleanArrayField": [true, false, true]}`, -// sqlColumn: "int_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field", -// "IntField" as "int_field", -// "StringField" as "string_field", -// "FloatField" as "float_field", -// "BooleanField" as "boolean_field", -// "IntArrayField" as "int_array_field", -// "StringArrayField" as "string_array_field", -// "FloatArrayField" as "float_array_field", -// "BooleanArrayField" as "boolean_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]', -// "IntField": 'integer', -// "StringField": 'varchar', -// "FloatField": 'float', -// "BooleanField": 'boolean', -// "IntArrayField": 'integer[]', -// "StringArrayField": 'varchar[]', -// "FloatArrayField": 'float[]', -// "BooleanArrayField": 'boolean[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field, -// raw.int_field, -// raw.string_field, -// raw.float_field, -// raw.boolean_field, -// raw.int_array_field, -// raw.string_array_field, -// raw.float_array_field, -// raw.boolean_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{int32(10)}, -// }, -// { -// name: "array of simple structs with null value", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": null}`, -// sqlColumn: "struct_array_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{nil}, -// }, -// { -// name: "array of simple structs with null value and non null value", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": null} -//{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}]}`, -// sqlColumn: "struct_array_field[1].struct_string_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// //wantData: []any{nil, "StringValue1"}, -// // NOTE: ordering is not guaranteed -// wantData: []any{"StringValue1", nil}, -// }, -// { -// name: "2 arrays of simple structs", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// { -// SourceName: "StructArrayField2", -// ColumnName: "struct_array_field2", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField2", ColumnName: "struct_string_field2", Type: "varchar"}, -// {SourceName: "StructIntField2", ColumnName: "struct_int_field2", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}], "StructArrayField2": [{"StructStringField2": "StringValue100", "StructIntField2": 100}, {"StructStringField2": "StringValue200", "StructIntField2": 200}]}`, -// sqlColumn: "struct_array_field2[1].struct_string_field2", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field", -// "StructArrayField2" as "struct_array_field2" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]', -// "StructArrayField2": 'struct("StructStringField2" varchar, "StructIntField2" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//), unnest_struct_array_field2 as ( -// select -// rowid, -// unnest(coalesce("struct_array_field2", array[]::struct("StructStringField2" varchar, "StructIntField2" integer)[])::struct("StructStringField2" varchar, "StructIntField2" integer)[]) as struct_array_field2 -// from -// raw -//), rebuild_unnest_struct_array_field2 as ( -// select -// rowid, -// struct_array_field2->>'StructStringField2' as StructArrayField2_StructStringField2, -// struct_array_field2->>'StructIntField2' as StructArrayField2_StructIntField2 -// from -// unnest_struct_array_field2 -//), grouped_unnest_struct_array_field2 as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field2 := StructArrayField2_StructStringField2::varchar, -// struct_int_field2 := StructArrayField2_StructIntField2::integer -// )) as struct_array_field2 -// from -// rebuild_unnest_struct_array_field2 -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field, -// coalesce(joined_struct_array_field2.struct_array_field2, null) as struct_array_field2 -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid -//left join -// grouped_unnest_struct_array_field2 joined_struct_array_field2 on raw.rowid = joined_struct_array_field2.rowid`, -// wantData: []any{"StringValue100"}, -// }, -// // TODO #parquet https://github.com/turbot/tailpipe/issues/new -// // { -// // name: "map types", -// // args: args{ -// // conversionSchema: &conversionSchema.TableSchema{ -// // Columns: []*conversionSchema.ColumnSchema{ -// // {SourceName: "BooleanMapField", ColumnName: "boolean_map_field", Type: "map"}, -// // {SourceName: "TinyIntMapField", ColumnName: "tinyint_map_field", Type: "map"}, -// // {SourceName: "SmallIntMapField", ColumnName: "smallint_map_field", Type: "map"}, -// // {SourceName: "IntegerMapField", ColumnName: "integer_map_field", Type: "map"}, -// // {SourceName: "BigIntMapField", ColumnName: "bigint_map_field", Type: "map"}, -// // {SourceName: "FloatMapField", ColumnName: "float_map_field", Type: "map"}, -// // {SourceName: "DoubleMapField", ColumnName: "double_map_field", Type: "map"}, -// // {SourceName: "VarcharMapField", ColumnName: "varchar_map_field", Type: "map"}, -// // {SourceName: "TimestampMapField", ColumnName: "timestamp_map_field", Type: "map"}, -// // }, -// // }, -// // json: `{"BooleanMapField": {"key1": true, "key2": false}, "TinyIntMapField": {"key1": 1, "key2": 2}, "SmallIntMapField": {"key1": 2, "key2": 3}, "IntegerMapField": {"key1": 3, "key2": 4}, "BigIntMapField": {"key1": 4, "key2": 5}, "FloatMapField": {"key1": 1.23, "key2": 2.34}, "DoubleMapField": {"key1": 4.56, "key2": 5.67}, "VarcharMapField": {"key1": "StringValue1", "key2": "StringValue2"}, "TimestampMapField": {"key1": "2024-01-01T00:00:00Z", "key2": "2024-01-02T00:00:00Z"}}`, -// // sqlColumn: "boolean_map_field", -// // }, -// // wantQuery: `select -// // json_extract(json, '$.BooleanMapField')::map(varchar, boolean> as boolean_map_field, -// // json_extract(json, '$.TinyIntMapField')::map(varchar, tinyint> as tinyint_map_field, -// // json_extract(json, '$.SmallIntMapField')::map(varchar, smallint) as smallint_map_field, -// // json_extract(json, '$.IntegerMapField')::map(varchar, integer) as integer_map_field, -// // json_extract(json, '$.BigIntMapField')::map(varchar, bigint) as bigint_map_field, -// // json_extract(json, '$.FloatMapField')::map(varchar, float) as float_map_field, -// // json_extract(json, '$.DoubleMapField')::map(varchar, double) as double_map_field, -// // json_extract(json, '$.VarcharMapField')::map(varchar, varchar) as varchar_map_field, -// // json_extract(json, '$.TimestampMapField')::map(varchar, timestamp) as timestamp_map_field -// //from read_json_auto('%s', format='newline_delimited')`, jsonlFilePath), -// // wantData: map[string]bool{"key1": true, "key2": false}, -// // }, -// } -// -// defer os.RemoveAll("test_data") -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// conversionSchema := schema.NewConversionSchema(&tt.args.schema.TableSchema) -// query := buildReadJsonQueryFormat(conversionSchema) -// -// // first check the quey is as expected -// if query != tt.wantQuery { -// t.Errorf("buildReadJsonQueryFormat(), got:\n%s\nwant:\n%s", query, tt.wantQuery) -// } -// -// gotData, err := executeQuery(t, query, tt.args.json, tt.args.sqlColumn) -// if err != nil { -// t.Errorf("error executing query: %s", err) -// } else if !reflect.DeepEqual(gotData, tt.wantData) { -// t.Errorf("buildReadJsonQueryFormat() query returned %v, want %v", gotData, tt.wantData) -// } -// }) -// } -//} -// -//func executeQuery(t *testing.T, queryFormat, json, sqlColumn string) (any, error) { -// -// // now verify the query runs -// // copy json to a jsonl file -// err := createJSONLFile(json) -// if err != nil { -// t.Fatalf("error creating jsonl file: %s", err) -// } -// defer os.Remove(jsonlFilePath) -// -// // render query with the file path -// query := fmt.Sprintf(queryFormat, jsonlFilePath) -// -// // get the data -// var data []any -// -// // execute in duckdb -// // build select queryz -// testQuery := fmt.Sprintf("select %s from (%s)", sqlColumn, query) -// rows, err := testDb.Query(testQuery) -// -// if err != nil { -// return nil, fmt.Errorf("error executing query: %w", err) -// } -// // Iterate over the results -// for rows.Next() { -// var d any -// -// if err := rows.Scan(&d); err != nil { -// return nil, fmt.Errorf("error scanning data: %w", err) -// } -// data = append(data, d) -// } -// -// return data, nil -//} -// -//func createJSONLFile(json string) error { -// // remove just in case -// os.Remove(jsonlFilePath) -// jsonlFile, err := os.Create(jsonlFilePath) -// if err != nil { -// return fmt.Errorf("error creating jsonl file: %w", err) -// } -// _, err = jsonlFile.WriteString(json) -// if err != nil { -// return fmt.Errorf("error writing to jsonl file: %w", err) -// } -// // close the file -// err = jsonlFile.Close() -// if err != nil { -// return fmt.Errorf("error closing jsonl file: %w", err) -// } -// return err -//} -// -// TODO KAI re-add -// -//func TestBuildValidationQuery(t *testing.T) { -// testCases := []struct { -// name string -// selectQuery string -// columnsToValidate []string -// expectedQuery string -// }{ -// { -// name: "single column", -// selectQuery: "select * from source", -// columnsToValidate: []string{"name"}, -// expectedQuery: `drop table if exists temp_data; -//create temp table temp_data as select * from source; -//select -// count(*) as total_rows, -// list(distinct col) as columns_with_nulls -//from ( -// select 'name' as col from temp_data where name is null -//) -//`, -// }, -// { -// name: "multiple columns", -// selectQuery: "select * from source", -// columnsToValidate: []string{"name", "email", "age"}, -// expectedQuery: `drop table if exists temp_data; -//create temp table temp_data as select * from source; -//select -// count(*) as total_rows, -// list(distinct col) as columns_with_nulls -//from ( -// select 'name' as col from temp_data where name is null -// union all -// select 'email' as col from temp_data where email is null -// union all -// select 'age' as col from temp_data where age is null -//) -//`, -// }, -// { -// name: "no columns", -// selectQuery: "select * from source", -// columnsToValidate: []string{}, -// expectedQuery: `drop table if exists temp_data; -//create temp table temp_data as select * from source; -//select -// count(*) as total_rows, -// list(distinct col) as columns_with_nulls -//from ( -//) -//`, -// }, -// } -// -// for _, tc := range testCases { -// t.Run(tc.name, func(t *testing.T) { -// worker := &conversionWorker{} -// actualQuery := worker.buildValidationQuery(tc.columnsToValidate) -// assert.Equal(t, tc.expectedQuery, actualQuery) -// }) -// } -//} diff --git a/internal/parquet/convertor.go b/internal/parquet/convertor.go index a31aadb4..cfe1bc3f 100644 --- a/internal/parquet/convertor.go +++ b/internal/parquet/convertor.go @@ -4,43 +4,45 @@ import ( "context" "errors" "fmt" - "github.com/spf13/viper" - pconstants "github.com/turbot/pipe-fittings/v2/constants" "log/slog" "sync" "sync/atomic" "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/database" ) -const defaultParquetWorkerCount = 5 -const chunkBufferLength = 1000 +// TODO #DL +// - think about max memory https://github.com/turbot/tailpipe/issues/478 +// - validation https://github.com/turbot/tailpipe/issues/479 -// the minimum memory to assign to each worker - -const minWorkerMemoryMb = 512 +const chunkBufferLength = 1000 // Converter struct executes all the conversions for a single collection -// it therefore has a unique execution id, and will potentially convert of multiple JSONL files +// it therefore has a unique execution executionId, and will potentially convert of multiple JSONL files // each file is assumed to have the filename format _.jsonl // so when new input files are available, we simply store the chunk number type Converter struct { - // the execution id - id string - - // the file chunks numbers available to process - chunks []int32 - chunkLock sync.Mutex - chunkSignal *sync.Cond - // the channel to send execution to the workers - jobChan chan *parquetJob + // the execution executionId + executionId string + + // the file scheduledChunks numbers available to process + scheduledChunks []int32 + + scheduleLock sync.Mutex + processLock sync.Mutex + // waitGroup to track job completion + // this is incremented when a file is scheduled and decremented when the file is processed wg sync.WaitGroup - // the cancel function for the context used to manage the job - cancel context.CancelFunc - // the number of chunks processed so far - completionCount int32 + // the number of jsonl files processed so far + //fileCount int32 + + // the number of conversions executed + //conversionCount int32 + // the number of rows written rowCount int64 // the number of rows which were NOT converted due to conversion errors encountered @@ -50,24 +52,23 @@ type Converter struct { sourceDir string // the dest file location destDir string - // helper to provide unique file roots - fileRootProvider *FileRootProvider - // the format string for the query to read the JSON chunks - thids is reused for all chunks, + // the format string for the query to read the JSON scheduledChunks - this is reused for all scheduledChunks, // with just the filename being added when the query is executed readJsonQueryFormat string // the table conversionSchema - populated when the first chunk arrives if the conversionSchema is not already complete conversionSchema *schema.ConversionSchema - // the source schema - used to build the conversionSchema + // the source schema - which may be partial - used to build the full conversionSchema + // we store separately for the purpose of change detection tableSchema *schema.TableSchema // viewQueryOnce ensures the schema inference only happens once for the first chunk, - // even if multiple chunks arrive concurrently. Combined with schemaWg, this ensures - // all subsequent chunks wait for the initial schema inference to complete before proceeding. + // even if multiple scheduledChunks arrive concurrently. Combined with schemaWg, this ensures + // all subsequent scheduledChunks wait for the initial schema inference to complete before proceeding. viewQueryOnce sync.Once - // schemaWg is used to block processing of subsequent chunks until the initial - // schema inference is complete. This ensures all chunks wait for the schema + // schemaWg is used to block processing of subsequent scheduledChunks until the initial + // schema inference is complete. This ensures all scheduledChunks wait for the schema // to be fully initialized before proceeding with their processing. schemaWg sync.WaitGroup @@ -75,12 +76,12 @@ type Converter struct { Partition *config.Partition // func which we call with updated row count statusFunc func(int64, int64, ...error) - // pluginPopulatesTpIndex indicates if the plugin populates the tp_index column (which is no longer required - // - tp_index values set by the plugin will be ignored) - pluginPopulatesTpIndex bool + + // the DuckDB database connection - this must have a ducklake attachment + db *database.DuckDb } -func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executionId string, partition *config.Partition, sourceDir string, tableSchema *schema.TableSchema, statusFunc func(int64, int64, ...error)) (*Converter, error) { +func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executionId string, partition *config.Partition, sourceDir string, tableSchema *schema.TableSchema, statusFunc func(int64, int64, ...error), db *database.DuckDb) (*Converter, error) { // get the data dir - this will already have been created by the config loader destDir := config.GlobalWorkspaceProfile.GetDataDir() @@ -88,73 +89,109 @@ func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executi tableSchema.NormaliseColumnTypes() w := &Converter{ - id: executionId, - chunks: make([]int32, 0, chunkBufferLength), // Pre-allocate reasonable capacity - Partition: partition, - cancel: cancel, - sourceDir: sourceDir, - destDir: destDir, - tableSchema: tableSchema, - statusFunc: statusFunc, - fileRootProvider: &FileRootProvider{}, - } - // create the condition variable using the same lock - w.chunkSignal = sync.NewCond(&w.chunkLock) - - // initialise the workers - if err := w.createWorkers(ctx); err != nil { - return nil, fmt.Errorf("failed to create workers: %w", err) + executionId: executionId, + scheduledChunks: make([]int32, 0, chunkBufferLength), // Pre-allocate reasonable capacity + Partition: partition, + sourceDir: sourceDir, + destDir: destDir, + tableSchema: tableSchema, + statusFunc: statusFunc, + db: db, } - // start the goroutine to schedule the jobs - go w.scheduler(ctx) // done return w, nil } -func (w *Converter) Close() { - slog.Info("closing Converter") - // close the close channel to signal to the job schedulers to exit - w.cancel() -} - -// AddChunk adds a new chunk to the list of chunks to be processed +// AddChunk adds a new chunk to the list of scheduledChunks to be processed // if this is the first chunk, determine if we have a full conversionSchema yet and if not infer from the chunk -// signal the scheduler that `chunks are available +// signal the scheduler that `scheduledChunks are available func (w *Converter) AddChunk(executionId string, chunk int32) error { var err error + + // wait on the schemaWg to ensure that schema inference is complete before processing the chunk w.schemaWg.Wait() // Execute schema inference exactly once for the first chunk. - // The WaitGroup ensures all subsequent chunks wait for this to complete. + // The WaitGroup ensures all subsequent scheduledChunks wait for this to complete. // If schema inference fails, the error is captured and returned to the caller. w.viewQueryOnce.Do(func() { - w.schemaWg.Add(1) - defer w.schemaWg.Done() - if err = w.buildConversionSchema(executionId, chunk); err != nil { - // err will be returned by the parent function - return - } - w.readJsonQueryFormat = w.buildReadJsonQueryFormat() + err = w.onFirstChunk(executionId, chunk) }) if err != nil { return fmt.Errorf("failed to infer schema: %w", err) } - w.chunkLock.Lock() - w.chunks = append(w.chunks, chunk) - w.chunkLock.Unlock() + // lock the schedule lock to ensure that we can safely add to the scheduled scheduledChunks + w.scheduleLock.Lock() + defer w.scheduleLock.Unlock() + + // add to scheduled scheduledChunks + w.scheduledChunks = append(w.scheduledChunks, chunk) + // increment the wait group to track the scheduled chunk w.wg.Add(1) - // Signal that new chunk is available - // Using Signal instead of Broadcast as only one worker needs to wake up - w.chunkSignal.Signal() + // ok try to lock the process lock - that will fail if another process is running + if w.processLock.TryLock() { + // so we have the process lock AND the schedule lock + // store the chunk to process + + // move the scheduled chunks to the chunks to process + // (scheduledChunks may be empty, in which case we will break out of the loop) + chunksToProcess := w.getChunksToProcess() + + // and process = we now have the process lock + // NOTE: process chunks will keep processing as long as there are scheduledChunks to process, including + // scheduledChunks that were scheduled while we were processing + go w.processChunks(chunksToProcess) + } + + return nil +} + +// getChunksToProcess returns the chunks to process, up to a maximum of maxChunksToProcess +// it also trims the scheduledChunks to remove the processed chunks +func (w *Converter) getChunksToProcess() []int32 { + // TODO #DL do we even need this https://github.com/turbot/tailpipe/issues/523 + const maxChunksToProcess = 2000 + var chunksToProcess []int32 + if len(w.scheduledChunks) > maxChunksToProcess { + slog.Debug("Converter.AddChunk limiting chunks to process to max", "scheduledChunks", len(w.scheduledChunks), "maxChunksToProcess", maxChunksToProcess) + chunksToProcess = w.scheduledChunks[:maxChunksToProcess] + // trim the scheduled chunks to remove the processed chunks + w.scheduledChunks = w.scheduledChunks[maxChunksToProcess:] + } else { + slog.Debug("Converter.AddChunk processing all scheduled chunks", "scheduledChunks", len(w.scheduledChunks)) + chunksToProcess = w.scheduledChunks + // clear the scheduled chunks + w.scheduledChunks = nil + } + return chunksToProcess +} + +// onFirstChunk is called when the first chunk is added to the converter +// it is responsible for building the conversion schema if it does not already exist +// (we must wait for the first chunk as we may need to infer the schema from the chunk data) +// once the conversion schema is built, we can create the DuckDB table for this partition and build the +// read query format string that we will use to read the JSON data from the file +func (w *Converter) onFirstChunk(executionId string, chunk int32) error { + w.schemaWg.Add(1) + defer w.schemaWg.Done() + if err := w.buildConversionSchema(executionId, chunk); err != nil { + // err will be returned by the parent function + return err + } + // create the DuckDB table fpr this partition if it does not already exist + if err := w.ensureDuckLakeTable(w.Partition.TableName); err != nil { + return fmt.Errorf("failed to create DuckDB table: %w", err) + } + w.readJsonQueryFormat = buildReadJsonQueryFormat(w.conversionSchema, w.Partition) return nil } // WaitForConversions waits for all jobs to be processed or for the context to be cancelled -func (w *Converter) WaitForConversions(ctx context.Context) { +func (w *Converter) WaitForConversions(ctx context.Context) error { slog.Info("Converter.WaitForConversions - waiting for all jobs to be processed or context to be cancelled.") // wait for the wait group within a goroutine so we can also check the context done := make(chan struct{}) @@ -166,67 +203,14 @@ func (w *Converter) WaitForConversions(ctx context.Context) { select { case <-ctx.Done(): slog.Info("WaitForConversions - context cancelled.") + return ctx.Err() case <-done: slog.Info("WaitForConversions - all jobs processed.") + return nil } } -// waitForSignal waits for the condition signal or context cancellation -// returns true if context was cancelled -func (w *Converter) waitForSignal(ctx context.Context) bool { - w.chunkLock.Lock() - defer w.chunkLock.Unlock() - - select { - case <-ctx.Done(): - return true - default: - w.chunkSignal.Wait() - return false - } -} - -// the scheduler is responsible for sending jobs to the workere -// it listens for signals on the chunkWrittenSignal channel and enqueues jobs when they arrive -func (w *Converter) scheduler(ctx context.Context) { - defer close(w.jobChan) - - for { - chunk, ok := w.getNextChunk() - if !ok { - if w.waitForSignal(ctx) { - slog.Debug("scheduler shutting down due to context cancellation") - return - } - continue - } - - select { - case <-ctx.Done(): - return - case w.jobChan <- &parquetJob{chunkNumber: chunk}: - slog.Debug("scheduler - sent job to worker", "chunk", chunk) - } - } -} - -// TODO currently this _does not_ process the chunks in order as this is more efficient from a buffer handling perspective -// however we may decide we wish to process chunks in order in the interest of restartability/tracking progress -func (w *Converter) getNextChunk() (int32, bool) { - w.chunkLock.Lock() - defer w.chunkLock.Unlock() - - if len(w.chunks) == 0 { - return 0, false - } - - // Take from end - more efficient as it avoids shifting elements - lastIdx := len(w.chunks) - 1 - chunk := w.chunks[lastIdx] - w.chunks = w.chunks[:lastIdx] - return chunk, true -} - +//nolint:unused // we will use this once we re-add conversion error handling func (w *Converter) addJobErrors(errorList ...error) { var failedRowCount int64 @@ -250,56 +234,32 @@ func (w *Converter) updateRowCount(count int64) { } // updateCompletionCount atomically increments the completion count -func (w *Converter) updateCompletionCount(count int32) { - atomic.AddInt32(&w.completionCount, count) -} - -// createWorkers initializes and starts parquet conversion workers based on configured memory limits -// It calculates the optimal number of workers and memory allocation per worker using the following logic: -// - If no memory limit is set, uses defaultParquetWorkerCount workers with defaultWorkerMemoryMb per worker -// - If memory limit is set, ensures each worker gets at least minWorkerMemoryMb, reducing worker count if needed -// - Reserves memory for the main process by dividing total memory by (workerCount + 1) -// - Creates and starts the calculated number of workers, each with their allocated memory -// Returns error if worker creation fails -func (w *Converter) createWorkers(ctx context.Context) error { - // determine the number of workers to start - // see if there was a memory limit - maxMemoryMb := viper.GetInt(pconstants.ArgMemoryMaxMb) - memoryPerWorkerMb := maxMemoryMb / defaultParquetWorkerCount - - workerCount := defaultParquetWorkerCount - if maxMemoryMb > 0 { - // calculate memory per worker and adjust worker count if needed - // - reserve memory for main process by dividing maxMemory by (workerCount + 1) - // - if calculated memory per worker is less than minimum required: - // - reduce worker count to ensure each worker has minimum required memory - // - ensure at least 1 worker remains - - if memoryPerWorkerMb < minWorkerMemoryMb { - // reduce worker count to ensure minimum memory per worker - workerCount = maxMemoryMb / minWorkerMemoryMb - if workerCount < 1 { - workerCount = 1 - } - memoryPerWorkerMb = maxMemoryMb / workerCount - if memoryPerWorkerMb < minWorkerMemoryMb { - return fmt.Errorf("not enough memory available for workers - require at least %d for a single worker", minWorkerMemoryMb) - } - } - slog.Info("Worker memory allocation", "workerCount", workerCount, "memoryPerWorkerMb", memoryPerWorkerMb, "maxMemoryMb", maxMemoryMb, "minWorkerMemoryMb", minWorkerMemoryMb) - } - - // create the job channel - w.jobChan = make(chan *parquetJob, workerCount*2) - - // start the workers - for i := 0; i < workerCount; i++ { - wk, err := newConversionWorker(w, memoryPerWorkerMb, i) - if err != nil { - return fmt.Errorf("failed to create worker: %w", err) - } - // start the worker - go wk.start(ctx) - } - return nil -} +//func (w *Converter) updateCompletionCount(fileCount, conversionCount int32) { +// atomic.AddInt32(&w.fileCount, fileCount) +// atomic.AddInt32(&w.conversionCount, conversionCount) +//} +// +//func (w *Converter) GetCompletionCount() int32 { +// return atomic.LoadInt32(&w.fileCount) +//} + +// TODO #DL think about memory +// https://github.com/turbot/tailpipe/issues/478 + +//func (w *conversionWorker) forceMemoryRelease() error { +// // we need to flush the memory to release it - do this by setting a low memory limit then the full one +// // NOTE: do not set the memory to zero as we have temp table data +// const minMemoryMb = 64 +// +// // Set to minimum memory - note the use of ? parameter +// if _, err := w.db.Exec("set max_memory = ? || 'MB';", minMemoryMb); err != nil { +// return fmt.Errorf("memory flush failed: %w", err) +// } +// +// // Reset to configured memory limit +// if _, err := w.db.Exec("set max_memory = ? || 'MB';", w.maxMemoryMb); err != nil { +// return fmt.Errorf("memory reset failed: %w", err) +// } +// return nil +// +//} diff --git a/internal/parquet/convertor_convert.go b/internal/parquet/convertor_convert.go new file mode 100644 index 00000000..a47821bb --- /dev/null +++ b/internal/parquet/convertor_convert.go @@ -0,0 +1,265 @@ +package parquet + +import ( + "errors" + "fmt" + "github.com/turbot/pipe-fittings/v2/utils" + "log" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/marcboeker/go-duckdb/v2" + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +func (w *Converter) processChunks(chunksToProcess []int32) { + // note we ALREADY HAVE THE PROCESS LOCK - be sure to release it when we are done + defer w.processLock.Unlock() + + for len(chunksToProcess) > 0 { + // build a list of filenames to process + filenamesToProcess, err := w.chunkNumbersToFilenames(chunksToProcess) + if err != nil { + // failed to convert these files - decrement the wait group + w.wg.Add(len(filenamesToProcess) * -1) + + // TODO #DL re-add error handling + // https://github.com/turbot/tailpipe/issues/480 + slog.Error("Error processing chunks", "error", err) + // store the failed conversion + //w.failedConversions = append(w.failedConversions, failedConversion{ + // filenames: filenamesToProcess, + // error: err, + //}, + //) + // just carry on + } + + // execute conversion query for the chunks + err = w.insertBatchIntoDuckLake(filenamesToProcess) + if err != nil { + // TODO #DL re-add error handling + // https://github.com/turbot/tailpipe/issues/480 + + // NOTE: the wait group will already have been decremented by insertBatchIntoDuckLake + // so we do not need to decrement it again here + + slog.Error("Error processing chunk", "filenames", filenamesToProcess, "error", err) + // store the failed conversion + //w.failedConversions = append(w.failedConversions, failedConversion{ + // filenames: filenamesToProcess, + // error: err, + //}, + //) + // just carry on + } + // delete the files after processing + for _, filename := range filenamesToProcess { + if err := os.Remove(filename); err != nil { + slog.Error("Failed to delete file after processing", "file", filename, "error", err) + } + } + + // now determine if there are more chunks to process + w.scheduleLock.Lock() + + // now get next chunks to process + chunksToProcess = w.getChunksToProcess() + + w.scheduleLock.Unlock() + } + + // if we get here, we have processed all scheduled chunks (but more may come later + log.Print("BatchProcessor: all scheduled chunks processed for execution") +} + +func (w *Converter) chunkNumbersToFilenames(chunks []int32) ([]string, error) { + var filenames = make([]string, len(chunks)) + var missingFiles []string + for i, chunkNumber := range chunks { + // build the source filename + jsonlFilePath := filepath.Join(w.sourceDir, table.ExecutionIdToJsonlFileName(w.executionId, chunkNumber)) + // verify file exists + if _, err := os.Stat(jsonlFilePath); os.IsNotExist(err) { + missingFiles = append(missingFiles, jsonlFilePath) + } + // remove single quotes from the file path to avoid issues with SQL queries + escapedPath := strings.ReplaceAll(jsonlFilePath, "'", "''") + filenames[i] = escapedPath + } + if len(missingFiles) > 0 { + return filenames, NewConversionError(fmt.Errorf("%s not found", utils.Pluralize("file", len(missingFiles))), 0, missingFiles...) + + } + return filenames, nil +} + +func (w *Converter) insertBatchIntoDuckLake(filenames []string) error { + t := time.Now() + // ensure we signal the converter when we are done + defer w.wg.Add(len(filenames) * -1) + + // copy the data from the jsonl file to a temp table + if err := w.copyChunkToTempTable(filenames); err != nil { + // copyChunkToTempTable will already have called handleSchemaChangeError anf handleConversionError + return err + } + + tempTime := time.Now() + + // TODO #DL re-add validation + // https://github.com/turbot/tailpipe/issues/479 + + // now validate the data + //if validateRowsError := w.validateRows(jsonlFilePath); validateRowsError != nil { + // // if the error is NOT RowValidationError, just return it + // if !errors.Is(validateRowsError, &RowValidationError{}) { + // return handleConversionError(validateRowsError, jsonlFilePath) + // } + // + // // so it IS a row validation error - the invalid rows will have been removed from the temp table + // // - process the rest of the chunk + // // ensure that we return the row validation error, merged with any other error we receive + // defer func() { + // if err == nil { + // err = validateRowsError + // } else { + // var conversionError *ConversionError + // if errors.As(validateRowsError, &conversionError) { + // // we have a conversion error - we need to set the row count to 0 + // // so we can report the error + // conversionError.Merge(err) + // } + // err = conversionError + // } + // }() + //} + + slog.Debug("about to insert rows into ducklake table") + + rowCount, err := w.insertIntoDucklake(w.Partition.TableName) + if err != nil { + slog.Error("failed to insert into DuckLake table", "table", w.Partition.TableName, "error", err) + return err + } + + td := tempTime.Sub(t) + cd := time.Since(tempTime) + total := time.Since(t) + + // Update counters and advance to the next batch + // if we have an error, return it below + // update the row count + w.updateRowCount(rowCount) + + slog.Debug("inserted rows into DuckLake table", "chunks", len(filenames), "row count", rowCount, "error", err, "temp time", td.Milliseconds(), "conversion time", cd.Milliseconds(), "total time ", total.Milliseconds()) + return nil +} + +func (w *Converter) copyChunkToTempTable(jsonlFilePaths []string) error { + var queryBuilder strings.Builder + + // Create SQL array of file paths + var fileSQL string + if len(jsonlFilePaths) == 1 { + + fileSQL = fmt.Sprintf("'%s'", jsonlFilePaths[0]) + } else { + // For multiple files, create a properly quoted array + var quotedPaths []string + for _, jsonFilePath := range jsonlFilePaths { + quotedPaths = append(quotedPaths, fmt.Sprintf("'%s'", jsonFilePath)) + } + fileSQL = "[" + strings.Join(quotedPaths, ", ") + "]" + } + + // render the read JSON query with the jsonl file path + // - this build a select clause which selects the required data from the JSONL file (with columns types specified) + selectQuery := fmt.Sprintf(w.readJsonQueryFormat, fileSQL) + + // Step: Prepare the temp table from JSONL input + // + // - Drop the temp table if it exists + // - Create a new temp table by executing the dselect query + queryBuilder.WriteString(fmt.Sprintf(` +drop table if exists temp_data; + +create temp table temp_data as + %s +`, selectQuery)) + + _, err := w.db.Exec(queryBuilder.String()) + if err != nil { + // if the error is a schema change error, determine whether the schema of these chunk is + // different to the inferred schema (pass the first json file) + return w.handleSchemaChangeError(err, jsonlFilePaths[0]) + } + return nil +} + +// insertIntoDucklakeForBatch writes a batch of rows from the temp_data table to the specified target DuckDB table. +// +// It selects rows based on rowid, using the provided startRowId and rowCount to control the range: +// - Rows with rowid > startRowId and rowid <= (startRowId + rowCount) are selected. +// +// This approach allows for efficient batching from the temporary table into the final destination table. +// +// To prevent schema mismatches, it explicitly lists columns in the INSERT statement based on the conversion schema. +// +// Returns the number of rows inserted and any error encountered. +func (w *Converter) insertIntoDucklake(targetTable string) (int64, error) { + // quote the table name + targetTable = fmt.Sprintf(`"%s"`, targetTable) + + // Build the final INSERT INTO ... SELECT statement using the fully qualified table name. + columns := w.conversionSchema.ColumnString + insertQuery := fmt.Sprintf(` + insert into %s (%s) + select %s from temp_data + `, targetTable, columns, columns) + + // Execute the insert statement + result, err := w.db.Exec(insertQuery) + if err != nil { + slog.Error(fmt.Sprintf("failed to insert data into DuckLake table db %p", w.db.DB), "table", targetTable, "error", err, "db", w.db.DB) + // It's helpful to wrap the error with context about what failed. + return 0, fmt.Errorf("failed to insert data into %s: %w", targetTable, err) + } + + // Get the number of rows that were actually inserted. + insertedRowCount, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get number of affected rows: %w", err) + } + + return insertedRowCount, nil +} + +// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema +// infer the schema of this chunk and compare - if they are different, return that in an error +func (w *Converter) handleSchemaChangeError(err error, jsonlFilePath string) error { + schemaChangeErr := w.detectSchemaChange(jsonlFilePath) + if schemaChangeErr != nil { + // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error + var e = &SchemaChangeError{} + if errors.As(schemaChangeErr, &e) { + // update err and fall through to handleConversionError - this wraps the error with additional row count info + err = e + } + } + + // just return the original error, wrapped with the row count + return handleConversionError(err, jsonlFilePath) +} + +// conversionRanOutOfMemory checks if the error is an out-of-memory error from DuckDB +func conversionRanOutOfMemory(err error) bool { + var duckDBErr = &duckdb.Error{} + if errors.As(err, &duckDBErr) { + return duckDBErr.Type == duckdb.ErrorTypeOutOfMemory + } + return false +} diff --git a/internal/parquet/convertor_ducklake.go b/internal/parquet/convertor_ducklake.go new file mode 100644 index 00000000..28e3b8a1 --- /dev/null +++ b/internal/parquet/convertor_ducklake.go @@ -0,0 +1,167 @@ +package parquet + +import ( + "fmt" + "strings" + + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +// determine whether we have a ducklake table for this table, and if so, whether it needs schema updating +func (w *Converter) ensureDuckLakeTable(tableName string) error { + query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", tableName) + var exists bool + if err := w.db.QueryRow(query).Scan(&exists); err != nil { + return err + } + if !exists { + return w.createDuckLakeTable(tableName) + } + return nil +} + +// createDuckLakeTable creates a DuckLake table based on the ConversionSchema +func (w *Converter) createDuckLakeTable(tableName string) error { + + // Generate the CREATE TABLE SQL + createTableSQL := w.buildCreateDucklakeTableSQL(tableName) + + // Execute the CREATE TABLE statement + _, err := w.db.Exec(createTableSQL) + if err != nil { + return fmt.Errorf("failed to create table %s: %w", tableName, err) + } + + // Set partitioning using ALTER TABLE + // partition by the partition, index, year and month + partitionColumns := []string{constants.TpPartition, constants.TpIndex, fmt.Sprintf("year(%s)", constants.TpTimestamp), fmt.Sprintf("month(%s)", constants.TpTimestamp)} + alterTableSQL := fmt.Sprintf(`alter table "%s" set partitioned by (%s);`, + tableName, + strings.Join(partitionColumns, ", ")) + + _, err = w.db.Exec(alterTableSQL) + if err != nil { + return fmt.Errorf("failed to set partitioning for table %s: %w", tableName, err) + } + + return nil +} + +// buildCreateDucklakeTableSQL generates the CREATE TABLE SQL statement based on the ConversionSchema +func (w *Converter) buildCreateDucklakeTableSQL(tableName string) string { + // Build column definitions in sorted order + var columnDefinitions []string + for _, column := range w.conversionSchema.Columns { + columnDef := w.buildColumnDefinition(column) + columnDefinitions = append(columnDefinitions, columnDef) + } + + return fmt.Sprintf(`create table if not exists "%s" ( +%s +);`, + tableName, + strings.Join(columnDefinitions, ",\n")) +} + +// buildColumnDefinition generates the SQL definition for a single column +func (w *Converter) buildColumnDefinition(column *schema.ColumnSchema) string { + columnName := fmt.Sprintf("\"%s\"", column.ColumnName) + + // Handle different column types + switch column.Type { + case "struct": + // For struct types, we need to build the struct definition + structDef := w.buildStructDefinition(column) + return fmt.Sprintf("\t%s %s", columnName, structDef) + case "json": + // json type + return fmt.Sprintf("\t%s json", columnName) + default: + // For scalar types, just use the type directly (lower case) + return fmt.Sprintf("\t%s %s", columnName, strings.ToLower(column.Type)) + } +} + +// buildStructDefinition generates the SQL struct definition for a struct column +func (w *Converter) buildStructDefinition(column *schema.ColumnSchema) string { + if len(column.StructFields) == 0 { + return "struct" + } + + var fieldDefinitions []string + for _, field := range column.StructFields { + fieldName := fmt.Sprintf("\"%s\"", field.ColumnName) + fieldType := strings.ToLower(field.Type) + + if field.Type == "struct" { + // Recursively build nested struct definition + nestedStruct := w.buildStructDefinition(field) + fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, nestedStruct)) + } else { + fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, fieldType)) + } + } + + return fmt.Sprintf("struct(%s)", strings.Join(fieldDefinitions, ", ")) +} + +// TODO #DL is this code needed - look at schema change detection +// https://github.com/turbot/tailpipe/issues/481 +//func (w *Converter) CheckTableSchema(db *sql.DB, tableName string, conversionSchema schema.ConversionSchema) (TableSchemaStatus, error) { +// // Check if table exists +// exists, err := w.tableExists(db, tableName) +// if err != nil { +// return TableSchemaStatus{}, err +// } +// +// if !exists { +// return TableSchemaStatus{}, nil +// } +// +// // Get existing schema +// existingSchema, err := w.getTableSchema(db, tableName) +// if err != nil { +// return TableSchemaStatus{}, fmt.Errorf("failed to retrieve schema: %w", err) +// } +// +// // Use constructor to create status from comparison +// diff := NewTableSchemaStatusFromComparison(existingSchema, conversionSchema) +// return diff, nil +//} +// +//func (w *Converter) tableExists(db *sql.DB, tableName string) (bool, error) { +// query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", tableName) +// var exists int +// if err := db.QueryRow(query).Scan(&exists); err != nil { +// return false, err +// } +// return exists == 1, nil +//} + +//func (w *Converter) getTableSchema(db *sql.DB, tableName string) (map[string]schema.ColumnSchema, error) { +// query := fmt.Sprintf("pragma table_info(%s);", tableName) +// rows, err := db.Query(query) +// if err != nil { +// return nil, err +// } +// defer rows.Close() +// +// schemaMap := make(map[string]schema.ColumnSchema) +// for rows.Next() { +// var name, dataType string +// var notNull, pk int +// var dfltValue sql.NullString +// +// if err := rows.Scan(&name, &dataType, ¬Null, &dfltValue, &pk); err != nil { +// return nil, err +// } +// +// schemaMap[name] = schema.ColumnSchema{ +// ColumnName: name, +// Type: dataType, +// } +// } +// +// return schemaMap, nil +//} diff --git a/internal/parquet/convertor_infer.go b/internal/parquet/convertor_infer.go deleted file mode 100644 index 058ebe86..00000000 --- a/internal/parquet/convertor_infer.go +++ /dev/null @@ -1,186 +0,0 @@ -package parquet - -import ( - "encoding/json" - "fmt" - "github.com/turbot/tailpipe-plugin-sdk/schema" - "github.com/turbot/tailpipe-plugin-sdk/table" - "github.com/turbot/tailpipe/internal/database" - "log" - "path/filepath" -) - -// populate the ConversionSchema -// determine if we have a full schema yet and if not infer from the chunk -func (w *Converter) buildConversionSchema(executionID string, chunk int32) error { - // if table schema is already complete, we can skip the inference and just populate the conversionSchema - // complete means that we have types for all columns in the table schema, and we are not mapping any source columns - if w.tableSchema.Complete() { - w.conversionSchema = schema.NewConversionSchema(w.tableSchema) - return nil - } - - // do the inference - conversionSchema, err := w.inferConversionSchema(executionID, chunk) - if err != nil { - return fmt.Errorf("failed to infer conversionSchema from first JSON file: %w", err) - } - - w.conversionSchema = conversionSchema - - // now validate the conversionSchema is complete - we should have types for all columns - // (if we do not that indicates a custom table definition was used which does not specify types for all optional fields - - // this should have caused a config validation error earlier on - return w.conversionSchema.EnsureComplete() -} - -func (w *Converter) inferConversionSchema(executionId string, chunkNumber int32) (*schema.ConversionSchema, error) { - jsonFileName := table.ExecutionIdToJsonlFileName(executionId, chunkNumber) - filePath := filepath.Join(w.sourceDir, jsonFileName) - - inferredSchema, err := w.InferSchemaForJSONLFile(filePath) - if err != nil { - return nil, err - } - return schema.NewConversionSchemaWithInferredSchema(w.tableSchema, inferredSchema), nil -} - -func (w *Converter) InferSchemaForJSONLFile(filePath string) (*schema.TableSchema, error) { - // TODO figure out why we need this hack - trying 2 different methods - inferredSchema, err := w.inferSchemaForJSONLFileWithDescribe(filePath) - if err != nil { - inferredSchema, err = w.inferSchemaForJSONLFileWithJSONStructure(filePath) - } - if err != nil { - return nil, fmt.Errorf("failed to infer conversionSchema from JSON file: %w", err) - } - inferredSchema.NormaliseColumnTypes() - return inferredSchema, nil -} - -// inferSchemaForJSONLFileWithJSONStructure infers the schema of a JSONL file using DuckDB -// it uses 2 different queries as depending on the data, one or the other has been observed to work -// (needs investigation) -func (w *Converter) inferSchemaForJSONLFileWithJSONStructure(filePath string) (*schema.TableSchema, error) { - // Open DuckDB connection - db, err := database.NewDuckDb() - if err != nil { - log.Fatalf("failed to open DuckDB connection: %v", err) - } - defer db.Close() - - // Query to infer schema using json_structure - query := ` - select json_structure(json)::varchar as schema - from read_json_auto(?) - limit 1; - ` - - var schemaStr string - err = db.QueryRow(query, filePath).Scan(&schemaStr) - if err != nil { - return nil, fmt.Errorf("failed to execute query: %w", err) - } - - // Parse the schema JSON - var fields map[string]string - if err := json.Unmarshal([]byte(schemaStr), &fields); err != nil { - return nil, fmt.Errorf("failed to parse schema JSON: %w", err) - } - - // Convert to TableSchema - res := &schema.TableSchema{ - Columns: make([]*schema.ColumnSchema, 0, len(fields)), - } - - // Convert each field to a column schema - for name, typ := range fields { - res.Columns = append(res.Columns, &schema.ColumnSchema{ - SourceName: name, - ColumnName: name, - Type: typ, - }) - } - - return res, nil -} - -func (w *Converter) inferSchemaForJSONLFileWithDescribe(filePath string) (*schema.TableSchema, error) { - - // Open DuckDB connection - db, err := database.NewDuckDb() - if err != nil { - log.Fatalf("failed to open DuckDB connection: %v", err) - } - defer db.Close() - - // Use DuckDB to describe the schema of the JSONL file - query := `SELECT column_name, column_type FROM (DESCRIBE (SELECT * FROM read_json_auto(?)))` - - rows, err := db.Query(query, filePath) - if err != nil { - return nil, fmt.Errorf("failed to query JSON schema: %w", err) - } - defer rows.Close() - - var res = &schema.TableSchema{} - - // Read the results - for rows.Next() { - var name, dataType string - err := rows.Scan(&name, &dataType) - if err != nil { - return nil, fmt.Errorf("failed to scan row: %w", err) - } - // Append inferred columns to the schema - res.Columns = append(res.Columns, &schema.ColumnSchema{ - SourceName: name, - ColumnName: name, - Type: dataType, - }) - } - - // Check for any errors from iterating over rows - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("failed during rows iteration: %w", err) - } - - return res, nil -} - -func (w *Converter) detectSchemaChange(filePath string) error { - inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) - if err != nil { - return fmt.Errorf("failed to infer schema from JSON file: %w", err) - } - // the conversion schema is the full schema for the table that we have alreadf inferred - conversionSchemaMap := w.conversionSchema.AsMap() - // the table schema is the (possibly partial) schema which was defined in config - we use this to exclude columns - // which have a type specified - tableSchemaMap := w.tableSchema.AsMap() - // Compare the inferred schema with the existing conversionSchema - var changedColumns []ColumnSchemaChange - for _, col := range inferredChunksSchema.Columns { - // if the table schema definition specifies a type for this column, ignore the columns (as we will use the defined type) - // we are only interested in a type change if the column is not defined in the table schema - if columnDef, ok := tableSchemaMap[col.ColumnName]; ok { - if columnDef.Type != "" { - // if the column is defined in the table schema, ignore it - continue - } - } - - existingCol, exists := conversionSchemaMap[col.SourceName] - if exists && col.Type != existingCol.Type { - changedColumns = append(changedColumns, ColumnSchemaChange{ - Name: col.SourceName, - OldType: existingCol.Type, - NewType: col.Type, - }) - } - } - if len(changedColumns) > 0 { - return &SchemaChangeError{ChangedColumns: changedColumns} - } - return nil -} diff --git a/internal/parquet/convertor_schema.go b/internal/parquet/convertor_schema.go index a8851ddb..11e10278 100644 --- a/internal/parquet/convertor_schema.go +++ b/internal/parquet/convertor_schema.go @@ -1,182 +1,171 @@ package parquet import ( + "encoding/json" "fmt" - "log/slog" - "strings" + "path/filepath" - "github.com/turbot/go-kit/helpers" - "github.com/turbot/tailpipe-plugin-sdk/constants" "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" + "github.com/turbot/tailpipe/internal/database" ) -// buildViewQuery builds a format string used to construct the conversion query which reads from the source ndjson file -/* -select - as - ... -from - read_ndjson( - '%s', - columns = { - : '', +// populate the ConversionSchema +// determine if we have a full schema yet and if not infer from the chunk +func (w *Converter) buildConversionSchema(executionID string, chunk int32) error { + // if table schema is already complete, we can skip the inference and just populate the conversionSchema + // complete means that we have types for all columns in the table schema, and we are not mapping any source columns + if w.tableSchema.Complete() { + w.conversionSchema = schema.NewConversionSchema(w.tableSchema) + return nil } - ) -where (tp_timestamp is null or tp_timestamp >= ) -*/ -func (w *Converter) buildReadJsonQueryFormat() string { - var tpTimestampMapped bool - - // first build the select clauses - use the table def columns - var selectClauses []string - for _, column := range w.conversionSchema.Columns { - - var selectClause string - switch column.ColumnName { - case constants.TpDate: - // skip this column - it is derived from tp_timestamp - continue - case constants.TpIndex: - // NOTE: we ignore tp_index in the source data and ONLY add it based ont he default or configured value - slog.Warn("tp_index is a reserved column name and should not be used in the source data. It will be added automatically based on the configured value.") - // set flag to indicate that the plugin populated the tp_index - // - the CLI may show a warning as plugins no longer need to do that - w.pluginPopulatesTpIndex = true - // skip this column - it will be populated manually using the partition config - continue - case constants.TpTimestamp: - tpTimestampMapped = true - // fallthrough to populate the select clasue as normal - fallthrough - default: - selectClause = getSelectSqlForField(column) - } - selectClauses = append(selectClauses, selectClause) + // do the inference + conversionSchema, err := w.inferConversionSchema(executionID, chunk) + if err != nil { + return fmt.Errorf("failed to infer conversionSchema from first JSON file: %w", err) } - // add the tp_index - this is determined by the partition - it defaults to "default" but may be overridden in the partition config - // NOTE: we DO NOT wrap the tp_index expression in quotes - that will have already been done as part of partition config validation - selectClauses = append(selectClauses, fmt.Sprintf("\t%s as \"tp_index\"", w.Partition.TpIndexColumn)) + w.conversionSchema = conversionSchema - // if we have a mapping for tp_timestamp, add tp_date as well - if tpTimestampMapped { - // Add tp_date after tp_timestamp is defined - selectClauses = append(selectClauses, ` case - when tp_timestamp is not null then date_trunc('day', tp_timestamp::timestamp) - end as tp_date`) - } + // now validate the conversionSchema is complete - we should have types for all columns + // (if we do not that indicates a custom table definition was used which does not specify types for all optional fields - + // this should have caused a config validation error earlier on + return w.conversionSchema.EnsureComplete() +} - // build column definitions - these will be passed to the read_json function - columnDefinitions := getReadJSONColumnDefinitions(w.conversionSchema.SourceColumns) +func (w *Converter) inferConversionSchema(executionId string, chunkNumber int32) (*schema.ConversionSchema, error) { + jsonFileName := table.ExecutionIdToJsonlFileName(executionId, chunkNumber) + filePath := filepath.Join(w.sourceDir, jsonFileName) - var whereClause string - if w.Partition.Filter != "" { - // we need to escape the % in the filter, as it is passed to the fmt.Sprintf function - filter := strings.ReplaceAll(w.Partition.Filter, "%", "%%") - whereClause = fmt.Sprintf("\nwhere %s", filter) + inferredSchema, err := w.InferSchemaForJSONLFile(filePath) + if err != nil { + return nil, err } - - res := fmt.Sprintf(`select -%s -from - read_ndjson( - '%%s', - %s - )%s`, strings.Join(selectClauses, ",\n"), helpers.Tabify(columnDefinitions, "\t"), whereClause) - - return res + return schema.NewConversionSchemaWithInferredSchema(w.tableSchema, inferredSchema), nil } -// return the column definitions for the row conversionSchema, in the format required for the duck db read_json_auto function -func getReadJSONColumnDefinitions(sourceColumns []schema.SourceColumnDef) string { - var str strings.Builder - str.WriteString("columns = {") - for i, column := range sourceColumns { - if i > 0 { - str.WriteString(", ") - } - str.WriteString(fmt.Sprintf(` - "%s": '%s'`, column.Name, column.Type)) +func (w *Converter) InferSchemaForJSONLFile(filePath string) (*schema.TableSchema, error) { + // depending on the data we have observed that one of the two queries will work + inferredSchema, err := w.inferSchemaForJSONLFileWithDescribe(w.db, filePath) + if err != nil { + inferredSchema, err = w.inferSchemaForJSONLFileWithJSONStructure(filePath) } - str.WriteString("\n}") - return str.String() + if err != nil { + return nil, fmt.Errorf("failed to infer conversionSchema from JSON file: %w", err) + } + inferredSchema.NormaliseColumnTypes() + return inferredSchema, nil } -// Return the SQL line to select the given field -func getSelectSqlForField(column *schema.ColumnSchema) string { +// inferSchemaForJSONLFileWithJSONStructure infers the schema of a JSONL file using DuckDB +// it uses 2 different queries as depending on the data, one or the other has been observed to work +// (needs investigation) +func (w *Converter) inferSchemaForJSONLFileWithJSONStructure(filePath string) (*schema.TableSchema, error) { + // Query to infer schema using json_structure + query := ` + select json_structure(json)::varchar as schema + from read_json_auto(?) + limit 1; + ` + + var schemaStr string + err := w.db.QueryRow(query, filePath).Scan(&schemaStr) + if err != nil { + return nil, fmt.Errorf("failed to execute query: %w", err) + } - // If the column has a transform, use it - if column.Transform != "" { - // as this is going into a string format, we need to escape % - escapedTransform := strings.ReplaceAll(column.Transform, "%", "%%") - return fmt.Sprintf("\t%s as \"%s\"", escapedTransform, column.ColumnName) + // Parse the schema JSON + var fields map[string]string + if err := json.Unmarshal([]byte(schemaStr), &fields); err != nil { + return nil, fmt.Errorf("failed to parse schema JSON: %w", err) } - // NOTE: we will have normalised column types to lower case - switch column.Type { - case "struct": - var str strings.Builder + // Convert to TableSchema + res := &schema.TableSchema{ + Columns: make([]*schema.ColumnSchema, 0, len(fields)), + } - // Start case logic to handle null values for the struct + // Convert each field to a column schema + for name, typ := range fields { + res.Columns = append(res.Columns, &schema.ColumnSchema{ + SourceName: name, + ColumnName: name, + Type: typ, + }) + } - str.WriteString(fmt.Sprintf("\tcase\n\t\twhen \"%s\" is null then null\n", column.SourceName)) - str.WriteString("\t\telse struct_pack(\n") + return res, nil +} - // Add nested fields to the struct_pack - for j, nestedColumn := range column.StructFields { - if j > 0 { - str.WriteString(",\n") - } - parentName := fmt.Sprintf("\"%s\"", column.SourceName) - str.WriteString(getTypeSqlForStructField(nestedColumn, parentName, 3)) - } +func (w *Converter) inferSchemaForJSONLFileWithDescribe(db *database.DuckDb, filePath string) (*schema.TableSchema, error) { + // Use DuckDB to describe the schema of the JSONL file + query := `SELECT column_name, column_type FROM (DESCRIBE (SELECT * FROM read_json_auto(?)))` - // Close struct_pack and case - str.WriteString("\n\t\t)\n") - str.WriteString(fmt.Sprintf("\tend as \"%s\"", column.ColumnName)) - return str.String() + rows, err := db.Query(query, filePath) + if err != nil { + return nil, fmt.Errorf("failed to query JSON schema: %w", err) + } + defer rows.Close() - case "json": - // Convert the value using json() - return fmt.Sprintf("\tjson(\"%s\") as \"%s\"", column.SourceName, column.ColumnName) + var res = &schema.TableSchema{} - default: - // Scalar fields - return fmt.Sprintf("\t\"%s\" as \"%s\"", column.SourceName, column.ColumnName) + // Read the results + for rows.Next() { + var name, dataType string + err := rows.Scan(&name, &dataType) + if err != nil { + return nil, fmt.Errorf("failed to scan row: %w", err) + } + // Append inferred columns to the schema + res.Columns = append(res.Columns, &schema.ColumnSchema{ + SourceName: name, + ColumnName: name, + Type: dataType, + }) } -} - -// Return the SQL line to pack the given field as a struct -func getTypeSqlForStructField(column *schema.ColumnSchema, parentName string, tabs int) string { - tab := strings.Repeat("\t", tabs) - switch column.Type { - case "struct": - var str strings.Builder + // Check for any errors from iterating over rows + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed during rows iteration: %w", err) + } - // Add case logic to handle null values for the struct - str.WriteString(fmt.Sprintf("%s\"%s\" := case\n", tab, column.ColumnName)) - str.WriteString(fmt.Sprintf("%s\twhen %s.\"%s\" is null then null\n", tab, parentName, column.SourceName)) - str.WriteString(fmt.Sprintf("%s\telse struct_pack(\n", tab)) + return res, nil +} - // Loop through nested fields and add them to the struct_pack - for j, nestedColumn := range column.StructFields { - if j > 0 { - str.WriteString(",\n") +func (w *Converter) detectSchemaChange(filePath string) error { + inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) + if err != nil { + return fmt.Errorf("failed to infer schema from JSON file: %w", err) + } + // the conversion schema is the full schema for the table that we have alreadf inferred + conversionSchemaMap := w.conversionSchema.AsMap() + // the table schema is the (possibly partial) schema which was defined in config - we use this to exclude columns + // which have a type specified + tableSchemaMap := w.tableSchema.AsMap() + // Compare the inferred schema with the existing conversionSchema + var changedColumns []ColumnSchemaChange + for _, col := range inferredChunksSchema.Columns { + // if the table schema definition specifies a type for this column, ignore the columns (as we will use the defined type) + // we are only interested in a type change if the column is not defined in the table schema + if columnDef, ok := tableSchemaMap[col.ColumnName]; ok { + if columnDef.Type != "" { + // if the column is defined in the table schema, ignore it + continue } - // Use the current field as the new parent for recursion - newParent := fmt.Sprintf("%s.\"%s\"", parentName, column.SourceName) - str.WriteString(getTypeSqlForStructField(nestedColumn, newParent, tabs+2)) } - // Close struct_pack and case - str.WriteString(fmt.Sprintf("\n%s\t)\n", tab)) - str.WriteString(fmt.Sprintf("%send", tab)) - return str.String() - - default: - // Scalar fields - return fmt.Sprintf("%s\"%s\" := %s.\"%s\"::%s", tab, column.ColumnName, parentName, column.SourceName, column.Type) + existingCol, exists := conversionSchemaMap[col.SourceName] + if exists && col.Type != existingCol.Type { + changedColumns = append(changedColumns, ColumnSchemaChange{ + Name: col.SourceName, + OldType: existingCol.Type, + NewType: col.Type, + }) + } + } + if len(changedColumns) > 0 { + return &SchemaChangeError{ChangedColumns: changedColumns} } + return nil } diff --git a/internal/parquet/convertor_validate.go b/internal/parquet/convertor_validate.go new file mode 100644 index 00000000..ab7d6769 --- /dev/null +++ b/internal/parquet/convertor_validate.go @@ -0,0 +1,117 @@ +package parquet + +import ( + "fmt" + "strings" +) + +// validateRows copies the data from the given select query to a temp table and validates required fields are non null +// it also validates that the schema of the chunk is the same as the inferred schema and if it is not, reports a useful error +// the query count of invalid rows and a list of null fields +// +//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 +func (w *Converter) validateRows(jsonlFilePaths []string) error { + // build array of required columns to validate + var requiredColumns []string + for _, col := range w.conversionSchema.Columns { + if col.Required { + // if the column is required, add it to the list of columns to validate + requiredColumns = append(requiredColumns, col.ColumnName) + } + } + + // if we have no columns to validate, biuld a validation query to return the number of invalid rows and the columns with nulls + validationQuery := w.buildValidationQuery(requiredColumns) + + row := w.db.QueryRow(validationQuery) + var failedRowCount int64 + var columnsWithNullsInterface []interface{} + + err := row.Scan(&failedRowCount, &columnsWithNullsInterface) + if err != nil { + return w.handleSchemaChangeError(err, jsonlFilePaths[0]) + } + + if failedRowCount == 0 { + // no rows with nulls - we are done + return nil + } + + // delete invalid rows from the temp table + if err := w.deleteInvalidRows(requiredColumns); err != nil { + // failed to delete invalid rows - return an error + err := handleConversionError(err, jsonlFilePaths...) + return err + } + + // Convert the interface slice to string slice + var columnsWithNulls []string + for _, col := range columnsWithNullsInterface { + if col != nil { + columnsWithNulls = append(columnsWithNulls, col.(string)) + } + } + + // we have a failure - return an error with details about which columns had nulls + return NewConversionError(NewRowValidationError(failedRowCount, columnsWithNulls), failedRowCount, jsonlFilePaths...) +} + +// buildValidationQuery builds a query to copy the data from the select query to a temp table +// it then validates that the required columns are not null, removing invalid rows and returning +// the count of invalid rows and the columns with nulls +// +//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 +func (w *Converter) buildValidationQuery(requiredColumns []string) string { + var queryBuilder strings.Builder + + // Build the validation query that: + // - Counts distinct rows that have null values in required columns + // - Lists all required columns that contain null values + queryBuilder.WriteString(`select + count(distinct rowid) as rows_with_required_nulls, -- Count unique rows with nulls in required columns + coalesce(list(distinct col), []) as required_columns_with_nulls -- List required columns that have null values, defaulting to empty list if NULL +from (`) + + // Step 3: For each required column we need to validate: + // - Create a query that selects rows where this column is null + // - Include the column name so we know which column had the null + // - UNION ALL combines all these results (faster than UNION as we don't need to deduplicate) + for i, col := range requiredColumns { + if i > 0 { + queryBuilder.WriteString(" union all\n") + } + // For each required column, create a query that: + // - Selects the rowid (to count distinct rows) + // - Includes the column name (to list which columns had nulls) + // - Only includes rows where this column is null + queryBuilder.WriteString(fmt.Sprintf(" select rowid, '%s' as col from temp_data where %s is null\n", col, col)) + } + + queryBuilder.WriteString(");") + + return queryBuilder.String() +} + +// buildNullCheckQuery builds a WHERE clause to check for null values in the specified columns +// +//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 +func (w *Converter) buildNullCheckQuery(requiredColumns []string) string { + + // build a slice of null check conditions + conditions := make([]string, len(requiredColumns)) + for i, col := range requiredColumns { + conditions[i] = fmt.Sprintf("%s is null", col) + } + return strings.Join(conditions, " or ") +} + +// deleteInvalidRows removes rows with null values in the specified columns from the temp table +// +//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 +func (w *Converter) deleteInvalidRows(requiredColumns []string) error { + whereClause := w.buildNullCheckQuery(requiredColumns) + query := fmt.Sprintf("delete from temp_data where %s;", whereClause) + + _, err := w.db.Exec(query) + return err +} diff --git a/internal/parquet/delete.go b/internal/parquet/delete.go deleted file mode 100644 index 2a0b187f..00000000 --- a/internal/parquet/delete.go +++ /dev/null @@ -1,153 +0,0 @@ -package parquet - -import ( - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - "time" - - "github.com/turbot/pipe-fittings/v2/utils" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" -) - -func DeleteParquetFiles(partition *config.Partition, from time.Time) (rowCount int, err error) { - db, err := database.NewDuckDb() - if err != nil { - return 0, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - - if from.IsZero() { - // if there is no from time, delete the entire partition folder - rowCount, err = deletePartition(db, dataDir, partition) - } else { - // otherwise delete partition data for a time range - rowCount, err = deletePartitionFrom(db, dataDir, partition, from) - } - if err != nil { - return 0, fmt.Errorf("failed to delete partition: %w", err) - } - - // delete all empty folders underneath the partition folder - partitionDir := filepaths.GetParquetPartitionPath(dataDir, partition.TableName, partition.ShortName) - pruneErr := filepaths.PruneTree(partitionDir) - if pruneErr != nil { - // do not return error - just log - slog.Warn("DeleteParquetFiles failed to prune empty folders", "error", pruneErr) - } - - return rowCount, nil -} - -func deletePartitionFrom(db *database.DuckDb, dataDir string, partition *config.Partition, from time.Time) (_ int, err error) { - parquetGlobPath := filepaths.GetParquetFileGlobForPartition(dataDir, partition.TableName, partition.ShortName, "") - - query := fmt.Sprintf(` - select - distinct '%s/tp_table=' || tp_table || '/tp_partition=' || tp_partition || '/tp_index=' || tp_index || '/tp_date=' || tp_date as hive_path, - count(*) over() as total_files - from read_parquet('%s', hive_partitioning=true) - where tp_partition = ? - and tp_date >= ?`, - dataDir, parquetGlobPath) - - rows, err := db.Query(query, partition.ShortName, from) - if err != nil { - // is this an error because there are no files? - if isNoFilesFoundError(err) { - return 0, nil - } - return 0, fmt.Errorf("failed to query parquet folder names: %w", err) - } - defer rows.Close() - - var folders []string - var count int - // Iterate over the results - for rows.Next() { - var folder string - if err := rows.Scan(&folder, &count); err != nil { - return 0, fmt.Errorf("failed to scan parquet folder name: %w", err) - } - folders = append(folders, folder) - } - - var errors = make(map[string]error) - for _, folder := range folders { - if err := os.RemoveAll(folder); err != nil { - errors[folder] = err - } - } - - return len(folders), nil -} - -func deletePartition(db *database.DuckDb, dataDir string, partition *config.Partition) (int, error) { - parquetGlobPath := filepaths.GetParquetFileGlobForPartition(dataDir, partition.TableName, partition.ShortName, "") - - // get count of parquet files - query := fmt.Sprintf(` - select count(distinct __duckdb_source_file) - from read_parquet('%s', hive_partitioning=true, filename='__duckdb_source_file') - where tp_partition = ? - `, parquetGlobPath) - - // Execute the query with a parameter for the tp_partition filter - q := db.QueryRow(query, partition.ShortName) - // read the result - var count int - err := q.Scan(&count) - if err != nil && !isNoFilesFoundError(err) { - return 0, fmt.Errorf("failed to query parquet file count: %w", err) - } - - partitionFolder := filepaths.GetParquetPartitionPath(dataDir, partition.TableName, partition.ShortName) - err = os.RemoveAll(partitionFolder) - if err != nil { - return 0, fmt.Errorf("failed to delete partition folder: %w", err) - } - return count, nil -} - -func isNoFilesFoundError(err error) bool { - return strings.HasPrefix(err.Error(), "IO Error: No files found") -} - -// deleteInvalidParquetFiles deletes invalid and temporary parquet files for a partition -func deleteInvalidParquetFiles(dataDir string, patterns []PartitionPattern) error { - var failures int - - for _, pattern := range patterns { - - slog.Info("deleteInvalidParquetFiles - deleting invalid parquet files", "table", pattern.Table, "partition", pattern.Partition) - - // get glob patterns for invalid and temp files - invalidGlob := filepaths.GetTempAndInvalidParquetFileGlobForPartition(dataDir, pattern.Table, pattern.Partition) - - // find all matching files - filesToDelete, err := filepath.Glob(invalidGlob) - if err != nil { - return fmt.Errorf("failed to find invalid files: %w", err) - } - - slog.Info("deleteInvalidParquetFiles", "invalid count", len(filesToDelete), "files", filesToDelete) - - // delete each file - for _, file := range filesToDelete { - if err := os.Remove(file); err != nil { - slog.Debug("failed to delete invalid parquet file", "file", file, "error", err) - failures++ - } - } - } - if failures > 0 { - return fmt.Errorf("failed to delete %d invalid parquet %s", failures, utils.Pluralize("file", failures)) - } - return nil -} diff --git a/internal/parquet/delete_test.go b/internal/parquet/delete_test.go deleted file mode 100644 index aeddb9b7..00000000 --- a/internal/parquet/delete_test.go +++ /dev/null @@ -1,356 +0,0 @@ -package parquet - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/hashicorp/hcl/v2" - "github.com/stretchr/testify/assert" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/filepaths" -) - -func Test_deleteInvalidParquetFiles(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "delete_invalid_parquet_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - // Create test partition - block := &hcl.Block{ - Labels: []string{"test_table", "test_partition"}, - } - partitionResource, _ := config.NewPartition(block, "partition.test_table.test_partition") - partition := partitionResource.(*config.Partition) - - // Create test files - testFiles := []struct { - name string - expected bool // whether the file should be deleted - }{ - { - name: "old_invalid.parquet.invalid", - expected: true, - }, - { - name: "new_invalid.parquet.invalid", - expected: true, - }, - { - name: "old_temp.parquet.tmp", - expected: true, - }, - { - name: "new_temp.parquet.tmp", - expected: true, - }, - { - name: "valid.parquet", - expected: false, - }, - } - - // Create the partition directory - partitionDir := filepaths.GetParquetPartitionPath(tempDir, partition.TableName, partition.ShortName) - if err := os.MkdirAll(partitionDir, 0755); err != nil { - t.Fatalf("Failed to create partition dir: %v", err) - } - - // Create test files - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - if err := os.WriteFile(filePath, []byte("test data"), 0644); err != nil { //nolint:gosec // test code - t.Fatalf("Failed to create test file %s: %v", tf.name, err) - } - } - - // Debug: Print directory structure - err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - rel, _ := filepath.Rel(tempDir, path) - if rel == "." { - return nil - } - if info.IsDir() { - t.Logf("DIR: %s", rel) - } else { - t.Logf("FILE: %s", rel) - } - return nil - }) - if err != nil { - t.Logf("Error walking directory: %v", err) - } - - // Debug: Print glob pattern - invalidGlob := filepaths.GetTempAndInvalidParquetFileGlobForPartition(tempDir, partition.TableName, partition.ShortName) - t.Logf("Glob pattern: %s", invalidGlob) - - // Run the delete function - patterns := []PartitionPattern{NewPartitionPattern(partition)} - err = deleteInvalidParquetFiles(tempDir, patterns) - if err != nil { - t.Fatalf("deleteInvalidParquetFiles failed: %v", err) - } - - // Check which files were deleted - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - _, err := os.Stat(filePath) - exists := err == nil - - if tf.expected { - assert.False(t, exists, "File %s should have been deleted", tf.name) - } else { - assert.True(t, exists, "File %s should not have been deleted", tf.name) - } - } -} - -func Test_deleteInvalidParquetFilesWithWildcards(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "delete_invalid_parquet_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - // Create test partitions - partitions := []struct { - table string - partition string - }{ - {"aws_cloudtrail", "cloudtrail"}, - {"aws_cloudtrail", "cloudwatch"}, - {"aws_ec2", "instances"}, - {"aws_ec2", "volumes"}, - } - - // Create test files for each partition - testFiles := []struct { - name string - expected bool - }{ - { - name: "invalid.parquet.invalid", - expected: true, - }, - { - name: "temp.parquet.tmp", - expected: true, - }, - { - name: "valid.parquet", - expected: false, - }, - } - - // Create directories and files for each partition - for _, p := range partitions { - partitionDir := filepaths.GetParquetPartitionPath(tempDir, p.table, p.partition) - if err := os.MkdirAll(partitionDir, 0755); err != nil { - t.Fatalf("Failed to create partition dir: %v", err) - } - - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - if err := os.WriteFile(filePath, []byte("test data"), 0644); err != nil { //nolint:gosec // test code - t.Fatalf("Failed to create test file %s: %v", tf.name, err) - } - } - } - - // Test cases with different wildcard patterns - tests := []struct { - name string - patterns []PartitionPattern - deleted map[string]bool // key is "table/partition", value is whether files should be deleted - }{ - { - name: "match all aws_cloudtrail partitions", - patterns: []PartitionPattern{{ - Table: "aws_cloudtrail", - Partition: "*", - }}, - deleted: map[string]bool{ - "aws_cloudtrail/cloudtrail": true, - "aws_cloudtrail/cloudwatch": true, - "aws_ec2/instances": false, - "aws_ec2/volumes": false, - }, - }, - { - name: "match all aws_* tables", - patterns: []PartitionPattern{{ - Table: "aws_*", - Partition: "*", - }}, - deleted: map[string]bool{ - "aws_cloudtrail/cloudtrail": true, - "aws_cloudtrail/cloudwatch": true, - "aws_ec2/instances": true, - "aws_ec2/volumes": true, - }, - }, - { - name: "match specific partitions across tables", - patterns: []PartitionPattern{ - {Table: "aws_cloudtrail", Partition: "cloudtrail"}, - {Table: "aws_ec2", Partition: "instances"}, - }, - deleted: map[string]bool{ - "aws_cloudtrail/cloudtrail": true, - "aws_cloudtrail/cloudwatch": false, - "aws_ec2/instances": true, - "aws_ec2/volumes": false, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Run the delete function - err = deleteInvalidParquetFiles(tempDir, tt.patterns) - if err != nil { - t.Fatalf("deleteInvalidParquetFiles failed: %v", err) - } - - // Check each partition - for _, p := range partitions { - partitionDir := filepaths.GetParquetPartitionPath(tempDir, p.table, p.partition) - key := fmt.Sprintf("%s/%s", p.table, p.partition) - shouldDelete := tt.deleted[key] - - // Check each file - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - _, err := os.Stat(filePath) - exists := err == nil - - if shouldDelete && tf.expected { - assert.False(t, exists, "[%s] File %s should have been deleted", key, tf.name) - } else { - assert.True(t, exists, "[%s] File %s should not have been deleted", key, tf.name) - } - } - } - - // Recreate the files for the next test - for _, p := range partitions { - partitionDir := filepaths.GetParquetPartitionPath(tempDir, p.table, p.partition) - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - if err := os.WriteFile(filePath, []byte("test data"), 0644); err != nil { //nolint:gosec // test code - t.Fatalf("Failed to recreate test file %s: %v", tf.name, err) - } - } - } - }) - } -} - -//func Test_shouldClearInvalidState(t *testing.T) { -// tests := []struct { -// name string -// invalidFromDate time.Time -// from time.Time -// want bool -// }{ -// { -// name: "both zero", -// invalidFromDate: time.Time{}, -// from: time.Time{}, -// want: true, -// }, -// { -// name: "invalidFromDate zero, from not zero", -// invalidFromDate: time.Time{}, -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: false, -// }, -// { -// name: "from zero, invalidFromDate not zero", -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// from: time.Time{}, -// want: true, -// }, -// { -// name: "invalidFromDate before from", -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// from: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// want: true, -// }, -// { -// name: "invalidFromDate equal to from", -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: true, -// }, -// { -// name: "invalidFromDate after from", -// invalidFromDate: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: false, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := shouldClearInvalidState(tt.invalidFromDate, tt.from) -// assert.Equal(t, tt.want, got) -// }) -// } -//} -// -//func Test_getDeleteInvalidDate(t *testing.T) { -// tests := []struct { -// name string -// from time.Time -// invalidFromDate time.Time -// want time.Time -// }{ -// { -// name: "both zero", -// from: time.Time{}, -// invalidFromDate: time.Time{}, -// want: time.Time{}, -// }, -// { -// name: "from zero, invalidFromDate not zero", -// from: time.Time{}, -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// }, -// { -// name: "from not zero, invalidFromDate zero", -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// invalidFromDate: time.Time{}, -// want: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// }, -// { -// name: "from before invalidFromDate", -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// invalidFromDate: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// want: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// }, -// { -// name: "from after invalidFromDate", -// from: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := getDeleteInvalidDate(tt.from, tt.invalidFromDate) -// assert.Equal(t, tt.want, got) -// }) -// } -//} diff --git a/internal/parquet/ducklake.go b/internal/parquet/ducklake.go new file mode 100644 index 00000000..f03ffd0f --- /dev/null +++ b/internal/parquet/ducklake.go @@ -0,0 +1,141 @@ +package parquet + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/database" +) + +func DeletePartition(ctx context.Context, partition *config.Partition, from, to time.Time, db *database.DuckDb) (rowCount int, err error) { + // TODO #DL https://github.com/turbot/tailpipe/issues/505 + // if we are using s3 do not delete for now as this does not work at present (need explicit S3 support I think) + // remove before release https://github.com/turbot/tailpipe/issues/520 + if envDir := os.Getenv("TAILPIPE_DATA_DIR"); strings.HasPrefix(envDir, "s3") { + slog.Warn("Skipping partition deletion for S3 data source", + "partition", partition.TableName, + "from", from, + "to", to, + ) + return 0, nil // return 0 rows affected, not an error + } + + // First check if the table exists using DuckLake metadata + tableExistsQuery := fmt.Sprintf(`select exists (select 1 from %s.ducklake_table where table_name = ?)`, constants.DuckLakeMetadataCatalog) + var tableExists bool + if err := db.QueryRowContext(ctx, tableExistsQuery, partition.TableName).Scan(&tableExists); err != nil { + return 0, fmt.Errorf("failed to check if table exists: %w", err) + } + + if !tableExists { + // Table doesn't exist, return 0 rows affected (not an error) + return 0, nil + } + + // build a delete query for the partition + // Note: table names cannot be parameterized, so we use string formatting for the table name + query := fmt.Sprintf(`delete from "%s" where tp_partition = ? and tp_timestamp >= ? and tp_timestamp <= ?`, partition.TableName) + // Execute the query with parameters for the partition and date range + result, err := db.ExecContext(ctx, query, partition.ShortName, from, to) + if err != nil { + return 0, fmt.Errorf("failed to delete partition: %w", err) + } + + // Get the number of rows affected by the delete operation + rowsAffected, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get rows affected count: %w", err) + } + rowCount = int(rowsAffected) + + // Only perform cleanup if we actually deleted some rows + if rowCount > 0 { + if err = DucklakeCleanup(ctx, db); err != nil { + return 0, err + } + } + + return rowCount, nil +} + +// DucklakeCleanup performs removes old snapshots deletes expired and unused parquet files from the DuckDB database. +func DucklakeCleanup(ctx context.Context, db *database.DuckDb) error { + slog.Info("Cleaning up DuckLake snapshots and expired files") + // now clean old snapshots + if err := expirePrevSnapshots(ctx, db); err != nil { + return err + } + // delete expired files + if err := cleanupExpiredFiles(ctx, db); err != nil { + return err + } + return nil +} + +// expirePrevSnapshots expires all snapshots but the latest +// Ducklake stores a snapshot corresponding to each database operation - this allows the tracking of the history of changes +// However we do not need (currently) take advantage of this ducklake functionality, so we can remove all but the latest snapshot +// To do this we get the date of the most recent snapshot and then expire all snapshots older than that date. +// We then call ducklake_cleanup to remove the expired files. +func expirePrevSnapshots(ctx context.Context, db *database.DuckDb) error { + slog.Info("Expiring old DuckLake snapshots") + defer slog.Info("DuckLake snapshot expiration complete") + + // 1) get the timestamp of the latest snapshot from the metadata schema + var latestTimestamp string + query := fmt.Sprintf(`select snapshot_time from %s.ducklake_snapshot order by snapshot_id desc limit 1`, constants.DuckLakeMetadataCatalog) + + err := db.QueryRowContext(ctx, query).Scan(&latestTimestamp) + if err != nil { + return fmt.Errorf("failed to get latest snapshot timestamp: %w", err) + } + + // Parse the snapshot time + // NOTE: rather than cast as timestamp, we read as a string then remove any timezone component + // This is because of the dubious behaviour of ducklake_expire_snapshots described below + parsedTime, err := time.Parse("2006-01-02 15:04:05.999-07", latestTimestamp) + if err != nil { + if err != nil { + return fmt.Errorf("failed to parse snapshot time '%s': %w", latestTimestamp, err) + } + } + // format the time + // TODO Note: ducklake_expire_snapshots expects a local time without timezone, + // i.e if the time is '2025-08-26 13:25:10.365 +0100', we should pass '2025-08-26 13:25:10.365' + // We need to raise a ducklake issue + formattedTime := parsedTime.Format("2006-01-02 15:04:05.000") + + slog.Debug("Latest snapshot timestamp", "timestamp", latestTimestamp) + + // 2) expire all snapshots older than the latest one + // Note: ducklake_expire_snapshots uses named parameters which cannot be parameterized with standard SQL placeholders + expireQuery := fmt.Sprintf(`call ducklake_expire_snapshots('%s', older_than => '%s')`, constants.DuckLakeCatalog, formattedTime) + + _, err = db.ExecContext(ctx, expireQuery) + if err != nil { + return fmt.Errorf("failed to expire old snapshots: %w", err) + } + + return nil +} + +// cleanupExpiredFiles deletes and files marked as expired in the ducklake system. +func cleanupExpiredFiles(ctx context.Context, db *database.DuckDb) error { + slog.Info("Cleaning up expired files in DuckLake") + defer slog.Info("DuckLake expired files cleanup complete") + + cleanupQuery := fmt.Sprintf("call ducklake_cleanup_old_files('%s', cleanup_all => true)", constants.DuckLakeCatalog) + + _, err := db.ExecContext(ctx, cleanupQuery) + if err != nil { + return fmt.Errorf("failed to cleanup expired files: %w", err) + } + + return nil +} diff --git a/internal/parquet/file_helpers.go b/internal/parquet/file_helpers.go index e5884420..4b903d4f 100644 --- a/internal/parquet/file_helpers.go +++ b/internal/parquet/file_helpers.go @@ -14,26 +14,6 @@ import ( "github.com/turbot/pipe-fittings/v2/utils" ) -// if this parquetFile ends with the partition segment, return the table and partition -func getPartitionFromPath(dirPath string) (string, string, bool) { - // if this is a partition folder, check if it matches the patterns - parts := strings.Split(dirPath, "/") - l := len(parts) - if l < 2 { - return "", "", false - } - - // Find the last two segments that match our pattern - for i := l - 1; i > 0; i-- { - if strings.HasPrefix(parts[i], "tp_partition=") && strings.HasPrefix(parts[i-1], "tp_table=") { - table := strings.TrimPrefix(parts[i-1], "tp_table=") - partition := strings.TrimPrefix(parts[i], "tp_partition=") - return table, partition, true - } - } - return "", "", false -} - // addExtensionToFiles renames all given files to add a the provided extension func addExtensionToFiles(fileNames []string, suffix string) ([]string, error) { if len(fileNames) == 0 { diff --git a/internal/parquet/file_helpers_test.go b/internal/parquet/file_helpers_test.go index de3b05af..e8b0f125 100644 --- a/internal/parquet/file_helpers_test.go +++ b/internal/parquet/file_helpers_test.go @@ -494,118 +494,6 @@ func Test_deleteFilesConcurrently(t *testing.T) { } } -func Test_getPartitionFromPath(t *testing.T) { - type args struct { - dirPath string - } - tests := []struct { - name string - args args - want string - want1 string - want2 bool - }{ - { - name: "valid partition path", - args: args{ - dirPath: "/data/tp_table=aws_cloudtrail/tp_partition=cloudtrail", - }, - want: "aws_cloudtrail", - want1: "cloudtrail", - want2: true, - }, - { - name: "valid partition path with additional segments", - args: args{ - dirPath: "/data/tp_table=aws_cloudtrail/tp_partition=cloudtrail/tp_index=123/tp_date=2024-01-01", - }, - want: "aws_cloudtrail", - want1: "cloudtrail", - want2: true, - }, - { - name: "invalid path - missing tp_table", - args: args{ - dirPath: "/data/tp_partition=cloudtrail", - }, - want: "", - want1: "", - want2: false, - }, - { - name: "invalid path - missing tp_partition", - args: args{ - dirPath: "/data/tp_table=aws_cloudtrail", - }, - want: "", - want1: "", - want2: false, - }, - { - name: "invalid path - wrong order", - args: args{ - dirPath: "/data/tp_partition=cloudtrail/tp_table=aws_cloudtrail", - }, - want: "", - want1: "", - want2: false, - }, - { - name: "invalid path - empty", - args: args{ - dirPath: "", - }, - want: "", - want1: "", - want2: false, - }, - { - name: "invalid path - root only", - args: args{ - dirPath: "/", - }, - want: "", - want1: "", - want2: false, - }, - { - name: "path with special characters", - args: args{ - dirPath: "/data/tp_table=aws@cloudtrail/tp_partition=cloud@trail", - }, - want: "aws@cloudtrail", - want1: "cloud@trail", - want2: true, - }, - { - name: "path with multiple partition segments", - args: args{ - dirPath: "/data/tp_table=aws_cloudtrail/tp_partition=cloudtrail/tp_partition=backup", - }, - want: "aws_cloudtrail", - want1: "cloudtrail", - want2: true, - }, - { - name: "path with escaped characters", - args: args{ - dirPath: "/data/tp_table=aws\\_cloudtrail/tp_partition=cloud\\_trail", - }, - want: "aws\\_cloudtrail", - want1: "cloud\\_trail", - want2: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1, got2 := getPartitionFromPath(tt.args.dirPath) - assert.Equalf(t, tt.want, got, "getPartitionFromPath(%v)", tt.args.dirPath) - assert.Equalf(t, tt.want1, got1, "getPartitionFromPath(%v)", tt.args.dirPath) - assert.Equalf(t, tt.want2, got2, "getPartitionFromPath(%v)", tt.args.dirPath) - }) - } -} - func Test_removeExtensionFromFiles(t *testing.T) { // Create a temporary directory for test files tempDir, err := os.MkdirTemp("", "remove_extension_test") diff --git a/internal/parquet/file_metadata.go b/internal/parquet/file_metadata.go new file mode 100644 index 00000000..165521e4 --- /dev/null +++ b/internal/parquet/file_metadata.go @@ -0,0 +1,103 @@ +package parquet + +import ( + "context" + "fmt" + + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/tailpipe/internal/database" +) + +// FileMetadata represents the result of a file metadata query +type FileMetadata struct { + FileSize int64 + FileCount int64 + RowCount int64 +} + +// TableExists checks if a table exists in the DuckLake metadata tables +func TableExists(ctx context.Context, tableName string, db *database.DuckDb) (bool, error) { + query := fmt.Sprintf(`select count(*) from %s.ducklake_table where table_name = ?`, constants.DuckLakeMetadataCatalog) + + var count int64 + err := db.QueryRowContext(ctx, query, tableName).Scan(&count) + if err != nil { + return false, fmt.Errorf("unable to check if table %s exists: %w", tableName, err) + } + + return count > 0, nil +} + +// GetTableFileMetadata gets file metadata for a specific table from DuckLake metadata tables +func GetTableFileMetadata(ctx context.Context, tableName string, db *database.DuckDb) (*FileMetadata, error) { + // first see if the table exists + exists, err := TableExists(ctx, tableName, db) + if err != nil { + return nil, fmt.Errorf("unable to check if table %s exists: %w", tableName, err) + } + if !exists { + // leave everything at zero + return &FileMetadata{}, nil + } + + query := fmt.Sprintf(`select + sum(f.file_size_bytes) as total_size, + count(*) as file_count, + sum(f.record_count) as row_count +from %s.ducklake_data_file f + join %s.ducklake_partition_info p on f.partition_id = p.partition_id + join %s.ducklake_table tp on p.table_id = tp.table_id +where tp.table_name = ? and f.end_snapshot is null`, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog) + + var totalSize, fileCount, rowCount int64 + err = db.QueryRowContext(ctx, query, tableName).Scan(&totalSize, &fileCount, &rowCount) + if err != nil { + return nil, fmt.Errorf("unable to obtain file metadata for table %s: %w", tableName, err) + } + + return &FileMetadata{ + FileSize: totalSize, + FileCount: fileCount, + RowCount: rowCount, + }, nil +} + +// GetPartitionFileMetadata gets file metadata for a specific partition from DuckLake metadata tables +func GetPartitionFileMetadata(ctx context.Context, tableName, partitionName string, db *database.DuckDb) (*FileMetadata, error) { + // first see if the table exists + exists, err := TableExists(ctx, tableName, db) + if err != nil { + return nil, fmt.Errorf("unable to check if table %s exists: %w", tableName, err) + } + if !exists { + // leave everything at zero + return &FileMetadata{}, nil + } + + query := fmt.Sprintf(`select + coalesce(sum(f.file_size_bytes), 0) as total_size, + coalesce(count(*), 0) as file_count, + coalesce(sum(f.record_count), 0) as row_count +from %s.ducklake_data_file f + join %s.ducklake_file_partition_value fpv on f.data_file_id = fpv.data_file_id + join %s.ducklake_table tp on fpv.table_id = tp.table_id +where tp.table_name = ? and fpv.partition_value = ? and f.end_snapshot is null`, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog) + + var totalSize, fileCount, rowCount int64 + err = db.QueryRowContext(ctx, query, tableName, partitionName).Scan(&totalSize, &fileCount, &rowCount) + if err != nil { + return nil, fmt.Errorf("unable to obtain file metadata for partition %s.%s: %w", tableName, partitionName, err) + } + + return &FileMetadata{ + FileSize: totalSize, + FileCount: fileCount, + RowCount: rowCount, + }, nil +} diff --git a/internal/parquet/file_root_provider.go b/internal/parquet/file_root_provider.go deleted file mode 100644 index d619d2cb..00000000 --- a/internal/parquet/file_root_provider.go +++ /dev/null @@ -1,35 +0,0 @@ -package parquet - -import ( - "fmt" - "log/slog" - "sync" - "time" -) - -// FileRootProvider provides a unique file root for parquet files -// based on the current time to the nanosecond. -// If multiple files are created in the same nanosecond, the provider will increment the time by a nanosecond -// to ensure the file root is unique. -type FileRootProvider struct { - // the last time a filename was provided - lastTime time.Time - // mutex - mutex sync.Mutex -} - -// GetFileRoot returns a unique file root for a parquet file -// format is "data__" -func (p *FileRootProvider) GetFileRoot() string { - p.mutex.Lock() - defer p.mutex.Unlock() - - now := time.Now() - if now.Sub(p.lastTime) < time.Microsecond { - slog.Debug("incrementing time") - now = now.Add(time.Microsecond) - } - p.lastTime = now - - return fmt.Sprintf("data_%s_%06d", now.Format("20060102150405"), now.Nanosecond()/1000) -} diff --git a/internal/parquet/migrate_tpindex.go b/internal/parquet/migrate_tpindex.go index 58505d4d..9e37485d 100644 --- a/internal/parquet/migrate_tpindex.go +++ b/internal/parquet/migrate_tpindex.go @@ -2,182 +2,177 @@ package parquet import ( "context" - "fmt" - "log/slog" - "strings" - - sdkconstants "github.com/turbot/tailpipe-plugin-sdk/constants" - "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" ) const ( - sourceFileColumnName = "__duckdb_source_file" - migrateTempTableName = "_raw_tp_data" +// sourceFileColumnName = "__duckdb_source_file" +// migrateTempTableName = "_raw_tp_data" ) +//nolint:unused // TODO re-add tp_index migration for DuckDB https://github.com/turbot/tailpipe/issues/475 func migrateTpIndex(ctx context.Context, db *database.DuckDb, baseDir string, updateFunc func(CompactionStatus), patterns []PartitionPattern) error { - fileRootProvider := &FileRootProvider{} - for _, partition := range config.GlobalConfig.Partitions { - if PartitionMatchesPatterns(partition.TableName, partition.ShortName, patterns) { - err := migrateTpIndexForPartition(ctx, db, baseDir, partition, fileRootProvider, updateFunc) - if err != nil { - if ctx.Err() != nil { - return err - } - return fmt.Errorf("failed to migrate tp_index for partition %s: %w", partition.UnqualifiedName, err) - } else { - slog.Info("Migrated tp_index files for partition", "partition", partition.UnqualifiedName, "index_expression", partition.TpIndexColumn) - } - } - } - return nil -} - -func migrateTpIndexForPartition(ctx context.Context, db *database.DuckDb, baseDir string, partition *config.Partition, fileRootProvider *FileRootProvider, updateFunc func(CompactionStatus)) error { - - // executeMigrationQuery runs the DuckDB query to migrate the tp_index files for a given partition. - // it read the partition data into a temporary table, then writes the data to with the migrated tp_index - // to intermediate the output files (with extension .tmp) and returns the list of output files. - outputFiles, err := executeMigrationQuery(ctx, db, baseDir, partition, fileRootProvider) - if err != nil { - return err - } - if len(outputFiles) == 0 { - return nil // nothing to migrate - } - - // read the source files from the temporary table - sourceFiles, err := readSourceFiles(ctx, db) - if err != nil { - return err - } - - // now rename the source files to add a .migrated extension - renamedSourceFiles, err := addExtensionToFiles(sourceFiles, ".migrated") - if err != nil { - if err := deleteFilesConcurrently(ctx, outputFiles, baseDir); err != nil { - slog.Error("Failed to delete temp files after migration failure", "error", err) - } - return err - } - - // rename the output files to remove the .tmp extension - if err := removeExtensionFromFiles(outputFiles, ".tmp"); err != nil { - if err := deleteFilesConcurrently(ctx, outputFiles, baseDir); err != nil { - slog.Error("Failed to delete temp files after migration failure", "error", err) - } - - if err := removeExtensionFromFiles(renamedSourceFiles, ".migrated"); err != nil { - slog.Error("Failed to rename source files back to original names after migration failure", "error", err) - } - return err - } - - // finally, delete the renamed source parquet files - if err := deleteFilesConcurrently(ctx, renamedSourceFiles, baseDir); err != nil { - slog.Error("Failed to delete renamed source parquet files after migration", "error", err) - } - - status := CompactionStatus{ - MigrateSource: len(sourceFiles), - MigrateDest: len(outputFiles), - PartitionIndexExpressions: map[string]string{ - partition.UnqualifiedName: partition.TpIndexColumn, - }, - } - updateFunc(status) - + //fileRootProvider := &FileRootProvider{} + //for _, partition := range config.GlobalConfig.Partitions { + // if PartitionMatchesPatterns(partition.TableName, partition.ShortName, patterns) { + // err := migrateTpIndexForPartition(ctx, db, baseDir, partition, fileRootProvider, updateFunc) + // if err != nil { + // if ctx.Err() != nil { + // return err + // } + // return fmt.Errorf("failed to migrate tp_index for partition %s: %w", partition.UnqualifiedName, err) + // } else { + // slog.Info("Migrated tp_index files for partition", "partition", partition.UnqualifiedName, "index_expression", partition.TpIndexColumn) + // } + // } + //} return nil } -// executeMigrationQuery runs the DuckDB query to migrate the tp_index files for a given partition. -// It reads the partition data into a temporary table, writes the data with the migrated tp_index -// to intermediate output files (with .tmp extension), and returns the list of output file paths. -func executeMigrationQuery(ctx context.Context, db *database.DuckDb, baseDir string, partition *config.Partition, fileRootProvider *FileRootProvider) ([]string, error) { - // Get the file glob pattern for all files in this partition - fileGlob := filepaths.GetParquetFileGlobForPartition(baseDir, partition.TableName, partition.ShortName, "") - - // get unique file root to use for the output files - fileRoot := fileRootProvider.GetFileRoot() - // columns to partition by - partitionColumns := []string{sdkconstants.TpTable, sdkconstants.TpPartition, sdkconstants.TpIndex, sdkconstants.TpDate} - - // build the query to read the parquet files into a temporary table - query := fmt.Sprintf(` -create or replace temp table %s as -select - *, - %s, -from read_parquet('%s', filename=%s); - -copy ( - select - * exclude (tp_index, %s), - %s as tp_index - from %s -) to '%s' ( - format parquet, - partition_by (%s), - return_files true, - overwrite_or_ignore, - filename_pattern '%s_{i}', - file_extension 'parquet.tmp' -); -`, - migrateTempTableName, // e.g. "_raw_tp_data" - sourceFileColumnName, // select filename - fileGlob, // parquet file glob path - sourceFileColumnName, // read filename column from parquet - sourceFileColumnName, // exclude source file column from the copy - partition.TpIndexColumn, // replacement tp_index expression - migrateTempTableName, // again used in the copy - baseDir, // output path - strings.Join(partitionColumns, ","), // partition columns - fileRoot, // filename root prefix - ) - - var rowCount int64 - var outputFilesRaw []interface{} - err := db.QueryRowContext(ctx, query).Scan(&rowCount, &outputFilesRaw) - if err != nil { - // if this is a no files found error, we can ignore it - if strings.Contains(err.Error(), "No files found") { - slog.Info("No files found for migration", "partition", partition.UnqualifiedName) - return nil, nil - } - return nil, fmt.Errorf("failed to scan return_files output: %w", err) - } - - outputFiles := make([]string, len(outputFilesRaw)) - for i, val := range outputFilesRaw { - if str, ok := val.(string); ok { - outputFiles[i] = str - } else { - return nil, fmt.Errorf("unexpected file path type %T at index %d", val, i) - } - } - - return outputFiles, nil -} - -// readSourceFiles reads the source files column from the temporary table created during the tp_index migration. -func readSourceFiles(ctx context.Context, db *database.DuckDb) ([]string, error) { - query := fmt.Sprintf(`select distinct %s from %s`, sourceFileColumnName, migrateTempTableName) - rows, err := db.QueryContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to read source files from temp table: %w", err) - } - defer rows.Close() - - var sourceFiles []string - for rows.Next() { - var path string - if err := rows.Scan(&path); err != nil { - return nil, fmt.Errorf("failed to scan source file path: %w", err) - } - sourceFiles = append(sourceFiles, path) - } - return sourceFiles, nil -} +// +//func migrateTpIndexForPartition(ctx context.Context, db *database.DuckDb, baseDir string, partition *config.Partition, fileRootProvider *FileRootProvider, updateFunc func(CompactionStatus)) error { +// +// // executeMigrationQuery runs the DuckDB query to migrate the tp_index files for a given partition. +// // it read the partition data into a temporary table, then writes the data to with the migrated tp_index +// // to intermediate the output files (with extension .tmp) and returns the list of output files. +// outputFiles, err := executeMigrationQuery(ctx, db, baseDir, partition, fileRootProvider) +// if err != nil { +// return err +// } +// if len(outputFiles) == 0 { +// return nil // nothing to migrate +// } +// +// // read the source files from the temporary table +// sourceFiles, err := readSourceFiles(ctx, db) +// if err != nil { +// return err +// } +// +// // now rename the source files to add a .migrated extension +// renamedSourceFiles, err := addExtensionToFiles(sourceFiles, ".migrated") +// if err != nil { +// if err := deleteFilesConcurrently(ctx, outputFiles, baseDir); err != nil { +// slog.Error("Failed to delete temp files after migration failure", "error", err) +// } +// return err +// } +// +// // rename the output files to remove the .tmp extension +// if err := removeExtensionFromFiles(outputFiles, ".tmp"); err != nil { +// if err := deleteFilesConcurrently(ctx, outputFiles, baseDir); err != nil { +// slog.Error("Failed to delete temp files after migration failure", "error", err) +// } +// +// if err := removeExtensionFromFiles(renamedSourceFiles, ".migrated"); err != nil { +// slog.Error("Failed to rename source files back to original names after migration failure", "error", err) +// } +// return err +// } +// +// // finally, delete the renamed source parquet files +// if err := deleteFilesConcurrently(ctx, renamedSourceFiles, baseDir); err != nil { +// slog.Error("Failed to delete renamed source parquet files after migration", "error", err) +// } +// +// status := CompactionStatus{ +// MigrateSource: len(sourceFiles), +// MigrateDest: len(outputFiles), +// PartitionIndexExpressions: map[string]string{ +// partition.UnqualifiedName: partition.TpIndexColumn, +// }, +// } +// updateFunc(status) +// +// return nil +//} +// +//// executeMigrationQuery runs the DuckDB query to migrate the tp_index files for a given partition. +//// It reads the partition data into a temporary table, writes the data with the migrated tp_index +//// to intermediate output files (with .tmp extension), and returns the list of output file paths. +//func executeMigrationQuery(ctx context.Context, db *database.DuckDb, baseDir string, partition *config.Partition, fileRootProvider *FileRootProvider) ([]string, error) { +// // Get the file glob pattern for all files in this partition +// fileGlob := "" //filepaths.GetParquetFileGlobForPartition(baseDir, partition.TableName, partition.ShortName, "") +// +// // get unique file root to use for the output files +// fileRoot := fileRootProvider.GetFileRoot() +// // columns to partition by +// partitionColumns := []string{sdkconstants.TpTable, sdkconstants.TpPartition, sdkconstants.TpIndex, sdkconstants.TpDate} +// +// // build the query to read the parquet files into a temporary table +// query := fmt.Sprintf(` +//create or replace temp table %s as +//select +// *, +// %s, +//from read_parquet('%s', filename=%s); +// +//copy ( +// select +// * exclude (tp_index, %s), +// %s as tp_index +// from %s +//) to '%s' ( +// format parquet, +// partition_by (%s), +// return_files true, +// overwrite_or_ignore, +// filename_pattern '%s_{i}', +// file_extension 'parquet.tmp' +//); +//`, +// migrateTempTableName, // e.g. "_raw_tp_data" +// sourceFileColumnName, // select filename +// fileGlob, // parquet file glob path +// sourceFileColumnName, // read filename column from parquet +// sourceFileColumnName, // exclude source file column from the copy +// partition.TpIndexColumn, // replacement tp_index expression +// migrateTempTableName, // again used in the copy +// baseDir, // output path +// strings.Join(partitionColumns, ","), // partition columns +// fileRoot, // filename root prefix +// ) +// +// var rowCount int64 +// var outputFilesRaw []interface{} +// err := db.QueryRowContext(ctx, query).Scan(&rowCount, &outputFilesRaw) +// if err != nil { +// // if this is a no files found error, we can ignore it +// if strings.Contains(err.Error(), "No files found") { +// slog.Info("No files found for migration", "partition", partition.UnqualifiedName) +// return nil, nil +// } +// return nil, fmt.Errorf("failed to scan return_files output: %w", err) +// } +// +// outputFiles := make([]string, len(outputFilesRaw)) +// for i, val := range outputFilesRaw { +// if str, ok := val.(string); ok { +// outputFiles[i] = str +// } else { +// return nil, fmt.Errorf("unexpected file path type %T at index %d", val, i) +// } +// } +// +// return outputFiles, nil +//} +// +//// readSourceFiles reads the source files column from the temporary table created during the tp_index migration. +//func readSourceFiles(ctx context.Context, db *database.DuckDb) ([]string, error) { +// query := fmt.Sprintf(`select distinct %s from %s`, sourceFileColumnName, migrateTempTableName) +// rows, err := db.QueryContext(ctx, query) +// if err != nil { +// return nil, fmt.Errorf("failed to read source files from temp table: %w", err) +// } +// defer rows.Close() +// +// var sourceFiles []string +// for rows.Next() { +// var path string +// if err := rows.Scan(&path); err != nil { +// return nil, fmt.Errorf("failed to scan source file path: %w", err) +// } +// sourceFiles = append(sourceFiles, path) +// } +// return sourceFiles, nil +//} diff --git a/internal/parquet/partition_key.go b/internal/parquet/partition_key.go new file mode 100644 index 00000000..9b0595bb --- /dev/null +++ b/internal/parquet/partition_key.go @@ -0,0 +1,259 @@ +package parquet + +import ( + "context" + "fmt" + "log/slog" + "sort" + "strings" + "time" + + "github.com/turbot/pipe-fittings/v2/constants" + + "github.com/turbot/tailpipe/internal/database" +) + +// partitionKey is used to uniquely identify a a combination of ducklake partition columns: +// tp_table, tp_partition, tp_index, year(tp_timestamp), month(tp_timestamp) +// It also stores the file and row stats for that partition key +type partitionKey struct { + tpTable string + tpPartition string + tpIndex string + year string // year(tp_timestamp) from partition value + month string // month(tp_timestamp) from partition value + fileCount int // number of files for this partition key + stats partitionKeyStats +} + +// getStats retrieves and populates partition key statistics including row count, max row id, and timestamp range. +// It queries the database to get comprehensive statistics for this partition key and stores them in the partitionKey struct. +func (p *partitionKey) getStats(ctx context.Context, db *database.DuckDb) error { + stats, err := newPartitionKeyStats(ctx, db, p) + if err != nil { + return err + } + + p.stats = *stats + return nil +} + +// query the ducklake_data_file table to get all partition keys combinations which satisfy the provided patterns, +// along with the file and row stats for each partition key combination +func getPartitionKeysMatchingPattern(ctx context.Context, db *database.DuckDb, patterns []PartitionPattern) ([]*partitionKey, error) { + // This query joins the DuckLake metadata tables to get partition key combinations: + // - ducklake_data_file: contains file metadata and links to tables + // - ducklake_file_partition_value: contains partition values for each file + // - ducklake_table: contains table names + // + // The partition key structure is: + // - fpv1 (index 0): tp_partition (e.g., "2024-07") + // - fpv2 (index 1): tp_index (e.g., "index1") + // - fpv3 (index 2): year(tp_timestamp) (e.g., "2024") + // - fpv4 (index 3): month(tp_timestamp) (e.g., "7") + // + // We group by these partition keys and count files per combination, + // filtering for active files (end_snapshot is null) + // NOTE: Assumes partitions are defined in order: tp_partition (0), tp_index (1), year(tp_timestamp) (2), month(tp_timestamp) (3) + query := `select + t.table_name as tp_table, + fpv1.partition_value as tp_partition, + fpv2.partition_value as tp_index, + fpv3.partition_value as year, + fpv4.partition_value as month, + count(*) as file_count +from __ducklake_metadata_tailpipe_ducklake.ducklake_data_file df +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv1 + on df.data_file_id = fpv1.data_file_id and fpv1.partition_key_index = 0 +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv2 + on df.data_file_id = fpv2.data_file_id and fpv2.partition_key_index = 1 +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv3 + on df.data_file_id = fpv3.data_file_id and fpv3.partition_key_index = 2 +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv4 + on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 +join __ducklake_metadata_tailpipe_ducklake.ducklake_table t + on df.table_id = t.table_id +where df.end_snapshot is null +group by + t.table_name, + fpv1.partition_value, + fpv2.partition_value, + fpv3.partition_value, + fpv4.partition_value;` + + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to get partition keys requiring compaction: %w", err) + } + defer rows.Close() + + var partitionKeys []*partitionKey + for rows.Next() { + var partitionKey = &partitionKey{} + + if err := rows.Scan(&partitionKey.tpTable, &partitionKey.tpPartition, &partitionKey.tpIndex, &partitionKey.year, &partitionKey.month, &partitionKey.fileCount); err != nil { + return nil, fmt.Errorf("failed to scan partition key row: %w", err) + } + // check whether this partition key matches any of the provided patterns and whether there are any files + if partitionKey.fileCount > 0 && PartitionMatchesPatterns(partitionKey.tpTable, partitionKey.tpPartition, patterns) { + partitionKeys = append(partitionKeys, partitionKey) + } + } + + // now get the stats for each partition key + for _, pk := range partitionKeys { + // populate the stats for the key + if err := pk.getStats(ctx, db); err != nil { + return nil, fmt.Errorf("failed to get stats for partition key %v: %w", pk, err) + } + } + + return partitionKeys, nil +} + +// getFileCountForPartitionKeys returns the count of parquet files for the provided partition keys +func getFileCountForPartitionKeys(ctx context.Context, db *database.DuckDb, partitionLKeys []*partitionKey) (int, error) { + slog.Info("Getting DuckLake parquet file count for specific partition keys") + + if len(partitionLKeys) == 0 { + return 0, nil + } + + // Build a query to count files only for the specified partition keys + query := fmt.Sprintf(`select count(*) from %s.ducklake_data_file df + join %s.ducklake_file_partition_value fpv1 on df.data_file_id = fpv1.data_file_id and fpv1.partition_key_index = 0 + join %s.ducklake_file_partition_value fpv2 on df.data_file_id = fpv2.data_file_id and fpv2.partition_key_index = 1 + join %s.ducklake_file_partition_value fpv3 on df.data_file_id = fpv3.data_file_id and fpv3.partition_key_index = 2 + join %s.ducklake_file_partition_value fpv4 on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 + where df.end_snapshot is null + and (fpv1.partition_value, fpv2.partition_value, fpv3.partition_value, fpv4.partition_value) in (`, + constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) + + // Build the IN clause with all partition key combinations + var values []string + for _, pk := range partitionLKeys { + value := fmt.Sprintf("('%s', '%s', '%s', '%s')", pk.tpPartition, pk.tpIndex, pk.year, pk.month) + values = append(values, value) + } + + query += strings.Join(values, ", ") + ")" + + var count int + err := db.QueryRowContext(ctx, query).Scan(&count) + if err != nil { + if ctx.Err() != nil { + return 0, err + } + return 0, fmt.Errorf("failed to get parquet file count for partition keys: %w", err) + } + slog.Info("DuckLake parquet file count retrieved for partition keys", "count", count, "partition_keys", len(partitionLKeys)) + return count, nil +} + +type partitionKeyStats struct { + rowCount int64 + maxRowId int64 + minTimestamp time.Time + maxTimestamp time.Time +} + +func newPartitionKeyStats(ctx context.Context, db *database.DuckDb, p *partitionKey) (*partitionKeyStats, error) { + var stats = &partitionKeyStats{} + + // Query to get row count and time range for this partition + countQuery := fmt.Sprintf(`select count(*), max(rowid), min(tp_timestamp), max(tp_timestamp) from "%s" + where tp_partition = ? + and tp_index = ? + and year(tp_timestamp) = ? + and month(tp_timestamp) = ?`, + p.tpTable) + + err := db.QueryRowContext(ctx, countQuery, + p.tpPartition, + p.tpIndex, + p.year, + p.month).Scan(&stats.rowCount, &stats.maxRowId, &stats.minTimestamp, &stats.maxTimestamp) + if err != nil { + return nil, fmt.Errorf("failed to get row count and time range for partition: %w", err) + } + + return stats, nil +} + +// findOverlappingFileRanges finds sets of files that have overlapping time ranges and converts them to unorderedDataTimeRange +func (p *partitionKey) findOverlappingFileRanges(fileRanges []fileTimeRange) ([]unorderedDataTimeRange, error) { + if len(fileRanges) <= 1 { + return []unorderedDataTimeRange{}, nil + } + + // Sort by start time - O(n log n) + sort.Slice(fileRanges, func(i, j int) bool { + return fileRanges[i].min.Before(fileRanges[j].min) + }) + + var unorderedRanges []unorderedDataTimeRange + processedFiles := make(map[string]struct{}) + + for i, currentFile := range fileRanges { + if _, processed := processedFiles[currentFile.path]; processed { + continue + } + + // Find all files that overlap with this one + overlappingFiles := p.findFilesOverlappingWith(currentFile, fileRanges[i+1:], processedFiles) + + // Only keep sets with multiple files (single files don't need compaction) + if len(overlappingFiles) > 1 { + // Convert overlapping files to unorderedDataTimeRange + timeRange, err := newUnorderedDataTimeRange(overlappingFiles) + if err != nil { + return nil, fmt.Errorf("failed to create unordered time range: %w", err) + } + unorderedRanges = append(unorderedRanges, timeRange) + } + } + + return unorderedRanges, nil +} + +// findFilesOverlappingWith finds all files that overlap with the given file +func (p *partitionKey) findFilesOverlappingWith(startFile fileTimeRange, remainingFiles []fileTimeRange, processedFiles map[string]struct{}) []fileTimeRange { + overlappingFileRanges := []fileTimeRange{startFile} + processedFiles[startFile.path] = struct{}{} + setMaxEnd := startFile.max + + for _, candidateFile := range remainingFiles { + if _, processed := processedFiles[candidateFile.path]; processed { + continue + } + + // Early termination: if candidate starts after set ends, no more overlaps + if candidateFile.min.After(setMaxEnd) { + break + } + + // Check if this file overlaps with any file in our set + if p.fileOverlapsWithSet(candidateFile, overlappingFileRanges) { + overlappingFileRanges = append(overlappingFileRanges, candidateFile) + processedFiles[candidateFile.path] = struct{}{} + + // Update set's max end time + if candidateFile.max.After(setMaxEnd) { + setMaxEnd = candidateFile.max + } + } + } + + return overlappingFileRanges +} + +// fileOverlapsWithSet checks if a file overlaps with any file in the set +func (p *partitionKey) fileOverlapsWithSet(candidateFile fileTimeRange, fileSet []fileTimeRange) bool { + for _, setFile := range fileSet { + if rangesOverlap(setFile, candidateFile) { + return true + } + } + return false +} diff --git a/internal/parquet/partition_key_test.go b/internal/parquet/partition_key_test.go new file mode 100644 index 00000000..1774b0c7 --- /dev/null +++ b/internal/parquet/partition_key_test.go @@ -0,0 +1,383 @@ +package parquet + +import ( + "testing" + "time" +) + +// timeString is a helper function to create time.Time from string +func timeString(timeStr string) time.Time { + t, err := time.Parse("2006-01-02 15:04:05", timeStr) + if err != nil { + panic(err) + } + return t +} + +func TestPartitionKeyRangeOperations(t *testing.T) { + pk := &partitionKey{} + + tests := []struct { + name string + testType string // "rangesOverlap", "findOverlappingFileRanges", "newUnorderedDataTimeRange" + input interface{} + expected interface{} + }{ + // Test cases for rangesOverlap function + { + name: "rangesOverlap - overlapping ranges", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - non-overlapping ranges", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-03 00:00:00"), max: timeString("2024-01-04 00:00:00")}, + }, + expected: false, + }, + { + name: "rangesOverlap - touching ranges (contiguous, not overlapping)", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: false, + }, + { + name: "rangesOverlap - identical ranges", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - partial overlap", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 12:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - one range completely inside another", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-05 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - ranges with same start time", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - ranges with same end time", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-02 00:00:00")}, + }, + expected: true, + }, + + // Test cases for findOverlappingFileRanges function + { + name: "findOverlappingFileRanges - no overlaps", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-03 00:00:00"), max: timeString("2024-01-04 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-05 00:00:00"), max: timeString("2024-01-06 00:00:00"), rowCount: 1500}, + }, + expected: []unorderedDataTimeRange{}, + }, + { + name: "findOverlappingFileRanges - simple overlap", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 3000, + }, + }, + }, + { + name: "findOverlappingFileRanges - cross-overlapping sets", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-02 12:00:00"), max: timeString("2024-01-04 00:00:00"), rowCount: 1500}, + {path: "file4", min: timeString("2024-01-03 12:00:00"), max: timeString("2024-01-05 00:00:00"), rowCount: 1800}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-05 00:00:00"), + RowCount: 6300, + }, + }, + }, + { + name: "findOverlappingFileRanges - multiple separate groups", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-05 00:00:00"), max: timeString("2024-01-06 00:00:00"), rowCount: 1500}, + {path: "file4", min: timeString("2024-01-05 12:00:00"), max: timeString("2024-01-07 00:00:00"), rowCount: 1800}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 3000, + }, + { + StartTime: timeString("2024-01-05 00:00:00"), + EndTime: timeString("2024-01-07 00:00:00"), + RowCount: 3300, + }, + }, + }, + { + name: "findOverlappingFileRanges - single file", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + }, + expected: []unorderedDataTimeRange{}, + }, + { + name: "findOverlappingFileRanges - empty input", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{}, + expected: []unorderedDataTimeRange{}, + }, + { + name: "findOverlappingFileRanges - three overlapping files", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-02 12:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 1500}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 4500, + }, + }, + }, + { + name: "findOverlappingFileRanges - files with identical time ranges", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 2000}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-02 00:00:00"), + RowCount: 3000, + }, + }, + }, + + // Test cases for newUnorderedDataTimeRange function + { + name: "newUnorderedDataTimeRange - single file", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-02 00:00:00"), + RowCount: 1000, + }, + }, + { + name: "newUnorderedDataTimeRange - multiple overlapping files", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-04 00:00:00"), rowCount: 1500}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), // earliest start + EndTime: timeString("2024-01-04 00:00:00"), // latest end + RowCount: 4500, // sum of all row counts + }, + }, + { + name: "newUnorderedDataTimeRange - files with zero row counts", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 0}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 1000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 1000, + }, + }, + { + name: "newUnorderedDataTimeRange - files with same start time", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 3000, + }, + }, + { + name: "newUnorderedDataTimeRange - files with same end time", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 2000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-02 00:00:00"), + RowCount: 3000, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + switch tt.testType { + case "rangesOverlap": + input := tt.input.(struct { + r1 fileTimeRange + r2 fileTimeRange + }) + result := rangesOverlap(input.r1, input.r2) + expected := tt.expected.(bool) + if result != expected { + t.Errorf("rangesOverlap() = %v, expected %v", result, expected) + } + + case "findOverlappingFileRanges": + input := tt.input.([]fileTimeRange) + expected := tt.expected.([]unorderedDataTimeRange) + result, err := pk.findOverlappingFileRanges(input) + if err != nil { + t.Fatalf("findOverlappingFileRanges() error = %v", err) + } + if !compareUnorderedRangesets(result, expected) { + t.Errorf("findOverlappingFileRanges() = %v, expected %v", result, expected) + } + + case "newUnorderedDataTimeRange": + input := tt.input.([]fileTimeRange) + expected := tt.expected.(unorderedDataTimeRange) + result, err := newUnorderedDataTimeRange(input) + if err != nil { + t.Fatalf("newUnorderedDataTimeRange() error = %v", err) + } + if !result.StartTime.Equal(expected.StartTime) { + t.Errorf("StartTime = %v, expected %v", result.StartTime, expected.StartTime) + } + if !result.EndTime.Equal(expected.EndTime) { + t.Errorf("EndTime = %v, expected %v", result.EndTime, expected.EndTime) + } + if result.RowCount != expected.RowCount { + t.Errorf("RowCount = %v, expected %v", result.RowCount, expected.RowCount) + } + } + }) + } +} + +// compareUnorderedRangesets compares two slices of unorderedDataTimeRange, ignoring order +func compareUnorderedRangesets(actual []unorderedDataTimeRange, expected []unorderedDataTimeRange) bool { + if len(actual) != len(expected) { + return false + } + + // Convert to sets for comparison using time range as key + actualSets := make(map[string]unorderedDataTimeRange) + expectedSets := make(map[string]unorderedDataTimeRange) + + for _, set := range actual { + key := set.StartTime.Format("2006-01-02 15:04:05") + "-" + set.EndTime.Format("2006-01-02 15:04:05") + actualSets[key] = set + } + + for _, set := range expected { + key := set.StartTime.Format("2006-01-02 15:04:05") + "-" + set.EndTime.Format("2006-01-02 15:04:05") + expectedSets[key] = set + } + + // Check if each set in actual has a matching set in expected + for key, actualSet := range actualSets { + expectedSet, exists := expectedSets[key] + if !exists || !unorderedRangesetsEqual(actualSet, expectedSet) { + return false + } + } + + return true +} + +// unorderedRangesetsEqual compares two unorderedDataTimeRange structs +func unorderedRangesetsEqual(a, b unorderedDataTimeRange) bool { + return a.StartTime.Equal(b.StartTime) && a.EndTime.Equal(b.EndTime) && a.RowCount == b.RowCount +} diff --git a/internal/parquet/read_json_query.go b/internal/parquet/read_json_query.go new file mode 100644 index 00000000..b1031998 --- /dev/null +++ b/internal/parquet/read_json_query.go @@ -0,0 +1,176 @@ +package parquet + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/turbot/go-kit/helpers" + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe/internal/config" +) + +// buildReadJsonQueryFormat creates a SQL query template for reading JSONL files with DuckDB. +// +// Returns a format string with a %s placeholder for the JSON filename that gets filled in when executed. +// The query is built by constructing a select clause for each field in the conversion schema, +// adding tp_index from partition config, and applying any partition filters (e.g. date filer) +// +// Example output: +// +// select "user_id" as "user_id", "name" as "user_name", "created_at" as "tp_timestamp", +// "default" as "tp_index" +// from read_ndjson(%s, columns = {"user_id": 'varchar', "name": 'varchar', "created_at": 'timestamp'}) +func buildReadJsonQueryFormat(conversionSchema *schema.ConversionSchema, partition *config.Partition) string { + var tpTimestampMapped bool + + // first build the select clauses - use the table def columns + var selectClauses []string + for _, column := range conversionSchema.Columns { + var selectClause string + switch column.ColumnName { + case constants.TpDate: + // skip this column - it is derived from tp_timestamp + continue + case constants.TpIndex: + // NOTE: we ignore tp_index in the source data and ONLY add it based ont he default or configured value + slog.Warn("tp_index is a reserved column name and should not be used in the source data. It will be added automatically based on the configured value.") + // skip this column - it will be populated manually using the partition config + continue + case constants.TpTimestamp: + tpTimestampMapped = true + // fallthrough to populate the select clasue as normal + fallthrough + default: + selectClause = getSelectSqlForField(column) + } + + selectClauses = append(selectClauses, selectClause) + } + + // add the tp_index - this is determined by the partition - it defaults to "default" but may be overridden in the partition config + // NOTE: we DO NOT wrap the tp_index expression in quotes - that will have already been done as part of partition config validation + selectClauses = append(selectClauses, fmt.Sprintf("\t%s as \"tp_index\"", partition.TpIndexColumn)) + + // if we have a mapping for tp_timestamp, add tp_date as well + // (if we DO NOT have tp_timestamp, the validation will fail - but we want the validation error - + // NOT an error when we try to select tp_date using tp_timestamp as source) + if tpTimestampMapped { + // Add tp_date after tp_timestamp is defined + selectClauses = append(selectClauses, ` case + when tp_timestamp is not null then date_trunc('day', tp_timestamp::timestamp) + end as tp_date`) + } + + // build column definitions - these will be passed to the read_json function + columnDefinitions := getReadJSONColumnDefinitions(conversionSchema.SourceColumns) + + var whereClause string + if partition.Filter != "" { + // we need to escape the % in the filter, as it is passed to the fmt.Sprintf function + filter := strings.ReplaceAll(partition.Filter, "%", "%%") + whereClause = fmt.Sprintf("\nwhere %s", filter) + } + + res := fmt.Sprintf(`select +%s +from + read_ndjson( + %%s, + %s + )%s`, strings.Join(selectClauses, ",\n"), helpers.Tabify(columnDefinitions, "\t"), whereClause) + + return res +} + +// return the column definitions for the row conversionSchema, in the format required for the duck db read_json_auto function +func getReadJSONColumnDefinitions(sourceColumns []schema.SourceColumnDef) string { + var str strings.Builder + str.WriteString("columns = {") + for i, column := range sourceColumns { + if i > 0 { + str.WriteString(", ") + } + str.WriteString(fmt.Sprintf(` + "%s": '%s'`, column.Name, column.Type)) + } + str.WriteString("\n}") + return str.String() +} + +// getSelectSqlForField builds a SELECT clause for a single field based on its schema definition. +// - If the field has a transform defined, it uses that transform expression. +// - For struct fields, it creates a struct_pack expression to properly construct the nested structure from the source JSON data. +// - All other field types are handled with simple column references. +func getSelectSqlForField(column *schema.ColumnSchema) string { + + // If the column has a transform, use it + if column.Transform != "" { + // as this is going into a string format, we need to escape % + escapedTransform := strings.ReplaceAll(column.Transform, "%", "%%") + return fmt.Sprintf("\t%s as \"%s\"", escapedTransform, column.ColumnName) + } + + // NOTE: we will have normalised column types to lower case + switch column.Type { + case "struct": + var str strings.Builder + + // Start case logic to handle null values for the struct + + str.WriteString(fmt.Sprintf("\tcase\n\t\twhen \"%s\" is null then null\n", column.SourceName)) + str.WriteString("\t\telse struct_pack(\n") + + // Add nested fields to the struct_pack + for j, nestedColumn := range column.StructFields { + if j > 0 { + str.WriteString(",\n") + } + parentName := fmt.Sprintf("\"%s\"", column.SourceName) + str.WriteString(getTypeSqlForStructField(nestedColumn, parentName, 3)) + } + + // Close struct_pack and case + str.WriteString("\n\t\t)\n") + str.WriteString(fmt.Sprintf("\tend as \"%s\"", column.ColumnName)) + return str.String() + default: + // Scalar fields + return fmt.Sprintf("\t\"%s\" as \"%s\"", column.SourceName, column.ColumnName) + } +} + +// Return the SQL line to pack the given field as a struct +func getTypeSqlForStructField(column *schema.ColumnSchema, parentName string, tabs int) string { + tab := strings.Repeat("\t", tabs) + + switch column.Type { + case "struct": + var str strings.Builder + + // Add case logic to handle null values for the struct + str.WriteString(fmt.Sprintf("%s\"%s\" := case\n", tab, column.ColumnName)) + str.WriteString(fmt.Sprintf("%s\twhen %s.\"%s\" is null then null\n", tab, parentName, column.SourceName)) + str.WriteString(fmt.Sprintf("%s\telse struct_pack(\n", tab)) + + // Loop through nested fields and add them to the struct_pack + for j, nestedColumn := range column.StructFields { + if j > 0 { + str.WriteString(",\n") + } + // Use the current field as the new parent for recursion + newParent := fmt.Sprintf("%s.\"%s\"", parentName, column.SourceName) + str.WriteString(getTypeSqlForStructField(nestedColumn, newParent, tabs+2)) + } + + // Close struct_pack and case + str.WriteString(fmt.Sprintf("\n%s\t)\n", tab)) + str.WriteString(fmt.Sprintf("%send", tab)) + return str.String() + + default: + // Scalar fields + return fmt.Sprintf("%s\"%s\" := %s.\"%s\"::%s", tab, column.ColumnName, parentName, column.SourceName, column.Type) + } +} diff --git a/internal/parquet/convertor_schema_test.go b/internal/parquet/read_json_query_test.go similarity index 100% rename from internal/parquet/convertor_schema_test.go rename to internal/parquet/read_json_query_test.go diff --git a/internal/parquet/schema_comparison.go b/internal/parquet/schema_comparison.go new file mode 100644 index 00000000..0b7c2588 --- /dev/null +++ b/internal/parquet/schema_comparison.go @@ -0,0 +1,64 @@ +package parquet + +import ( + "fmt" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "strings" +) + +type TableSchemaStatus struct { + TableExists bool + SchemaMatches bool + CanMigrate bool + SchemaDiff string +} + +// TODO #DL check if we need this https://github.com/turbot/tailpipe/issues/481 + +func NewTableSchemaStatusFromComparison(existingSchema map[string]schema.ColumnSchema, conversionSchema schema.ConversionSchema) TableSchemaStatus { + var diffParts []string + canMigrate := true + + // Create map of new schema for quick lookup + newSchemaMap := make(map[string]*schema.ColumnSchema) + for _, column := range conversionSchema.Columns { + newSchemaMap[column.ColumnName] = column + } + + // Check for removed columns + for existingColName := range existingSchema { + if _, exists := newSchemaMap[existingColName]; !exists { + diffParts = append(diffParts, fmt.Sprintf("- column %s removed", existingColName)) + canMigrate = false + } + } + + // Check for new/modified columns + hasNewColumns := false + for _, column := range conversionSchema.Columns { + existingCol, ok := existingSchema[column.ColumnName] + if !ok { + diffParts = append(diffParts, fmt.Sprintf("+ column %s added (%s)", column.ColumnName, column.Type)) + hasNewColumns = true + continue + } + + if existingCol.Type != column.Type { + diffParts = append(diffParts, fmt.Sprintf("~ column %s type changed: %s → %s", + column.ColumnName, existingCol.Type, column.Type)) + canMigrate = false + } + } + + matches := len(diffParts) == 0 + if !matches && canMigrate { + canMigrate = hasNewColumns // Only true if we only have additive changes + } + + return TableSchemaStatus{ + TableExists: true, + SchemaMatches: matches, + CanMigrate: canMigrate, + SchemaDiff: strings.Join(diffParts, "\n"), + } +} diff --git a/internal/parse/load_config_test.go b/internal/parse/load_config_test.go index f442f9f9..cb600da0 100644 --- a/internal/parse/load_config_test.go +++ b/internal/parse/load_config_test.go @@ -1,155 +1,871 @@ package parse -// TODO enable and fix this test -//func TestLoadTailpipeConfig(t *testing.T) { -// type args struct { -// configPath string -// partition string -// } -// tests := []struct { -// name string -// args args -// want *config.TailpipeConfig -// wantErr bool -// }{ -// // TODO #testing add more test cases -// { -// name: "static tables", -// args: args{ -// configPath: "test_data/static_table_config", -// partition: "partition.aws_cloudtrail_log.cloudtrail_logs", -// }, -// want: &config.TailpipeConfig{ -// PluginVersions: nil, -// Partitions: map[string]*config.Partition{ -// "partition.aws_cloudtrail_log.cloudtrail_logs": {}, -// "partition.aws_vpc_flow_log.flow_logs": {}, -// }, -// }, -// -// wantErr: false, -// }, -// { -// name: "dynamic tables", -// args: args{ -// configPath: "test_data/custom_table_config", -// partition: "partition.aws_cloudtrail_log.cloudtrail_logs", -// }, -// want: &config.TailpipeConfig{ -// Partitions: map[string]*config.Partition{ -// "my_csv_log.test": { -// HclResourceImpl: modconfig.HclResourceImpl{ -// FullName: "partition.my_csv_log.test", -// ShortName: "test", -// UnqualifiedName: "my_csv_log.test", -// DeclRange: hcl.Range{ -// Filename: "test_data/custom_table_config/resources.tpc", -// Start: hcl.Pos{ -// Line: 2, -// Column: 30, -// Byte: 30, -// }, -// End: hcl.Pos{ -// Line: 10, -// Column: 2, -// Byte: 230, -// }, -// }, -// BlockType: "partition", -// }, -// TableName: "my_csv_log", -// Plugin: &plugin.Plugin{ -// Instance: "custom", -// Alias: "custom", -// Plugin: "/plugins/turbot/custom@latest", -// }, -// Source: config.Source{ -// Type: "file_system", -// Config: &config.HclBytes{ -// Hcl: []byte("extensions = [\".csv\"]\npaths = [\"/Users/kai/tailpipe_data/logs\"]"), -// Range: hclhelpers.NewRange(hcl.Range{ -// Filename: "test_data/custom_table_config/resources.tpc", -// Start: hcl.Pos{ -// Line: 4, -// Column: 9, -// Byte: 68, -// }, -// End: hcl.Pos{ -// Line: 5, -// Column: 30, -// Byte: 139, -// }, -// }), -// }, -// }, -// }, -// }, -// CustomTables: map[string]*config.Table{ -// "my_csv_log": { -// HclResourceImpl: modconfig.HclResourceImpl{ -// FullName: "partition.my_csv_log.test", -// ShortName: "test", -// UnqualifiedName: "my_csv_log.test", -// DeclRange: hcl.Range{ -// Filename: "test_data/custom_table_config/resources.tpc", -// Start: hcl.Pos{ -// Line: 2, -// Column: 30, -// Byte: 30, -// }, -// End: hcl.Pos{ -// Line: 10, -// Column: 2, -// Byte: 230, -// }, -// }, -// BlockType: "partition", -// }, -// //Mode: schema.ModePartial, -// Columns: []config.ColumnSchema{ -// { -// Name: "tp_timestamp", -// Source: utils.ToPointer("time_local"), -// }, -// { -// Name: "tp_index", -// Source: utils.ToPointer("account_id"), -// }, -// { -// Name: "org_id", -// Source: utils.ToPointer("org"), -// }, -// { -// Name: "user_id", -// Type: utils.ToPointer("varchar"), -// }, -// }, -// }, -// }, -// }, -// -// wantErr: false, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// tailpipeDir, er := filepath.Abs(tt.args.configPath) -// if er != nil { -// t.Errorf("failed to build absolute config filepath from %s", tt.args.configPath) -// } -// // set app_specific.InstallDir -// app_specific.InstallDir = tailpipeDir -// -// tailpipeConfig, err := parseTailpipeConfig(tt.args.configPath) -// if (err != nil) != tt.wantErr { -// t.Errorf("LoadTailpipeConfig() error = %v, wantErr %v", err, tt.wantErr) -// return -// } -// -// if !reflect.DeepEqual(tailpipeConfig, tt.want) { -// t.Errorf("LoadTailpipeConfig() = %v, want %v", tailpipeConfig, tt.want) -// } -// }) -// } -//} +import ( + "fmt" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/v2/app_specific" + "github.com/turbot/pipe-fittings/v2/hclhelpers" + "github.com/turbot/pipe-fittings/v2/modconfig" + "github.com/turbot/pipe-fittings/v2/plugin" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/pipe-fittings/v2/versionfile" + "github.com/turbot/tailpipe/internal/config" +) + +func pluginVersionsEqual(l, r map[string]*versionfile.InstalledVersion) (bool, string) { + if (l == nil) != (r == nil) { + return false, "PluginVersions presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("PluginVersions length mismatch: got %d want %d", len(l), len(r)) + } + for k, v := range l { + wv, ok := r[k] + if !ok { + return false, fmt.Sprintf("PluginVersions missing key '%s' in want", k) + } + if (v == nil) != (wv == nil) { + return false, fmt.Sprintf("PluginVersions['%s'] presence mismatch", k) + } + if v != nil { + if v.Name != wv.Name { + return false, fmt.Sprintf("PluginVersions['%s'].Name mismatch: got '%s' want '%s'", k, v.Name, wv.Name) + } + if v.Version != wv.Version { + return false, fmt.Sprintf("PluginVersions['%s'].Version mismatch: got '%s' want '%s'", k, v.Version, wv.Version) + } + if v.ImageDigest != wv.ImageDigest { + return false, fmt.Sprintf("PluginVersions['%s'].ImageDigest mismatch: got '%s' want '%s'", k, v.ImageDigest, wv.ImageDigest) + } + if v.BinaryDigest != wv.BinaryDigest { + return false, fmt.Sprintf("PluginVersions['%s'].BinaryDigest mismatch: got '%s' want '%s'", k, v.BinaryDigest, wv.BinaryDigest) + } + if v.BinaryArchitecture != wv.BinaryArchitecture { + return false, fmt.Sprintf("PluginVersions['%s'].BinaryArchitecture mismatch: got '%s' want '%s'", k, v.BinaryArchitecture, wv.BinaryArchitecture) + } + if v.InstalledFrom != wv.InstalledFrom { + return false, fmt.Sprintf("PluginVersions['%s'].InstalledFrom mismatch: got '%s' want '%s'", k, v.InstalledFrom, wv.InstalledFrom) + } + if v.StructVersion != wv.StructVersion { + return false, fmt.Sprintf("PluginVersions['%s'].StructVersion mismatch: got '%d' want '%d'", k, v.StructVersion, wv.StructVersion) + } + if (v.Metadata == nil) != (wv.Metadata == nil) { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata presence mismatch", k) + } + if v.Metadata != nil { + if len(v.Metadata) != len(wv.Metadata) { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata length mismatch", k) + } + for mk, ma := range v.Metadata { + mb, ok := wv.Metadata[mk] + if !ok { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata missing key '%s'", k, mk) + } + if len(ma) != len(mb) { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata['%s'] length mismatch", k, mk) + } + maCopy, mbCopy := append([]string(nil), ma...), append([]string(nil), mb...) + sort.Strings(maCopy) + sort.Strings(mbCopy) + for i := range maCopy { + if maCopy[i] != mbCopy[i] { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata['%s'][%d] mismatch: got '%s' want '%s'", k, mk, i, maCopy[i], mbCopy[i]) + } + } + } + } + } + } + return true, "" +} + +func connectionsEqual(l, r map[string]*config.TailpipeConnection) (bool, string) { + if (l == nil) != (r == nil) { + return false, "Connections presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("Connections length mismatch: got %d want %d", len(l), len(r)) + } + for k, conn := range l { + wconn, ok := r[k] + if !ok { + return false, fmt.Sprintf("Connections missing key '%s' in want", k) + } + if (conn == nil) != (wconn == nil) { + return false, fmt.Sprintf("Connections['%s'] presence mismatch", k) + } + if conn != nil { + if conn.HclResourceImpl.FullName != wconn.HclResourceImpl.FullName { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.FullName, wconn.HclResourceImpl.FullName) + } + if conn.HclResourceImpl.ShortName != wconn.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.ShortName, wconn.HclResourceImpl.ShortName) + } + if conn.HclResourceImpl.UnqualifiedName != wconn.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.UnqualifiedName, wconn.HclResourceImpl.UnqualifiedName) + } + if conn.HclResourceImpl.BlockType != wconn.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.BlockType, wconn.HclResourceImpl.BlockType) + } + if conn.Plugin != wconn.Plugin { + return false, fmt.Sprintf("Connections['%s'].Plugin mismatch: got '%s' want '%s'", k, conn.Plugin, wconn.Plugin) + } + zero := hclhelpers.Range{} + connZero := conn.HclRange == zero + wconnZero := wconn.HclRange == zero + if connZero != wconnZero { + return false, fmt.Sprintf("Connections['%s'].HclRange presence mismatch", k) + } + if !connZero && !wconnZero { + if !reflect.DeepEqual(conn.HclRange, wconn.HclRange) { + gr, wr := conn.HclRange, wconn.HclRange + return false, fmt.Sprintf("Connections['%s'].HclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + } + return true, "" +} + +func customTablesEqual(l, r map[string]*config.Table) (bool, string) { + if (l == nil) != (r == nil) { + return false, "CustomTables presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("CustomTables length mismatch: got %d want %d", len(l), len(r)) + } + for k, ct := range l { + wct, ok := r[k] + if !ok { + return false, fmt.Sprintf("CustomTables missing key '%s' in want", k) + } + if (ct == nil) != (wct == nil) { + return false, fmt.Sprintf("CustomTables['%s'] presence mismatch", k) + } + if ct != nil { + if ct.HclResourceImpl.FullName != wct.HclResourceImpl.FullName { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.FullName, wct.HclResourceImpl.FullName) + } + if ct.HclResourceImpl.ShortName != wct.HclResourceImpl.ShortName { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.ShortName, wct.HclResourceImpl.ShortName) + } + if ct.HclResourceImpl.UnqualifiedName != wct.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.UnqualifiedName, wct.HclResourceImpl.UnqualifiedName) + } + if ct.HclResourceImpl.BlockType != wct.HclResourceImpl.BlockType { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.BlockType, wct.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := ct.HclResourceImpl.DeclRange == zero + bZero := wct.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(ct.HclResourceImpl.DeclRange, wct.HclResourceImpl.DeclRange) { + gr, wr := ct.HclResourceImpl.DeclRange, wct.HclResourceImpl.DeclRange + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + if ct.DefaultSourceFormat != nil && wct.DefaultSourceFormat != nil { + if ct.DefaultSourceFormat.Type != wct.DefaultSourceFormat.Type { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.Type mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.Type, wct.DefaultSourceFormat.Type) + } + if ct.DefaultSourceFormat.PresetName != wct.DefaultSourceFormat.PresetName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.PresetName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.PresetName, wct.DefaultSourceFormat.PresetName) + } + if ct.DefaultSourceFormat.HclResourceImpl.FullName != wct.DefaultSourceFormat.HclResourceImpl.FullName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.FullName, wct.DefaultSourceFormat.HclResourceImpl.FullName) + } + if ct.DefaultSourceFormat.HclResourceImpl.ShortName != wct.DefaultSourceFormat.HclResourceImpl.ShortName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.ShortName, wct.DefaultSourceFormat.HclResourceImpl.ShortName) + } + if ct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName != wct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName, wct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName) + } + if ct.DefaultSourceFormat.HclResourceImpl.BlockType != wct.DefaultSourceFormat.HclResourceImpl.BlockType { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.BlockType, wct.DefaultSourceFormat.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := ct.DefaultSourceFormat.HclResourceImpl.DeclRange == zero + bZero := wct.DefaultSourceFormat.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(ct.DefaultSourceFormat.HclResourceImpl.DeclRange, wct.DefaultSourceFormat.HclResourceImpl.DeclRange) { + gr, wr := ct.DefaultSourceFormat.HclResourceImpl.DeclRange, wct.DefaultSourceFormat.HclResourceImpl.DeclRange + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + } + if len(ct.Columns) != len(wct.Columns) { + return false, fmt.Sprintf("CustomTables['%s'].Columns length mismatch: got %d want %d", k, len(ct.Columns), len(wct.Columns)) + } + for i := range ct.Columns { + ac, bc := ct.Columns[i], wct.Columns[i] + if ac.Name != bc.Name { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Name mismatch: got '%s' want '%s'", k, i, ac.Name, bc.Name) + } + if ac.Type != nil && bc.Type != nil && *ac.Type != *bc.Type { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Type mismatch: got '%s' want '%s'", k, i, *ac.Type, *bc.Type) + } + if ac.Source != nil && bc.Source != nil && *ac.Source != *bc.Source { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Source mismatch: got '%s' want '%s'", k, i, *ac.Source, *bc.Source) + } + if ac.Description != nil && bc.Description != nil && *ac.Description != *bc.Description { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Description mismatch", k, i) + } + if ac.Required != nil && bc.Required != nil && *ac.Required != *bc.Required { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Required mismatch", k, i) + } + if ac.NullIf != nil && bc.NullIf != nil && *ac.NullIf != *bc.NullIf { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].NullIf mismatch", k, i) + } + if ac.Transform != nil && bc.Transform != nil && *ac.Transform != *bc.Transform { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Transform mismatch", k, i) + } + } + mfA := append([]string(nil), ct.MapFields...) + if len(mfA) == 0 { + mfA = []string{"*"} + } + mfB := append([]string(nil), wct.MapFields...) + if len(mfB) == 0 { + mfB = []string{"*"} + } + sort.Strings(mfA) + sort.Strings(mfB) + if len(mfA) != len(mfB) { + return false, fmt.Sprintf("CustomTables['%s'].MapFields length mismatch: got %d want %d", k, len(mfA), len(mfB)) + } + for i := range mfA { + if mfA[i] != mfB[i] { + return false, fmt.Sprintf("CustomTables['%s'].MapFields[%d] mismatch: got '%s' want '%s'", k, i, mfA[i], mfB[i]) + } + } + if ct.NullIf != wct.NullIf { + return false, fmt.Sprintf("CustomTables['%s'].NullIf mismatch: got '%s' want '%s'", k, ct.NullIf, wct.NullIf) + } + } + } + return true, "" +} + +func formatsEqual(l, r map[string]*config.Format) (bool, string) { + if (l == nil) != (r == nil) { + return false, "Formats presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("Formats length mismatch: got %d want %d", len(l), len(r)) + } + for k, f := range l { + wf, ok := r[k] + if !ok { + return false, fmt.Sprintf("Formats missing key '%s' in want", k) + } + if (f == nil) != (wf == nil) { + return false, fmt.Sprintf("Formats['%s'] presence mismatch", k) + } + if f != nil { + if f.Type != wf.Type { + return false, fmt.Sprintf("Formats['%s'].Type mismatch: got '%s' want '%s'", k, f.Type, wf.Type) + } + if f.HclResourceImpl.FullName != wf.HclResourceImpl.FullName { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, f.HclResourceImpl.FullName, wf.HclResourceImpl.FullName) + } + if f.HclResourceImpl.ShortName != wf.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, f.HclResourceImpl.ShortName, wf.HclResourceImpl.ShortName) + } + if f.HclResourceImpl.UnqualifiedName != wf.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, f.HclResourceImpl.UnqualifiedName, wf.HclResourceImpl.UnqualifiedName) + } + if f.HclResourceImpl.BlockType != wf.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, f.HclResourceImpl.BlockType, wf.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := f.HclResourceImpl.DeclRange == zero + bZero := wf.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(f.HclResourceImpl.DeclRange, wf.HclResourceImpl.DeclRange) { + gr, wr := f.HclResourceImpl.DeclRange, wf.HclResourceImpl.DeclRange + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + if f.PresetName != "" && wf.PresetName != "" && f.PresetName != wf.PresetName { + return false, fmt.Sprintf("Formats['%s'].PresetName mismatch: got '%s' want '%s'", k, f.PresetName, wf.PresetName) + } + } + } + return true, "" +} + +func partitionsEqual(l, r map[string]*config.Partition) (bool, string) { + if (l == nil) != (r == nil) { + return false, "Partitions presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("Partitions length mismatch: got %d want %d", len(l), len(r)) + } + for k, p := range l { + wp, ok := r[k] + if !ok { + return false, fmt.Sprintf("Partitions missing key '%s' in want", k) + } + if (p == nil) != (wp == nil) { + return false, fmt.Sprintf("Partitions['%s'] presence mismatch", k) + } + if p != nil { + if p.HclResourceImpl.FullName != wp.HclResourceImpl.FullName { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, p.HclResourceImpl.FullName, wp.HclResourceImpl.FullName) + } + if p.HclResourceImpl.ShortName != wp.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, p.HclResourceImpl.ShortName, wp.HclResourceImpl.ShortName) + } + if p.HclResourceImpl.UnqualifiedName != wp.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, p.HclResourceImpl.UnqualifiedName, wp.HclResourceImpl.UnqualifiedName) + } + if p.HclResourceImpl.BlockType != wp.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, p.HclResourceImpl.BlockType, wp.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := p.HclResourceImpl.DeclRange == zero + bZero := wp.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(p.HclResourceImpl.DeclRange, wp.HclResourceImpl.DeclRange) { + gr, wr := p.HclResourceImpl.DeclRange, wp.HclResourceImpl.DeclRange + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + if p.TableName != wp.TableName { + return false, fmt.Sprintf("Partitions['%s'].TableName mismatch: got '%s' want '%s'", k, p.TableName, wp.TableName) + } + if p.Source.Type != wp.Source.Type { + return false, fmt.Sprintf("Partitions['%s'].Source.Type mismatch: got '%s' want '%s'", k, p.Source.Type, wp.Source.Type) + } + if (p.Source.Connection == nil) != (wp.Source.Connection == nil) { + return false, fmt.Sprintf("Partitions['%s'].Source.Connection presence mismatch", k) + } + if p.Source.Connection != nil && wp.Source.Connection != nil { + if p.Source.Connection.HclResourceImpl.UnqualifiedName != wp.Source.Connection.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Partitions['%s'].Source.Connection.HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, p.Source.Connection.HclResourceImpl.UnqualifiedName, wp.Source.Connection.HclResourceImpl.UnqualifiedName) + } + } + if (p.Source.Format == nil) != (wp.Source.Format == nil) { + return false, fmt.Sprintf("Partitions['%s'].Source.Format presence mismatch", k) + } + if p.Source.Format != nil && wp.Source.Format != nil { + pf, of := p.Source.Format, wp.Source.Format + if pf.Type != of.Type { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.Type mismatch: got '%s' want '%s'", k, pf.Type, of.Type) + } + if pf.PresetName != of.PresetName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.PresetName mismatch: got '%s' want '%s'", k, pf.PresetName, of.PresetName) + } + if pf.HclResourceImpl.FullName != of.HclResourceImpl.FullName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.FullName, of.HclResourceImpl.FullName) + } + if pf.HclResourceImpl.ShortName != of.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.ShortName, of.HclResourceImpl.ShortName) + } + if pf.HclResourceImpl.UnqualifiedName != of.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.UnqualifiedName, of.HclResourceImpl.UnqualifiedName) + } + if pf.HclResourceImpl.BlockType != of.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.BlockType, of.HclResourceImpl.BlockType) + } + } + if (p.Source.Config == nil) != (wp.Source.Config == nil) { + return false, fmt.Sprintf("Partitions['%s'].Source.Config presence mismatch", k) + } + if p.Source.Config != nil && p.Source.Config.Range != wp.Source.Config.Range { + return false, fmt.Sprintf("Partitions['%s'].Source.Config.Range mismatch", k) + } + if !(len(p.Config) == 0 && len(wp.Config) == 0) { + if string(p.Config) != string(wp.Config) { + return false, fmt.Sprintf("Partitions['%s'].Config bytes mismatch", k) + } + if p.ConfigRange != wp.ConfigRange { + return false, fmt.Sprintf("Partitions['%s'].ConfigRange mismatch", k) + } + } + if p.Filter != wp.Filter || p.TpIndexColumn != wp.TpIndexColumn { + return false, fmt.Sprintf("Partitions['%s'].Filter/TpIndexColumn mismatch", k) + } + if (p.CustomTable == nil) != (wp.CustomTable == nil) { + return false, fmt.Sprintf("Partitions['%s'].CustomTable presence mismatch", k) + } + if p.CustomTable != nil && wp.CustomTable != nil { + if !reflect.DeepEqual(p.CustomTable, wp.CustomTable) { + return false, fmt.Sprintf("Partitions['%s'].CustomTable mismatch", k) + } + } + if p.Plugin != nil && wp.Plugin != nil { + if p.Plugin.Instance != wp.Plugin.Instance { + return false, fmt.Sprintf("Partitions['%s'].Plugin.Instance mismatch: got '%s' want '%s'", k, p.Plugin.Instance, wp.Plugin.Instance) + } + if p.Plugin.Alias != wp.Plugin.Alias { + return false, fmt.Sprintf("Partitions['%s'].Plugin.Alias mismatch: got '%s' want '%s'", k, p.Plugin.Alias, wp.Plugin.Alias) + } + if p.Plugin.Plugin != wp.Plugin.Plugin { + return false, fmt.Sprintf("Partitions['%s'].Plugin.Plugin mismatch: got '%s' want '%s'", k, p.Plugin.Plugin, wp.Plugin.Plugin) + } + } + } + } + return true, "" +} + +func tailpipeConfigEqual(l, r *config.TailpipeConfig) (bool, string) { + if l == nil || r == nil { + if l == r { + return true, "" + } + return false, "nil vs non-nil TailpipeConfig" + } + if ok, msg := pluginVersionsEqual(l.PluginVersions, r.PluginVersions); !ok { + return false, msg + } + if ok, msg := partitionsEqual(l.Partitions, r.Partitions); !ok { + return false, msg + } + if ok, msg := connectionsEqual(l.Connections, r.Connections); !ok { + return false, msg + } + if ok, msg := customTablesEqual(l.CustomTables, r.CustomTables); !ok { + return false, msg + } + if ok, msg := formatsEqual(l.Formats, r.Formats); !ok { + return false, msg + } + return true, "" +} + +func TestParseTailpipeConfig(t *testing.T) { + type args struct { + configPath string + partition string + } + tests := []struct { + name string + args args + want *config.TailpipeConfig + wantErr bool + }{ + // TODO #testing add more test cases https://github.com/turbot/tailpipe/issues/506 + { + name: "static tables", + args: args{ + configPath: "test_data/static_table_config", + partition: "partition.aws_cloudtrail_log.cloudtrail_logs", + }, + want: &config.TailpipeConfig{ + PluginVersions: map[string]*versionfile.InstalledVersion{}, + Partitions: map[string]*config.Partition{ + "aws_cloudtrail_log.cloudtrail_logs": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws_cloudtrail_log.cloudtrail_logs", + ShortName: "cloudtrail_logs", + UnqualifiedName: "aws_cloudtrail_log.cloudtrail_logs", + DeclRange: hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 3, Column: 50, Byte: 103}, + End: hcl.Pos{Line: 9, Column: 2, Byte: 252}, + }, + BlockType: "partition", + }, + TableName: "aws_cloudtrail_log", + Source: config.Source{ + Type: "file_system", + Config: &config.HclBytes{ + Hcl: []byte("extensions = [\".csv\"]\npaths = [\"/Users/kai/tailpipe_data/logs\"]"), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 6, + Column: 6, + Byte: 157, + }, + End: hcl.Pos{ + Line: 7, + Column: 29, + Byte: 244, + }, + }), + }, + }, + Config: []byte(" plugin = \"aws\"\n"), + ConfigRange: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 4, + Column: 5, + Byte: 109, + }, + End: hcl.Pos{ + Line: 4, + Column: 19, + Byte: 123, + }, + }), + }, + "aws_vpc_flow_log.flow_logs": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws_vpc_flow_log.flow_logs", + ShortName: "flow_logs", + UnqualifiedName: "aws_vpc_flow_log.flow_logs", + DeclRange: hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 12, Column: 42, Byte: 351}, + End: hcl.Pos{Line: 22, Column: 2, Byte: 636}, + }, + BlockType: "partition", + }, + TableName: "aws_vpc_flow_log", + Source: config.Source{ + Type: "aws_cloudwatch", + Config: &config.HclBytes{ + Hcl: []byte( + "log_group_name = \"/victor/vpc/flowlog\"\n" + + "start_time = \"2024-08-12T07:56:26Z\"\n" + + "end_time = \"2024-08-13T07:56:26Z\"\n" + + "access_key = \"REPLACE\"\n" + + "secret_key = \"REPLACE\"\n" + + "session_token = \"REPLACE\"", + ), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 15, Column: 6, Byte: 408}, + End: hcl.Pos{Line: 20, Column: 34, Byte: 628}, + }), + }, + }, + // Unknown attr captured at partition level + Config: []byte(" plugin = \"aws\"\n"), + ConfigRange: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 13, Column: 5, Byte: 357}, + End: hcl.Pos{Line: 13, Column: 19, Byte: 371}, + }), + }, + }, + Connections: map[string]*config.TailpipeConnection{}, + CustomTables: map[string]*config.Table{}, + Formats: map[string]*config.Format{}, + }, + + wantErr: false, + }, + { + name: "dynamic tables", + args: args{ + configPath: "test_data/custom_table_config", + }, + want: &config.TailpipeConfig{ + Partitions: map[string]*config.Partition{ + "my_csv_log.test": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "my_csv_log.test", + ShortName: "test", + UnqualifiedName: "my_csv_log.test", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 2, + Column: 30, + Byte: 30, + }, + End: hcl.Pos{ + Line: 10, + Column: 2, + Byte: 239, + }, + }, + BlockType: "partition", + }, + TableName: "my_csv_log", + Plugin: &plugin.Plugin{ + Instance: "custom", + Alias: "custom", + Plugin: "/plugins/turbot/custom@latest", + }, + Source: config.Source{ + Type: "file_system", + Format: &config.Format{ + Type: "delimited", + PresetName: "", + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "delimited.csv_logs", + ShortName: "csv_logs", + UnqualifiedName: "delimited.csv_logs", + BlockType: "format", + }, + }, + Config: &config.HclBytes{ + Hcl: []byte("extensions = [\".csv\"]\npaths = [\"/Users/kai/tailpipe_data/logs\"]"), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 4, + Column: 9, + Byte: 68, + }, + End: hcl.Pos{ + Line: 5, + Column: 30, + Byte: 139, + }, + }), + }, + }, + }, + }, + CustomTables: map[string]*config.Table{ + "my_csv_log": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "table.my_csv_log", + ShortName: "my_csv_log", + UnqualifiedName: "my_csv_log", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 14, + Column: 21, + Byte: 295, + }, + End: hcl.Pos{ + Line: 29, + Column: 2, + Byte: 602, + }, + }, + BlockType: "table", + }, + //Mode: schema.ModePartial, + Columns: []config.Column{ + { + Name: "tp_timestamp", + Source: utils.ToPointer("time_local"), + }, + { + Name: "tp_index", + Source: utils.ToPointer("account_id"), + }, + { + Name: "org_id", + Source: utils.ToPointer("org"), + }, + { + Name: "user_id", + Type: utils.ToPointer("varchar"), + }, + }, + }, + }, + Connections: map[string]*config.TailpipeConnection{}, + Formats: map[string]*config.Format{ + "delimited.csv_default_logs": { + Type: "delimited", + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "delimited.csv_default_logs", + ShortName: "csv_default_logs", + UnqualifiedName: "delimited.csv_default_logs", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 33, + Column: 39, + Byte: 644, + }, + End: hcl.Pos{ + Line: 35, + Column: 2, + Byte: 648, + }, + }, + BlockType: "format", + }, + }, + "delimited.csv_logs": { + Type: "delimited", + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "delimited.csv_logs", + ShortName: "csv_logs", + UnqualifiedName: "delimited.csv_logs", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 37, + Column: 32, + Byte: 681, + }, + End: hcl.Pos{ + Line: 40, + Column: 2, + Byte: 743, + }, + }, + BlockType: "format", + }, + Config: &config.HclBytes{ + Hcl: []byte( + " header = false\n\n delimiter = \"\\t\"\n", + ), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 38, Column: 5, Byte: 687}, + End: hcl.Pos{Line: 39, Column: 30, Byte: 741}, + }), + }, + }, + }, + PluginVersions: map[string]*versionfile.InstalledVersion{}, + }, + + wantErr: false, + }, + { + name: "invalid path", + args: args{ + configPath: "test_data/does_not_exist", + }, + want: &config.TailpipeConfig{ + PluginVersions: map[string]*versionfile.InstalledVersion{}, + Partitions: map[string]*config.Partition{}, + Connections: map[string]*config.TailpipeConnection{}, + CustomTables: map[string]*config.Table{}, + Formats: map[string]*config.Format{}, + }, + wantErr: false, + }, + { + name: "malformed hcl", + args: args{ + configPath: "test_data/malformed_config", + }, + want: nil, + wantErr: true, + }, + { + name: "invalid partition labels", + args: args{ + configPath: "test_data/invalid_partition_labels", + }, + want: nil, + wantErr: true, + }, + { + name: "connections config", + args: args{ + configPath: "test_data/connections_config", + }, + want: &config.TailpipeConfig{ + PluginVersions: map[string]*versionfile.InstalledVersion{}, + Partitions: map[string]*config.Partition{ + "aws_alb_connection_log.aws_alb_connection_log": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws_alb_connection_log.aws_alb_connection_log", + ShortName: "aws_alb_connection_log", + UnqualifiedName: "aws_alb_connection_log.aws_alb_connection_log", + DeclRange: hcl.Range{Filename: "test_data/connections_config/resources.tpc", Start: hcl.Pos{Line: 8, Column: 61, Byte: 155}, End: hcl.Pos{Line: 13, Column: 2, Byte: 278}}, + BlockType: "partition", + }, + TableName: "aws_alb_connection_log", + Source: config.Source{ + Type: "aws_s3_bucket", + Connection: &config.TailpipeConnection{ + HclResourceImpl: modconfig.HclResourceImpl{UnqualifiedName: "aws.primary"}, + }, + Config: &config.HclBytes{ + Range: hclhelpers.NewRange(hcl.Range{Filename: "test_data/connections_config/resources.tpc", Start: hcl.Pos{Line: 11, Column: 5, Byte: 228}, End: hcl.Pos{Line: 11, Column: 49, Byte: 272}}), + }, + }, + }, + }, + Connections: map[string]*config.TailpipeConnection{ + "aws.primary": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws.primary", + ShortName: "primary", + UnqualifiedName: "aws.primary", + BlockType: "connection", + }, + Plugin: "aws", + HclRange: hclhelpers.NewRange(hcl.Range{Filename: "test_data/connections_config/resources.tpc", Start: hcl.Pos{Line: 2, Column: 3, Byte: 31}, End: hcl.Pos{Line: 4, Column: 23, Byte: 90}}), + }, + }, + CustomTables: map[string]*config.Table{}, + Formats: map[string]*config.Format{}, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tailpipeDir, er := filepath.Abs(tt.args.configPath) + if er != nil { + t.Errorf("failed to build absolute config filepath from %s", tt.args.configPath) + } + // set app_specific.InstallDir + app_specific.InstallDir = tailpipeDir + + tailpipeConfig, err := parseTailpipeConfig(tt.args.configPath) + if (err.Error != nil) != tt.wantErr { + t.Errorf("LoadTailpipeConfig() error = %v, wantErr %v", err.Error, tt.wantErr) + return + } + + // use TailpipeConfig.EqualConfig for all cases (ignores Source.Config.Hcl differences) + if ok, msg := tailpipeConfigEqual(tailpipeConfig, tt.want); !ok { + t.Errorf("TailpipeConfig mismatch: %s", msg) + return + } + + }) + } +} \ No newline at end of file diff --git a/internal/parse/test_data/connections_config/resources.tpc b/internal/parse/test_data/connections_config/resources.tpc new file mode 100644 index 00000000..7873c440 --- /dev/null +++ b/internal/parse/test_data/connections_config/resources.tpc @@ -0,0 +1,13 @@ +connection "aws" "primary" { + profile = "primary" + plugin = "aws" + region = "us-east-1" +} + + +partition "aws_alb_connection_log" "aws_alb_connection_log" { + source "aws_s3_bucket" { + connection = connection.aws.primary + bucket = "alb-connection-logs-test-tailpipe" + } +} diff --git a/internal/parse/test_data/custom_table_config/resources.tpc b/internal/parse/test_data/custom_table_config/resources.tpc index 48f2abcd..9e6a0b19 100644 --- a/internal/parse/test_data/custom_table_config/resources.tpc +++ b/internal/parse/test_data/custom_table_config/resources.tpc @@ -5,14 +5,14 @@ partition "my_csv_log" "test"{ extensions = [".csv"] # format MUST be set for a custom table - format = format.csv_logs + format = format.delimited.csv_logs } } # define a custom table 'my_log' table "my_csv_log" { - format = format.csv_default_logs + format = format.delimited.csv_default_logs # the partition to use column "tp_timestamp" { source = "time_local" diff --git a/internal/parse/test_data/invalid_partition_labels/resources.tpc b/internal/parse/test_data/invalid_partition_labels/resources.tpc new file mode 100644 index 00000000..4a2ec632 --- /dev/null +++ b/internal/parse/test_data/invalid_partition_labels/resources.tpc @@ -0,0 +1,4 @@ +partition my_csv_log { + # missing 2nd label + source file_system { paths = ["/tmp"] } +} diff --git a/internal/parse/test_data/malformed_config/resources.tpc b/internal/parse/test_data/malformed_config/resources.tpc new file mode 100644 index 00000000..ec27ba1f --- /dev/null +++ b/internal/parse/test_data/malformed_config/resources.tpc @@ -0,0 +1,5 @@ +partition aws_cloudtrail_log cloudtrail_logs { + source file_system { + paths = ["/tmp"] + } + # missing closing brace here intentionally diff --git a/internal/plugin/installation_actions.go b/internal/plugin/installation_actions.go index 6bd2c82b..72f0adab 100644 --- a/internal/plugin/installation_actions.go +++ b/internal/plugin/installation_actions.go @@ -116,7 +116,7 @@ func List(ctx context.Context, pluginVersions map[string]*versionfile.InstalledV // detectLocalPlugin returns true if the modTime of the `pluginBinary` is after the installation date as recorded in the installation data // this may happen when a plugin is installed from the registry, but is then compiled from source func detectLocalPlugin(installation *versionfile.InstalledVersion, pluginBinary string) bool { - // TODO this should no longer be necessary as we now have a "local" version number in the versions file? + // TODO this should no longer be necessary as we now have a "local" version number in the versions file? https://github.com/turbot/tailpipe/issues/507 if installation == nil { return true } diff --git a/internal/plugin/plugin_manager.go b/internal/plugin/plugin_manager.go index c5ee3ac2..765e790e 100644 --- a/internal/plugin/plugin_manager.go +++ b/internal/plugin/plugin_manager.go @@ -617,7 +617,7 @@ func loadPluginVersionFile(ctx context.Context) (*versionfile.PluginVersionFile, return nil, err } - // TODO KAI CHECK THIS + // TODO CHECK THIS https://github.com/turbot/tailpipe/issues/507 // add any "local" plugins (i.e. plugins installed under the 'local' folder) into the version file ew := pluginVersions.AddLocalPlugins(ctx) if ew.Error != nil { diff --git a/internal/query/execute.go b/internal/query/execute.go index bdfc874b..31c73877 100644 --- a/internal/query/execute.go +++ b/internal/query/execute.go @@ -70,7 +70,7 @@ func ExecuteQuery(ctx context.Context, query string, db *database.DuckDb) (int, // show output _, rowErrors := querydisplay.ShowOutput(ctx, result) if rowErrors > 0 { - // TODO #errors find a way to return the error + // TODO #errors find a way to return the error https://github.com/turbot/pipe-fittings/issues/745 return rowErrors, fmt.Errorf("query execution failed") } return 0, nil diff --git a/memtest/go.mod b/memtest/go.mod deleted file mode 100644 index 30a39ee5..00000000 --- a/memtest/go.mod +++ /dev/null @@ -1,35 +0,0 @@ -module memtest - -go 1.24 - -toolchain go1.24.1 - -require github.com/marcboeker/go-duckdb/v2 v2.2.0 - -require ( - github.com/apache/arrow-go/v18 v18.1.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/duckdb/duckdb-go-bindings v0.1.14 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9 // indirect - github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9 // indirect - github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9 // indirect - github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9 // indirect - github.com/go-viper/mapstructure/v2 v2.3.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/google/flatbuffers v25.1.24+incompatible // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 // indirect - github.com/marcboeker/go-duckdb/mapping v0.0.7 // indirect - github.com/pierrec/lz4/v4 v4.1.22 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/tools v0.29.0 // indirect - golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect -) diff --git a/memtest/go.sum b/memtest/go.sum deleted file mode 100644 index 9e7b8d71..00000000 --- a/memtest/go.sum +++ /dev/null @@ -1,74 +0,0 @@ -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= -github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= -github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= -github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/duckdb/duckdb-go-bindings v0.1.14 h1:57DCZuuKQ65gRQxFG+XGnqVQtMADKY/noozmCjYs+zE= -github.com/duckdb/duckdb-go-bindings v0.1.14/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9 h1:K95YlR28Fb3+n3D6RcBzdznNVGcCnrGaAZqs52JUFOs= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9 h1:wY3kXm1/GSK4ES8pfBIeRHxscZomEVFWTS4GOifrZCs= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9 h1:ypZyeNMA9oRAIBE/pVGfrsXzYqEM+ZRkbV/lxw7Cf5E= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9 h1:TVBDwDSanIttQCH76UpDJ9rQAq4cYNM4R7h5Xu0y/rA= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9 h1:okFoG+evMiXnyUK+cI67V0MpvKbstO6MaXlXXotst3k= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= -github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= -github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= -github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 h1:6mq16sPGJPo8Tkkl6UIsXuaNv467LjHLBscRyJl2Qhc= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.7/go.mod h1:FdvmqJOwVdfFZLpV+anBFlTUOzfU/NdIRET37mIEczY= -github.com/marcboeker/go-duckdb/mapping v0.0.7 h1:t0BaNmLXj76RKs/x80A/ZTe+KzZDimO2Ji8ct4YnPu4= -github.com/marcboeker/go-duckdb/mapping v0.0.7/go.mod h1:EH3RSabeePOUePoYDtF0LqfruXPtVB3M+g03QydZsck= -github.com/marcboeker/go-duckdb/v2 v2.2.0 h1:xxruuYD7vWvybY52xWzV0vvHKa1IjpDDOq6T846ax/s= -github.com/marcboeker/go-duckdb/v2 v2.2.0/go.mod h1:B7swJ38GcOEm9PI0IdfkZYqn5CtIjRUiQG4ZBr3hnyc= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= -gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/memtest/main.go b/memtest/main.go deleted file mode 100644 index 9137570c..00000000 --- a/memtest/main.go +++ /dev/null @@ -1,254 +0,0 @@ -package main - -// #include -import "C" -import ( - "context" - "database/sql" - "fmt" - _ "github.com/marcboeker/go-duckdb/v2" - "log" - "os" - "path/filepath" - "strconv" - "strings" - "time" -) - -// ensureOutputDirectory creates the output directory if it doesn't exist -func ensureOutputDirectory() error { - outputDir := "./output" - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - return os.MkdirAll(outputDir, 0755) - } - return nil -} - -const ( - // Use a relative path for the output directory - queryFormat = `copy (select * from read_ndjson('%s')) - to './output' ( - format parquet, - partition_by (tp_index,tp_date), - overwrite_or_ignore, - return_files true - );` - // Query to get memory usage - memoryQuery = "SELECT temporary_storage_bytes FROM duckdb_memory() WHERE tag = 'COLUMN_DATA'" -) - -func main() { - if len(os.Args) < 2 { - log.Fatalf("Usage: %s ", os.Args[0]) - } - - // Validate file exists - filename := os.Args[1] - if _, err := os.Stat(filename); os.IsNotExist(err) { - log.Fatalf("File does not exist: %s", filename) - } - - // Parse filename to get parameters - baseName := strings.TrimSuffix(filepath.Base(filename), ".jsonl") - params := strings.Split(baseName, "_") - if len(params) < 5 { - log.Fatalf("Skipping invalid filename format: %s (expected at least 5 parts, got %d)", filename, len(params)) - return - } - - // Extract parameters from filename - rows, _ := strconv.Atoi(strings.TrimSuffix(params[1], "rows")) - cols, _ := strconv.Atoi(strings.TrimSuffix(params[2], "cols")) - indexes, _ := strconv.Atoi(strings.TrimSuffix(params[3], "indexes")) - dates, _ := strconv.Atoi(strings.TrimSuffix(params[4], "dates")) - partitions := indexes * dates - - // Ensure output directory exists - if err := ensureOutputDirectory(); err != nil { - log.Fatalf("Failed to create output directory: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - db, err := sql.Open("duckdb", ":memory:") - if err != nil { - log.Fatalf("Failed to open connection: %v", err) - } - defer db.Close() - - resultChan := monitorMemoryUsage(ctx, 250*time.Millisecond) - - // Run the query - _, err = runQueryAndGetMemory(db, filename) - if err != nil { - log.Fatalf("Failed to get memory usage: %v", err) - } - - cancel() // Signal memory monitoring to stop - - // Ensure channel read doesn't block indefinitely - select { - case maxMemory := <-resultChan: - fmt.Printf("%d, %d, %d, %d\n", rows, cols, partitions, maxMemory/(1024*1024)) - case <-time.After(5 * time.Second): - log.Fatal("Timed out waiting for memory results") - } - - //processBadFiles() - - //// Create CSV file with datetime stamp - //timestamp := time.Now().Format("20060102_150405") - //csvFilename := fmt.Sprintf("memory_results_%s.csv", timestamp) - //csvFile, err := os.Create(csvFilename) - //if err != nil { - // log.Fatalf("Failed to create CSV file: %v", err) - //} - //defer csvFile.Close() - // - //writer := csv.NewWriter(csvFile) - //defer writer.Flush() - // - //// Write CSV header - //header := []string{"rows", "columns", "indexes", "dates", "partitions", "filename", "memory_mb", "query_error"} - //if err := writer.Write(header); err != nil { - // log.Fatalf("Failed to write CSV header: %v", err) - //} - // - //// Get all generated files - //files, err := filepath.Glob("testdata/generated/*.jsonl") - //if err != nil { - // log.Fatalf("Failed to find generated files: %v", err) - //} - //// Sort in reverse order - //sort.Sort(sort.Reverse(sort.StringSlice(files))) - // - //// Process each file - //for _, file := range files { - // processFile(file, writer) - //} - -} - -// -//func processBadFiles() { -// -// var files = []string{ -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/bad.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_9000.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_7500.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_5000.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_2500.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_1000.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_500.jsonl", -// "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/subsets/bad_100.jsonl", -// } -// -// for _, file := range files { -// db, err := sql.Open("duckdb", ":memory:") -// if err != nil { -// log.Fatalf("Failed to open connection: %v", err) -// } -// memoryMB, queryErr := runQueryAndGetMemory(db, file) -// db.Close() -// -// if queryErr != nil { -// fmt.Printf("Failed to get memory usage for %s: %v", file, queryErr) -// } else { -// fmt.Printf("Memory usage for %s: %.2f MB\n", file, memoryMB) -// } -// } -//} -// -//func processFile(fileName string, writer *csv.Writer) { -// db, err := sql.Open("duckdb", ":memory:") -// if err != nil { -// log.Fatalf("Failed to open connection: %v", err) -// } -// -// file, err := filepath.Abs(fileName) -// if err != nil { -// log.Printf("Failed to get absolute path for %s: %v", file, err) -// return -// } -// // Parse filename to get parameters -// baseName := strings.TrimSuffix(filepath.Base(file), ".jsonl") -// params := strings.Split(baseName, "_") -// if len(params) < 5 { -// log.Printf("Skipping invalid filename format: %s (expected at least 5 parts, got %d)", file, len(params)) -// return -// } -// -// // Extract parameters from filename -// rows, _ := strconv.Atoi(strings.TrimSuffix(params[1], "rows")) -// cols, _ := strconv.Atoi(strings.TrimSuffix(params[2], "cols")) -// indexes, _ := strconv.Atoi(strings.TrimSuffix(params[3], "indexes")) -// dates, _ := strconv.Atoi(strings.TrimSuffix(params[4], "dates")) -// -// // Get memory usage and error -// memoryMB, queryErr := runQueryAndGetMemory(db, file) -// if queryErr != nil { -// log.Printf("Failed to get memory usage for %s: %v", fileName, queryErr) -// } else { -// fmt.Printf("Memory usage for %s: %.2f MB\n", fileName, memoryMB) -// } -// -// // Calculate memory string -// var memoryStr string -// if queryErr != nil { -// memoryStr = "" -// } else { -// memoryStr = fmt.Sprintf("%.2f", memoryMB) -// } -// -// // Write to CSV -// record := []string{ -// strconv.Itoa(rows), -// strconv.Itoa(cols), -// strconv.Itoa(indexes), -// strconv.Itoa(dates), -// strconv.Itoa(indexes * dates), -// fileName, -// memoryStr, -// fmt.Sprintf("%v", queryErr), -// } -// if err := writer.Write(record); err != nil { -// log.Printf("Failed to write record for %s: %v", fileName, err) -// } -//} - -func runQueryAndGetMemory(db *sql.DB, filename string) (float64, error) { - // Validate database connection - if err := db.Ping(); err != nil { - return 0, fmt.Errorf("database connection error: %v", err) - } - - // Get memory usage before query - var memoryBefore int64 - err := db.QueryRow(memoryQuery).Scan(&memoryBefore) - if err != nil { - return 0, fmt.Errorf("failed to get initial memory usage: %v", err) - } - - // Prepare and run the query - query := fmt.Sprintf(queryFormat, filename) - - // Use context-aware Exec to allow for timeout/cancellation - _, err = db.Exec(query) - if err != nil { - return 0, fmt.Errorf("failed to run query on %s: %v", filename, err) - } - - // Get memory usage after query - var memoryAfter int64 - err = db.QueryRow(memoryQuery).Scan(&memoryAfter) - if err != nil { - return 0, fmt.Errorf("failed to get final memory usage: %v", err) - } - - // Convert bytes to MB - memoryMB := float64(memoryAfter-memoryBefore) / (1024 * 1024) - //fmt.Printf("Memory usage for %s: %.2f MB (before: %d bytes, after: %d bytes)\n", - // filepath.Base(filename), memoryMB, memoryBefore, memoryAfter) - return memoryMB, nil -} diff --git a/memtest/memory_results_.csv b/memtest/memory_results_.csv deleted file mode 100644 index efb5c2f1..00000000 --- a/memtest/memory_results_.csv +++ /dev/null @@ -1,37 +0,0 @@ -Filename,Rows,Cols,Partitions,MemoryMB -test_1000rows_100cols_100indexes_10dates.jsonl,1000, 100, 1000, 3732 -test_1000rows_100cols_1indexes_10dates.jsonl,1000, 100, 10, 0 -test_1000rows_100cols_25indexes_10dates.jsonl,1000, 100, 250, 1010 -test_1000rows_100cols_400indexes_10dates.jsonl,1000, 100, 4000, 3707 -test_1000rows_10cols_100indexes_10dates.jsonl,1000, 10, 1000, 1277 -test_1000rows_10cols_1indexes_10dates.jsonl,1000, 10, 10, 0 -test_1000rows_10cols_25indexes_10dates.jsonl,1000, 10, 250, 0 -test_1000rows_10cols_400indexes_10dates.jsonl,1000, 10, 4000, 1321 -test_1000rows_25cols_100indexes_10dates.jsonl,1000, 25, 1000, 1288 -test_1000rows_25cols_1indexes_10dates.jsonl,1000, 25, 10, 0 -test_1000rows_25cols_25indexes_10dates.jsonl,1000, 25, 250, 0 -test_1000rows_25cols_400indexes_10dates.jsonl,1000, 25, 4000, 1307 -test_4000rows_100cols_100indexes_10dates.jsonl,4000, 100, 1000, 4028 -test_4000rows_100cols_1indexes_10dates.jsonl,4000, 100, 10, 0 -test_4000rows_100cols_25indexes_10dates.jsonl,4000, 100, 250, 1073 -test_4000rows_100cols_400indexes_10dates.jsonl,4000, 100, 4000, 15453 -test_4000rows_10cols_100indexes_10dates.jsonl,4000, 10, 1000, 1353 -test_4000rows_10cols_1indexes_10dates.jsonl,4000, 10, 10, 0 -test_4000rows_10cols_25indexes_10dates.jsonl,4000, 10, 250, 0 -test_4000rows_10cols_400indexes_10dates.jsonl,4000, 10, 4000, 5191 -test_4000rows_25cols_100indexes_10dates.jsonl,4000, 25, 1000, 1385 -test_4000rows_25cols_1indexes_10dates.jsonl,4000, 25, 10, 0 -test_4000rows_25cols_25indexes_10dates.jsonl,4000, 25, 250, 0 -test_4000rows_25cols_400indexes_10dates.jsonl,4000, 25, 4000, 5180 -test_8000rows_100cols_100indexes_10dates.jsonl,8000, 100, 1000, 3939 -test_8000rows_100cols_1indexes_10dates.jsonl,8000, 100, 10, 168 -test_8000rows_100cols_25indexes_10dates.jsonl,8000, 100, 250, 1112 -test_8000rows_100cols_400indexes_10dates.jsonl,8000, 100, 4000, 16051 -test_8000rows_10cols_100indexes_10dates.jsonl,8000, 10, 1000, 1359 -test_8000rows_10cols_1indexes_10dates.jsonl,8000, 10, 10, 0 -test_8000rows_10cols_25indexes_10dates.jsonl,8000, 10, 250, 417 -test_8000rows_10cols_400indexes_10dates.jsonl,8000, 10, 4000, 5290 -test_8000rows_25cols_100indexes_10dates.jsonl,8000, 25, 1000, 1378 -test_8000rows_25cols_1indexes_10dates.jsonl,8000, 25, 10, 0 -test_8000rows_25cols_25indexes_10dates.jsonl,8000, 25, 250, 418 -test_8000rows_25cols_400indexes_10dates.jsonl,8000, 25, 4000, 5330 diff --git a/memtest/run_mem.sh b/memtest/run_mem.sh deleted file mode 100755 index 693655d7..00000000 --- a/memtest/run_mem.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Create output CSV file with headers -timestamp=$(date '+%Y%m%d_%H%M%S') -output_file="memory_results_${timestamp}.csv" -echo "Creating CSV output file: $output_file" -echo "Filename,Rows,Cols,Partitions,MemoryMB" > "$output_file" - -# Determine correct directory name -if [ -d "testsdata/generated" ]; then - data_dir="testsdata/generated" -elif [ -d "testdata/generated" ]; then - data_dir="testdata/generated" -else - echo "Error: Neither testsdata/generated nor testdata/generated directory found" - exit 1 -fi - -echo "Using data directory: $data_dir" -file_count=$(ls -1 $data_dir/*.jsonl 2>/dev/null | wc -l) -echo "Found $file_count files to process" - -# Process each file in the directory -processed_count=0 -success_count=0 - -for file in $data_dir/*.jsonl; do - if [ -f "$file" ]; then - filename=$(basename "$file") - echo "Processing file $((processed_count+1))/$file_count: $filename" - - # Run the memtest app and capture its output - output=$(./memtest "$file" 2>&1) - exit_code=$? - - # Check if the command was successful - if [ $exit_code -eq 0 ]; then - # Try both formats of output (with or without spaces) - csv_data=$(echo "$output" | grep -E '^[0-9]+,[ ]*[0-9]+,[ ]*[0-9]+,[ ]*[0-9]+$') - if [ -z "$csv_data" ]; then - csv_data=$(echo "$output" | grep -E '^[0-9]+, [0-9]+, [0-9]+, [0-9]+$') - fi - - if [ -n "$csv_data" ]; then - echo "$filename,$csv_data" >> "$output_file" - success_count=$((success_count+1)) - else - echo "Warning: Could not extract memory data from output" - fi - else - echo "Error: memtest failed for $file" - fi - - processed_count=$((processed_count+1)) - fi -done - -echo "Processing complete. Results saved to $output_file" -echo "Processed $success_count/$file_count files successfully" \ No newline at end of file diff --git a/memtest/short.jsonl b/memtest/short.jsonl deleted file mode 100644 index dfeddee7..00000000 --- a/memtest/short.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"tp_id":"d04arpup281gmmlsnjf0","tp_source_type":"file","tp_ingest_timestamp":"2025-04-23T10:07:51.833525+01:00","tp_timestamp":"2025-03-27T08:00:00Z","tp_table":"aws_cost_and_usage_report","tp_partition":"cody2","tp_index":"339713003993","tp_date":"2025-03-27T00:00:00Z","tp_source_ip":null,"tp_destination_ip":null,"tp_source_name":null,"tp_source_location":"/Users/kai/tailpipe_data/cur/cur-2-0-daily-csv/data/BILLING_PERIOD=2025-03/cur-2-0-daily-csv-00001.csv.gz","bill_billing_entity":"AWS","bill_billing_period_end_date":"2025-04-01T00:00:00Z","bill_billing_period_start_date":"2025-03-01T00:00:00Z","bill_bill_type":"Anniversary","bill_invoice_id":"2104370321","bill_invoicing_entity":"Amazon Web Services, Inc.","bill_payer_account_id":"936717460871","bill_payer_account_name":"Pipeling Scale Testing Management","cost_category":{"cost_category":"{}"},"discount":{"discount":"{}"},"identity_line_item_id":"np7mqdc4tujpl53f5sfb5trz3s4j4r5r5fdfci5uoftphuyxb2ga","identity_time_interval":"2025-03-27T00:00:00Z/2025-03-28T00:00:00Z","line_item_availability_zone":"","line_item_blended_cost":8e-7,"line_item_blended_rate":"0.0000004000","line_item_currency_code":"USD","line_item_legal_entity":"Amazon Web Services, Inc.","line_item_line_item_description":"$0.40 per million Amazon SQS standard requests in Tier1 in EU (London)","line_item_line_item_type":"Usage","line_item_net_unblended_rate":"","line_item_normalization_factor":0,"line_item_normalized_usage_amount":0,"line_item_operation":"GetQueueAttributes","line_item_product_code":"AWSQueueService","line_item_resource_id":"arn:aws:sqs:eu-west-2:339713003993:pipeling-test-DC3FEBB1-8316-4CE9-B98E-C96A8C5B8168","line_item_tax_type":"","line_item_unblended_cost":8e-7,"line_item_unblended_rate":"0.0000004000","line_item_usage_account_id":"339713003993","line_item_usage_account_name":"Pipeling Scale Testing 001","line_item_usage_amount":2,"line_item_usage_end_date":"2025-03-27T10:00:00Z","line_item_usage_start_date":"2025-03-27T08:00:00Z","line_item_usage_type":"EUW2-Requests-Tier1","pricing_currency":"USD","pricing_lease_contract_length":"","pricing_offering_class":"","pricing_public_on_demand_cost":8e-7,"pricing_public_on_demand_rate":"0.0000004000","pricing_purchase_option":"","pricing_rate_code":"7DSEXZJZCF4MMFKF.JRTCKXETXF.VXGXCWQKTY","pricing_rate_id":"143259033430","pricing_term":"OnDemand","pricing_unit":"Requests","product":{"comment":"","fee_code":"","fee_description":"","from_location":"","from_location_type":"","from_region_code":"","instance_family":"","instance_type":"","instancesku":"","location":"EU (London)","location_type":"AWS Region","operation":"","pricing_unit":"","product":"{\"queue_type\":\"Standard\",\"message_delivery_order\":\"Not Guaranteed\",\"product_name\":\"Amazon Simple Queue Service\",\"message_delivery_frequency\":\"At Least Once\",\"region\":\"eu-west-2\",\"servicename\":\"Amazon Simple Queue Service\",\"group_description\":\"Amazon SQS Requests\",\"group\":\"SQS-APIRequest-Tier1\"}","product_family":"API Request","region_code":"eu-west-2","servicecode":"AWSQueueService","sku":"7DSEXZJZCF4MMFKF","to_location":"","to_location_type":"","to_region_code":"","usagetype":"EUW2-Requests-Tier1"},"product_comment":"","product_fee_code":"","product_fee_description":"","product_from_location":"","product_from_location_type":"","product_from_region_code":"","product_instancesku":"","product_instance_family":"","product_instance_type":"","product_location":"EU (London)","product_location_type":"AWS Region","product_operation":"","product_pricing_unit":"","product_product_family":"API Request","product_region_code":"eu-west-2","product_sku":"7DSEXZJZCF4MMFKF","product_servicecode":"AWSQueueService","product_to_location_type":"","product_to_location":"","product_to_region_code":"","product_usagetype":"EUW2-Requests-Tier1","reservation":{"amortized_upfront_cost_for_usage":0,"amortized_upfront_fee_for_billing_period":0,"availability_zone":"","effective_cost":0,"modification_status":"","normalized_units_per_reservation":"","number_of_reservations":"","recurring_fee_for_usage":0,"reservation_arn":"","subscription_id":"14795039463","total_reserved_normalized_units":"","total_reserved_units":"","units_per_reservation":"","unused_amortized_upfront_fee_for_billing_period":0,"unused_normalized_unit_quantity":0,"unused_quantity":0,"unused_recurring_fee":0},"reservation_amortized_upfront_cost_for_usage":0,"reservation_amortized_upfront_fee_for_billing_period":0,"reservation_reservation_arn":"","reservation_availability_zone":"","reservation_effective_cost":0,"reservation_modification_status":"","reservation_normalized_units_per_reservation":"","reservation_number_of_reservations":"","reservation_recurring_fee_for_usage":0,"reservation_subscription_id":"14795039463","reservation_total_reserved_normalized_units":"","reservation_total_reserved_units":"","reservation_units_per_reservation":"","reservation_unused_amortized_upfront_fee_for_billing_period":0,"reservation_unused_normalized_unit_quantity":0,"reservation_unused_quantity":0,"reservation_unused_recurring_fee":0,"resource_tags":{},"savings_plan_amortized_upfront_commitment_for_billing_period":0,"savings_plan_instance_type_family":"","savings_plan_offering_type":"","savings_plan_payment_option":"","savings_plan_purchase_term":"","savings_plan_recurring_commitment_for_billing_period":0,"savings_plan_region":"","savings_plan_savings_plan_arn":"","savings_plan_savings_plan_effective_cost":0,"savings_plan_savings_plan_rate":"0.0","savings_plan_total_commitment_to_date":"0.0","savings_plan_used_commitment":"0.0","split_line_item_parent_resource_id":"","split_line_item_split_usage_ratio":""} \ No newline at end of file diff --git a/memtest/short2.jsonl b/memtest/short2.jsonl deleted file mode 100644 index ddfe863d..00000000 --- a/memtest/short2.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"account_id":"id-986-536","account_name":"value_0_118","bill_type":"value_0_442","billing_entity":"value_0_413","billing_period_end":"value_0_168","billing_period_start":"value_0_845","cost_category":"39.90","currency_code":"value_0_281","invoice_id":"id-979-437","legal_entity_name":"value_0_817","line_item_description":"value_0_56","line_item_normalized_usage_amount":"84.51","line_item_tax_type":"value_0_820","line_item_unblended_cost":"65.14","line_item_unblended_rate":"82.56","line_item_usage_account_id":"id-233-134","line_item_usage_amount":"88.89","line_item_usage_end_date":"2025-04-02T11:39:00+01:00","line_item_usage_start_date":"2025-03-28T11:39:00Z","line_item_usage_type":"value_0_879","pricing_term":"value_0_229","product_code":"value_0_887","product_family":"value_0_675","product_from_location":"value_0_830","product_location":"value_0_17","product_name":"value_0_337","product_to_location":"value_0_103","provider":"id-572-981","reservation_arn":"id-634-183","savings_plan_total_commitment_to_date":"83.67","savings_plan_used_commitment":"24.11","split_line_item_path":"value_0_8","split_line_item_split_percentange":"value_0_680","tags":"value_0_352","tp_date":"2026-04-23 11:39:00","tp_id":"d7rpup6gmmlsnjft","tp_index":"339713007821","tp_ingest_timestamp":"2026-04-23T11:39:00+01:00","tp_source_type":"file","usage_type":"value_0_936"} diff --git a/memtest/testdata/generate.go b/memtest/testdata/generate.go deleted file mode 100644 index f64ae5fa..00000000 --- a/memtest/testdata/generate.go +++ /dev/null @@ -1,167 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "log" - "math/rand" - "os" - "strconv" - "strings" - "time" -) - -// DynamicRow represents a row with dynamic fields, matching the SDK's structure -type DynamicRow struct { - // The output columns, as a map of string to interface{} - OutputColumns map[string]interface{} -} - -// MarshalJSON overrides JSON serialization to include the dynamic columns -func (l *DynamicRow) MarshalJSON() ([]byte, error) { - return json.Marshal(l.OutputColumns) -} - -type IndexDate struct { - Index string - Date time.Time -} - -func main() { - // Parse command line arguments - filename := flag.String("file", "test_bad_format.jsonl", "Output filename") - rowCount := flag.Int("rows", 1000, "Number of rows to generate") - colCount := flag.Int("cols", 100, "Number of data columns") - indexCount := flag.Int("indexes", 5, "Number of distinct tp_index values") - datesPerIndex := flag.Int("dates", 5, "Number of dates per index") - flag.Parse() - - // Create the output file - file, err := os.Create(*filename) - if err != nil { - log.Fatalf("Failed to create file: %v", err) - } - defer file.Close() - - // Generate random seed - rand.Seed(time.Now().UnixNano()) - - // Generate column names (for dynamically added fields) - // Based on what we observed in the bad.jsonl file, these would be fields like - // savings_plan_total, savings_plan_used, etc. - columnNames := []string{ - "account_id", "account_name", "billing_entity", "billing_period_start", "billing_period_end", - "bill_type", "cost_category", "currency_code", "invoice_id", "legal_entity_name", - "line_item_description", "line_item_normalized_usage_amount", "line_item_tax_type", - "line_item_unblended_cost", "line_item_unblended_rate", "line_item_usage_account_id", - "line_item_usage_amount", "line_item_usage_end_date", "line_item_usage_start_date", - "line_item_usage_type", "pricing_term", "product_code", "product_family", "product_from_location", - "product_location", "product_name", "product_to_location", "provider", "reservation_arn", - "savings_plan_total_commitment_to_date", "savings_plan_used_commitment", - "split_line_item_path", "split_line_item_split_percentange", "tags", "usage_type", - } - - // Add numeric columns for the remaining count requested - for i := len(columnNames); i < *colCount; i++ { - columnNames = append(columnNames, fmt.Sprintf("column_%d", i)) - } - - // Generate index/date combinations - combos := make([]IndexDate, 0) - startDate := time.Now().AddDate(1, 0, 0) // Future date - - // For each index, generate multiple dates - for i := 0; i < *indexCount; i++ { - // Using a numeric string like in the bad file (e.g., 339713003993) - index := fmt.Sprintf("%d", 339713000000+rand.Intn(10000)) - - // Generate dates for this index - for j := 0; j < *datesPerIndex; j++ { - // Spread dates over a month - date := startDate.AddDate(0, 0, j*(30/(*datesPerIndex))) - combos = append(combos, IndexDate{ - Index: index, - Date: date, - }) - } - } - - // Create a slice to track which combinations have been used - usedCombos := make([]bool, len(combos)) - combosUsed := 0 - - // Generate and write rows - for i := 0; i < *rowCount; i++ { - // Create a row with OutputColumns to match SDK's DynamicRow - row := &DynamicRow{ - OutputColumns: make(map[string]interface{}), - } - - // Add the fixed fields - row.OutputColumns["tp_id"] = fmt.Sprintf("d%drpup%dgmmlsnjf%c", rand.Intn(10), rand.Intn(10), 'a'+rand.Intn(26)) - row.OutputColumns["tp_source_type"] = "file" - row.OutputColumns["tp_ingest_timestamp"] = time.Now().AddDate(1, 0, 0).Format(time.RFC3339) - - // Add dynamic fields - for _, col := range columnNames { - // Set different types of data based on column name pattern - if strings.Contains(col, "cost") || strings.Contains(col, "amount") || - strings.Contains(col, "rate") || strings.Contains(col, "commitment") { - // Numeric values as strings - row.OutputColumns[col] = strconv.FormatFloat(rand.Float64()*100, 'f', 2, 64) - } else if strings.Contains(col, "date") { - // Date values - date := time.Now().AddDate(0, 0, -rand.Intn(30)) - row.OutputColumns[col] = date.Format(time.RFC3339) - } else if strings.Contains(col, "id") || strings.Contains(col, "arn") { - // ID values - row.OutputColumns[col] = fmt.Sprintf("id-%d-%d", rand.Intn(1000), rand.Intn(1000)) - } else { - // Default string values - row.OutputColumns[col] = fmt.Sprintf("value_%d_%d", i, rand.Intn(1000)) - } - } - - // Select a combination ensuring we use all combinations - var combo IndexDate - if combosUsed < len(combos) { - // Use each combination at least once - for j := 0; j < len(combos); j++ { - if !usedCombos[j] { - combo = combos[j] - usedCombos[j] = true - combosUsed++ - break - } - } - } else { - // After using all combinations, randomly select from them - combo = combos[rand.Intn(len(combos))] - } - - // Set the partition fields - row.OutputColumns["tp_index"] = combo.Index - row.OutputColumns["tp_date"] = combo.Date.Format("2006-01-02 15:04:05") // Format as observed in bad file - - // Add empty object field to help DuckDB parse the structure - row.OutputColumns["resource_tags"] = map[string]interface{}{} - - // Convert to JSON - jsonData, err := json.Marshal(row) - if err != nil { - log.Fatalf("Failed to marshal JSON: %v", err) - } - - // Write to file - if _, err := file.Write(jsonData); err != nil { - log.Fatalf("Failed to write to file: %v", err) - } - if _, err := file.WriteString("\n"); err != nil { - log.Fatalf("Failed to write newline: %v", err) - } - } - - fmt.Printf("Generated %d rows with %d columns, %d indexes, and %d dates per index in flat format to %s\n", - *rowCount, *colCount+5, *indexCount, *datesPerIndex, *filename) // +5 for the fixed fields -} diff --git a/memtest/testdata/generate_all.sh b/memtest/testdata/generate_all.sh deleted file mode 100755 index 06f67413..00000000 --- a/memtest/testdata/generate_all.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Create output directory -mkdir -p generated - -# Array of row counts -ROWS=(1000 3000 4000 6000 8000) - -# Array of column counts -# Array of column counts -COLUMNS=(10 25 50 100) - -# Array of index counts -INDEXES=(1 25 100 400) - -# Array of dates per index -DATES=(10) - -# Generate all combinations -for rows in "${ROWS[@]}"; do - for cols in "${COLUMNS[@]}"; do - for indexes in "${INDEXES[@]}"; do - for dates in "${DATES[@]}"; do - filename="generated/test_${rows}rows_${cols}cols_${indexes}indexes_${dates}dates.jsonl" - echo "Generating $filename..." - go run generate.go -file="$filename" -rows=$rows -cols=$cols -indexes=$indexes -dates=$dates - done - done - done -done - -echo "All files generated in the 'generated' directory" \ No newline at end of file diff --git a/memtest/testdata/subset/subset_creator.go b/memtest/testdata/subset/subset_creator.go deleted file mode 100644 index 5a226800..00000000 --- a/memtest/testdata/subset/subset_creator.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "log" - "os" - "path/filepath" -) - -func main() { - // Define the source file - sourceFile := "/Users/kai/Dev/github/turbot/tailpipe/memtest/testdata/bad.jsonl" - - // Define the line counts for the subsets - lineCounts := []int{100, 500, 1000, 2500, 5000, 7500, 9000} - - // Read the source file - file, err := os.Open(sourceFile) - if err != nil { - log.Fatalf("Failed to open source file: %v", err) - } - defer file.Close() - - // Read all lines from the source file - scanner := bufio.NewScanner(file) - var lines []string - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - - if err := scanner.Err(); err != nil { - log.Fatalf("Error reading source file: %v", err) - } - - totalLines := len(lines) - fmt.Printf("Source file contains %d lines\n", totalLines) - - // Create output directory - outputDir := filepath.Join(filepath.Dir(sourceFile), "subsets") - if err := os.MkdirAll(outputDir, 0755); err != nil { - log.Fatalf("Failed to create output directory: %v", err) - } - - // Generate subset files - for _, count := range lineCounts { - if count > totalLines { - fmt.Printf("Skipping %d lines (requested more than available)\n", count) - continue - } - - outputFile := filepath.Join(outputDir, fmt.Sprintf("bad_%d.jsonl", count)) - fmt.Printf("Creating subset with %d lines: %s\n", count, outputFile) - - out, err := os.Create(outputFile) - if err != nil { - log.Fatalf("Failed to create output file %s: %v", outputFile, err) - } - - writer := bufio.NewWriter(out) - for i := 0; i < count; i++ { - if i < len(lines) { - fmt.Fprintln(writer, lines[i]) - } - } - - writer.Flush() - out.Close() - } - - fmt.Println("Subset files created successfully in the 'subsets' directory") - fmt.Println("You can now use these files in your application") -} diff --git a/tailpipe_data_generator/go.mod b/tailpipe_data_generator/go.mod deleted file mode 100644 index 3caa80bb..00000000 --- a/tailpipe_data_generator/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module tailpipe_data_generator - -go 1.24 - -require github.com/elastic/go-grok v0.0.0-20240508093839-cd1fbee3a5d3 diff --git a/tailpipe_data_generator/main.go b/tailpipe_data_generator/main.go deleted file mode 100644 index abf6129f..00000000 --- a/tailpipe_data_generator/main.go +++ /dev/null @@ -1,129 +0,0 @@ -package main - -import ( - "fmt" - "math/rand" - "os" - "path/filepath" - "time" -) - -const ( - baseDir = "/Users/kai/tailpipe_data/dated" - numAccounts = 10 - numFilesPerAccount = 10 -) - -func main() { - // Create the base directory - err := os.MkdirAll(baseDir, 0755) - if err != nil { - fmt.Printf("Error creating base directory: %v\n", err) - return - } - - fmt.Println("Created base directory:", baseDir) - - // Create account directories and files - for i := 1; i <= numAccounts; i++ { - accountID := fmt.Sprintf("account%03d", i) - accountDir := filepath.Join(baseDir, accountID) - - // Create account directory - err := os.MkdirAll(accountDir, 0755) - if err != nil { - fmt.Printf("Error creating account directory %s: %v\n", accountID, err) - continue - } - - fmt.Println("Created account directory:", accountDir) - - // Create files in the account directory - for j := 1; j <= numFilesPerAccount; j++ { - // Get deterministic date from the last 10 days based on index - // Using modulo to ensure we cycle through the days - dayIndex := j % 10 - fileDate := getDateFromLast10Days(dayIndex) - year := fileDate.Year() - month := int(fileDate.Month()) - day := fileDate.Day() - - // Create filename in the format: account_id_year_month_day_idx.log - filename := fmt.Sprintf("%s_%d_%02d_%02d_%02d.log", accountID, year, month, day, j) - filePath := filepath.Join(accountDir, filename) - - // Create file with some random content - content := generateRandomLogContent(accountID, fileDate, 10+rand.Intn(20)) - err := os.WriteFile(filePath, []byte(content), 0644) - if err != nil { - fmt.Printf("Error creating file %s: %v\n", filename, err) - continue - } - - fmt.Printf("Created file: %s\n", filePath) - } - } - -} - -// Get a specific date from the last 10 days based on index (0-9) -func getDateFromLast10Days(dayIndex int) time.Time { - now := time.Now() - // Get midnight today - today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - // Subtract the specified number of days (0 = today, 9 = 9 days ago) - return today.AddDate(0, 0, -dayIndex) -} - -// Generate log content with chronologically ordered timestamps -func generateRandomLogContent(accountID string, fileDate time.Time, numLines int) string { - logLevels := []string{"INFO", "DEBUG", "WARN", "ERROR"} - operations := []string{"READ", "WRITE", "UPDATE", "DELETE", "LOGIN", "LOGOUT", "PROCESS"} - - var content string - - // Get just the date part (year, month, day) and start at 6:00 AM - year, month, day := fileDate.Date() - startTime := time.Date(year, month, day, 6, 0, 0, 0, fileDate.Location()) - - // Calculate time interval between log entries to spread them throughout the day (until 9:00 PM) - dayDuration := 15 * time.Hour // 6:00 AM to 9:00 PM - interval := dayDuration / time.Duration(numLines) - - // Add a small random variation to each interval (±30 seconds) - // to make logs look more natural while maintaining chronological order - maxVariation := 30 * time.Second - - currentTime := startTime - - for i := 0; i < numLines; i++ { - // Add a small random variation to the timestamp to make it look more natural - // but still maintain chronological order - variation := time.Duration(rand.Int63n(int64(maxVariation))) - (maxVariation / 2) - timestamp := currentTime.Add(variation) - - logLevel := logLevels[rand.Intn(len(logLevels))] - operation := operations[rand.Intn(len(operations))] - status := rand.Intn(2) == 0 // Random boolean - - statusStr := "SUCCESS" - if !status { - statusStr = "FAILURE" - } - - line := fmt.Sprintf("[%s] %s: Operation %s for account %s completed with %s [timestamp=%s]\n", - timestamp.Format("2006-01-02 15:04:05"), - logLevel, - operation, - accountID, - statusStr, - timestamp.Format("2006-01-02T15:04:05.000Z07:00")) - - content += line - - // Advance to the next timestamp - currentTime = currentTime.Add(interval) - } - - return content -} diff --git a/tests/acceptance/test_files/from_and_to.bats b/tests/acceptance/test_files/from_and_to.bats index b44e4fff..71db1874 100644 --- a/tests/acceptance/test_files/from_and_to.bats +++ b/tests/acceptance/test_files/from_and_to.bats @@ -2,6 +2,7 @@ load "$LIB_BATS_ASSERT/load.bash" load "$LIB_BATS_SUPPORT/load.bash" @test "verify --from works in tailpipe query" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { @@ -28,6 +29,7 @@ EOF } @test "verify --from works when ISO 8601 datetime is passed" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { @@ -54,6 +56,7 @@ EOF } @test "verify --from works when ISO 8601 datetime with milliseconds is passed" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { @@ -80,6 +83,7 @@ EOF } @test "verify --from works when RFC 3339 datetime with timezone is passed" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { diff --git a/tests/acceptance/test_files/partition_tests.bats b/tests/acceptance/test_files/partition_tests.bats index 4128964a..1272e1fb 100644 --- a/tests/acceptance/test_files/partition_tests.bats +++ b/tests/acceptance/test_files/partition_tests.bats @@ -55,6 +55,7 @@ EOF } @test "verify invalid filter syntax" { + skip "TODO - re-enable this test, when the error handling is fixed in ducklake - https://github.com/turbot/tailpipe/issues/544" # Create a test partition configuration with invalid filter cat << EOF > $TAILPIPE_INSTALL_DIR/config/invalid_filter_test.tpc partition "chaos_all_columns" "invalid_filter_test_1" { From de2d5b50722effa4cfbd6c5db48d3919366ea61b Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 9 Sep 2025 10:28:28 +0100 Subject: [PATCH 28/61] feat: refactor connect command to return init scripts and consolidate DuckDB setup. Closes #550. - Query command uses filtered db connection to re-add support for `from` and `to` flags - Update `connect` command to return init script instead of direct database path - Move connectDucklake from pipe-fittings to tailpipe internal/database - Add WithDuckLakeReadonly option for DuckDB creation, with optional filters - Use a read only connection DB wherever possible - Add GetDucklakeInitCommands to centralize DuckDB initialization - Refactor connectDucklake to be a DuckDb method using shared SQL logic - Consolidate partition pattern logic and move to parquet package - Add cleanupOldInitFiles to clean up old initialization files - Update generateInitFilename to use "tailpipe_init" prefix - Add SqlCommand type to encapsulate SQL commands and descriptions --- cmd/collect.go | 69 +--- cmd/collect_test.go | 250 -------------- cmd/compact.go | 64 ++-- cmd/connect.go | 372 ++++++++++++++++++++- cmd/partition.go | 12 +- cmd/query.go | 12 +- cmd/table.go | 10 +- go.mod | 2 +- go.sum | 3 +- internal/collector/collector.go | 9 +- internal/config/partition.go | 2 +- internal/constants/connect.go | 6 + internal/database/duck_db.go | 221 +++++++++--- internal/database/duck_db_options.go | 18 +- internal/database/sql_command.go | 7 + internal/database/tables.go | 9 +- internal/database/views.go | 34 ++ internal/parquet/compact.go | 205 ++++++++---- internal/parquet/compaction_status.go | 102 +++++- internal/parquet/compaction_types.go | 62 +++- internal/parquet/ducklake.go | 1 + internal/parquet/migrate_tpindex.go | 178 ---------- internal/parquet/partition_key.go | 133 ++------ internal/parquet/partition_pattern.go | 89 ++++- internal/parquet/partition_pattern_test.go | 296 ++++++++++++++++ internal/parquet/reorder_metadata.go | 41 +++ internal/parse/decode.go | 4 +- internal/query/execute_test.go | 3 + 28 files changed, 1400 insertions(+), 814 deletions(-) create mode 100644 internal/constants/connect.go create mode 100644 internal/database/sql_command.go create mode 100644 internal/database/views.go delete mode 100644 internal/parquet/migrate_tpindex.go create mode 100644 internal/parquet/partition_pattern_test.go create mode 100644 internal/parquet/reorder_metadata.go diff --git a/cmd/collect.go b/cmd/collect.go index 7ad4f440..28350d60 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,12 +5,11 @@ import ( "errors" "fmt" "log/slog" - "strconv" "os" + "strconv" "strings" "time" - "github.com/danwakefield/fnmatch" "github.com/hashicorp/hcl/v2" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -25,6 +24,7 @@ import ( "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" ) @@ -72,7 +72,6 @@ func runCollectCmd(cmd *cobra.Command, args []string) { if err != nil { if errors.Is(err, context.Canceled) { - err = nil fmt.Println("Collection cancelled.") //nolint:forbidigo // ui output } else { error_helpers.ShowError(ctx, err) @@ -216,7 +215,7 @@ func getPartitions(args []string) ([]*config.Partition, error) { continue } - partitionNames, err := getPartitionsForArg(maps.Keys(tailpipeConfig.Partitions), arg) + partitionNames, err := parquet.GetPartitionsForArg(tailpipeConfig.Partitions, arg) if err != nil { errorList = append(errorList, err) } else if len(partitionNames) == 0 { @@ -320,68 +319,6 @@ func getSyntheticPartition(arg string) (*config.Partition, bool) { return partition, true } -func getPartitionsForArg(partitions []string, arg string) ([]string, error) { - tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) - if err != nil { - return nil, err - } - // now match the partition - var res []string - for _, partition := range partitions { - pattern := tablePattern + "." + partitionPattern - if fnmatch.Match(pattern, partition, fnmatch.FNM_CASEFOLD) { - res = append(res, partition) - } - } - return res, nil -} - -func getPartitionMatchPatternsForArg(partitions []string, arg string) (string, string, error) { - var tablePattern, partitionPattern string - parts := strings.Split(arg, ".") - switch len(parts) { - case 1: - var err error - tablePattern, partitionPattern, err = getPartitionMatchPatternsForSinglePartName(partitions, arg) - if err != nil { - return "", "", err - } - case 2: - // use the args as provided - tablePattern = parts[0] - partitionPattern = parts[1] - default: - return "", "", fmt.Errorf("invalid partition name: %s", arg) - } - return tablePattern, partitionPattern, nil -} - -// getPartitionMatchPatternsForSinglePartName returns the table and partition patterns for a single part name -// e.g. if the arg is "aws*" -func getPartitionMatchPatternsForSinglePartName(partitions []string, arg string) (string, string, error) { - var tablePattern, partitionPattern string - // '*' is not valid for a single part arg - if arg == "*" { - return "", "", fmt.Errorf("invalid partition name: %s", arg) - } - // check whether there is table with this name - // partitions is a list of Unqualified names, i.e. . - for _, partition := range partitions { - table := strings.Split(partition, ".")[0] - - // if the arg matches a table name, set table pattern to the arg and partition pattern to * - if fnmatch.Match(arg, table, fnmatch.FNM_CASEFOLD) { - tablePattern = arg - partitionPattern = "*" - return tablePattern, partitionPattern, nil - } - } - // so there IS NOT a table with this name - set table pattern to * and user provided partition name - tablePattern = "*" - partitionPattern = arg - return tablePattern, partitionPattern, nil -} - func setExitCodeForCollectError(err error) { // if exit code already set, leave as is if exitCode != 0 || err == nil { diff --git a/cmd/collect_test.go b/cmd/collect_test.go index a5b27db8..73fc0c99 100644 --- a/cmd/collect_test.go +++ b/cmd/collect_test.go @@ -1,261 +1,11 @@ package cmd import ( - "reflect" "testing" "github.com/turbot/tailpipe/internal/config" ) -func Test_getPartition(t *testing.T) { - type args struct { - partitions []string - name string - } - tests := []struct { - name string - args args - want []string - wantErr bool - }{ - { - name: "Invalid partition name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "*", - }, - wantErr: true, - }, - { - name: "Full partition name, exists", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.p1", - }, - want: []string{"aws_s3_cloudtrail_log.p1"}, - }, - { - name: "Full partition name, does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.p3", - }, - want: nil, - }, - { - name: "Table name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - }, - { - name: "Table name (exists) with wildcard", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.*", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - }, - { - name: "Table name (exists) with ?", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.p?", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - }, - { - name: "Table name (exists) with non matching partition wildacard", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.d*?", - }, - want: nil, - }, - { - name: "Table name (does not exist)) with wildcard", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "foo.*", - }, - want: nil, - }, - { - name: "Partition short name, exists", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "p1", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, - }, - { - name: "Table wildcard, partition short name, exists", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "*.p1", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, - }, - { - name: "Partition short name, does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "p3", - }, - want: nil, - }, - { - name: "Table wildcard, partition short name, does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "*.p3", - }, - want: nil, - }, - { - name: "Table wildcard, no dot", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "aws*", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getPartitionsForArg(tt.args.partitions, tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("getPartitions() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getPartitions() got = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_getPartitionMatchPatternsForArg(t *testing.T) { - type args struct { - partitions []string - arg string - } - tests := []struct { - name string - args args - wantTablePattern string - wantPartPattern string - wantErr bool - }{ - { - name: "Valid table and partition pattern", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - arg: "aws_s3_cloudtrail_log.p1", - }, - wantTablePattern: "aws_s3_cloudtrail_log", - wantPartPattern: "p1", - }, - { - name: "Wildcard partition pattern", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, - arg: "aws_s3_cloudtrail_log.*", - }, - wantTablePattern: "aws_s3_cloudtrail_log", - wantPartPattern: "*", - }, - { - name: "Wildcard in table and partition both", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, - arg: "aws*.*", - }, - wantTablePattern: "aws*", - wantPartPattern: "*", - }, - { - name: "Wildcard table pattern", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, - arg: "*.p1", - }, - wantTablePattern: "*", - wantPartPattern: "p1", - }, - { - name: "Invalid partition name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - arg: "*", - }, - wantErr: true, - }, - { - name: "Table exists without partition", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - arg: "aws_s3_cloudtrail_log", - }, - wantTablePattern: "aws_s3_cloudtrail_log", - wantPartPattern: "*", - }, - { - name: "Partition only, multiple tables", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, - arg: "p1", - }, - wantTablePattern: "*", - wantPartPattern: "p1", - }, - { - name: "Invalid argument with multiple dots", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1"}, - arg: "aws.s3.cloudtrail", - }, - wantErr: true, - }, - { - name: "Non-existing table name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1"}, - arg: "non_existing_table.p1", - }, - wantTablePattern: "non_existing_table", - wantPartPattern: "p1", - }, - { - name: "Partition name does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1"}, - arg: "p2", - }, - wantTablePattern: "*", - wantPartPattern: "p2", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotTablePattern, gotPartPattern, err := getPartitionMatchPatternsForArg(tt.args.partitions, tt.args.arg) - if (err != nil) != tt.wantErr { - t.Errorf("getPartitionMatchPatternsForArg() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotTablePattern != tt.wantTablePattern { - t.Errorf("getPartitionMatchPatternsForArg() gotTablePattern = %v, want %v", gotTablePattern, tt.wantTablePattern) - } - if gotPartPattern != tt.wantPartPattern { - t.Errorf("getPartitionMatchPatternsForArg() gotPartPattern = %v, want %v", gotPartPattern, tt.wantPartPattern) - } - }) - } -} - func Test_getSyntheticPartition(t *testing.T) { tests := []struct { name string diff --git a/cmd/compact.go b/cmd/compact.go index 10ffd7e0..550854e8 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -12,7 +12,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" - "github.com/turbot/go-kit/types" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" @@ -53,7 +52,13 @@ func runCompactCmd(cmd *cobra.Command, args []string) { } if err != nil { setExitCodeForCompactError(err) - error_helpers.ShowError(ctx, err) + + if errors.Is(err, context.Canceled) { + //nolint:forbidigo // ui + fmt.Println("Compact cancelled") + } else { + error_helpers.ShowError(ctx, err) + } } }() @@ -65,14 +70,7 @@ func runCompactCmd(cmd *cobra.Command, args []string) { slog.Info("Compacting parquet files") - // if the flag was provided, migrate the tp_index files - if viper.GetBool(pconstants.ArgReindex) { - // TODO #DL update tpIndex migration for ducklake - // https://github.com/turbot/tailpipe/issues/475 - panic("Reindexing is not yet implemented for ducklake") - } - - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + db, err := database.NewDuckDb(database.WithDuckLake()) error_helpers.FailOnError(err) defer db.Close() @@ -82,41 +80,30 @@ func runCompactCmd(cmd *cobra.Command, args []string) { } // Get table and partition patterns - patterns, err := getPartitionPatterns(args, maps.Keys(config.GlobalConfig.Partitions)) + patterns, err := parquet.GetPartitionPatternsForArgs(maps.Keys(config.GlobalConfig.Partitions), args...) error_helpers.FailOnErrorWithMessage(err, "failed to get partition patterns") // do the compaction - status, err := doCompaction(ctx, db, patterns) - if errors.Is(err, context.Canceled) { - // TODO verify - // clear error so we don't show it with normal error reporting - err = nil - } + status, err := doCompaction(ctx, db, patterns) + // print the final status + statusString := status.VerboseString() if err == nil { - // print the final status - statusString := status.VerboseString() - if statusString == "" { - statusString = "No files to compact." - } - if ctx.Err() != nil { - // instead show the status as cancelled - statusString = "Compaction cancelled: " + statusString - } - fmt.Println(statusString) //nolint:forbidigo // ui } // defer block will show the error } -func doCompaction(ctx context.Context, db *database.DuckDb, patterns []parquet.PartitionPattern) (*parquet.CompactionStatus, error) { +func doCompaction(ctx context.Context, db *database.DuckDb, patterns []*parquet.PartitionPattern) (*parquet.CompactionStatus, error) { s := spinner.New( spinner.CharSets[14], 100*time.Millisecond, spinner.WithHiddenCursor(true), spinner.WithWriter(os.Stdout), ) + // if the flag was provided, migrate the tp_index files + reindex := viper.GetBool(pconstants.ArgReindex) // start and stop spinner around the processing s.Start() @@ -127,30 +114,17 @@ func doCompaction(ctx context.Context, db *database.DuckDb, patterns []parquet.P updateTotals := func(updatedStatus parquet.CompactionStatus) { status = &updatedStatus - s.Suffix = fmt.Sprintf(" compacting parquet files (%0.1f%% of %s rows)", status.ProgressPercent, types.ToHumanisedString(status.TotalRows)) + if status.Message != "" { + s.Suffix = " compacting parquet files: " + status.Message + } } // do compaction - err := parquet.CompactDataFiles(ctx, db, updateTotals, patterns...) + err := parquet.CompactDataFiles(ctx, db, updateTotals, reindex, patterns...) return status, err } -// getPartitionPatterns returns the table and partition patterns for the given partition args -func getPartitionPatterns(partitionArgs []string, partitions []string) ([]parquet.PartitionPattern, error) { - var res []parquet.PartitionPattern - for _, arg := range partitionArgs { - tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) - if err != nil { - return nil, fmt.Errorf("error processing partition arg '%s': %w", arg, err) - } - - res = append(res, parquet.PartitionPattern{Table: tablePattern, Partition: partitionPattern}) - } - - return res, nil -} - func setExitCodeForCompactError(err error) { // set exit code only if an error occurred and no exit code is already set if exitCode != 0 || err == nil { diff --git a/cmd/connect.go b/cmd/connect.go index 3b03995f..f56d0ccc 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -1,23 +1,31 @@ package cmd import ( + "context" "encoding/json" "fmt" + "golang.org/x/exp/maps" + "log" "os" "path/filepath" "strings" + "time" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/thediveo/enumflag/v2" - "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" "github.com/turbot/pipe-fittings/v2/connection" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/error_helpers" + pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" + "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/parquet" ) // variable used to assign the output mode flag @@ -28,15 +36,55 @@ func connectCmd() *cobra.Command { Use: "connect [flags]", Args: cobra.ArbitraryArgs, Run: runConnectCmd, - Short: "Return a connection string for the ducklake database", - Long: "Return a connection string for the ducklake database.", + Short: "Return the path of SQL script to initialise DuckDB to use the tailpipe database", + Long: `Return the path of SQL script to initialise DuckDB to use the tailpipe database. + +The generated SQL script contains: +- DuckDB extension installations (sqlite, ducklake) +- Database attachment configuration +- View definitions with optional filters + +Examples: + # Basic usage - generate init script + tailpipe connect + + # Filter by time range + tailpipe connect --from "2024-01-01" --to "2024-01-31" + + # Filter by specific partitions + tailpipe connect --partition "aws_cloudtrail_log.recent" + + # Filter by indexes with wildcards + tailpipe connect --index "prod-*" --index "staging" + + # Combine multiple filters + tailpipe connect --from "T-7d" --partition "aws.*" --index "prod-*" + + # Output as JSON + tailpipe connect --output json + +Time formats supported: + - ISO 8601 date: 2024-01-01 + - ISO 8601 datetime: 2024-01-01T15:04:05 + - RFC 3339 with timezone: 2024-01-01T15:04:05Z + - Relative time: T-7d, T-2Y, T-10m, T-180d + +The generated script can be used with DuckDB: + duckdb -init /path/to/generated/script.sql`, } + // args `from` and `to` accept: + // - ISO 8601 date (2024-01-01) + // - ISO 8601 datetime (2006-01-02T15:04:05) + // - ISO 8601 datetime with ms (2006-01-02T15:04:05.000) + // - RFC 3339 datetime with timezone (2006-01-02T15:04:05Z07:00) + // - relative time formats (T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) + cmdconfig.OnCmd(cmd). - AddStringFlag(pconstants.ArgFrom, "", "Specify the start time", cmdconfig.FlagOptions.Deprecated("'from' is not supported with ducklake")). - AddStringFlag(pconstants.ArgTo, "", "Specify the end time", cmdconfig.FlagOptions.Deprecated("'to'' is not supported with ducklake")). - AddStringSliceFlag(pconstants.ArgIndex, nil, "Specify the index to use", cmdconfig.FlagOptions.Deprecated("'index' is not supported with ducklake")). - AddStringSliceFlag(pconstants.ArgPartition, nil, "Specify the partition to use", cmdconfig.FlagOptions.Deprecated("'partition' is not supported with ducklake")). + AddStringFlag(pconstants.ArgFrom, "", "Specify the start time"). + AddStringFlag(pconstants.ArgTo, "", "Specify the end time"). + AddStringSliceFlag(pconstants.ArgIndex, nil, "Specify the index to use"). + AddStringSliceFlag(pconstants.ArgPartition, nil, "Specify the partition to use"). AddVarFlag(enumflag.New(&connectOutputMode, pconstants.ArgOutput, constants.ConnectOutputModeIds, enumflag.EnumCaseInsensitive), pconstants.ArgOutput, fmt.Sprintf("Output format; one of: %s", strings.Join(constants.FlagValues(constants.PluginOutputModeIds), ", "))) @@ -45,9 +93,17 @@ func connectCmd() *cobra.Command { } func runConnectCmd(cmd *cobra.Command, _ []string) { + var err error + var initFilePath string ctx := cmd.Context() - dataPath := config.GlobalWorkspaceProfile.GetDataDir() - dbFilePath := filepath.Join(dataPath, "metadata.sqlite") + + defer func() { + if r := recover(); r != nil { + err = helpers.ToError(r) + } + setExitCodeForConnectError(err) + displayOutput(ctx, initFilePath, err) + }() // if diagnostic mode is set, print out config and return if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { @@ -55,18 +111,132 @@ func runConnectCmd(cmd *cobra.Command, _ []string) { return } + initFilePath, err = generateInitFile(ctx) + + // we are done - the defer block will print either the filepath (if successful) or the error (if not) + +} + +func generateInitFile(ctx context.Context) (string, error) { + // cleanup the old db files if not in use + err := cleanupOldInitFiles() + if err != nil { + return "", err + } + + // generate a filename to write the init sql to, inside the data dir + initFilePath := generateInitFilename(config.GlobalWorkspaceProfile.GetDataDir()) + + // get the sql to attach readonly to the database + commands := database.GetDucklakeInitCommands(true) + + // build the filters from the to, from and index args + // these will be used in the view definitions + filters, err := getFilters() + if err != nil { + return "", fmt.Errorf("error building filters: %w", err) + } + + // create a temporary duckdb instance pass to get the view definitions + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) + if err != nil { + return "", fmt.Errorf("failed to create duckdb: %w", err) + } + defer db.Close() + + // get the view creation SQL, with filters applied + viewCommands, err := database.GetCreateViewsSql(ctx, db, filters...) + if err != nil { + return "", err + } + commands = append(commands, viewCommands...) + + // now build a string + var str strings.Builder + for _, cmd := range commands { + str.WriteString(fmt.Sprintf("-- %s\n%s;\n\n", cmd.Description, cmd.Command)) + } + // write out the init file + err = os.WriteFile(initFilePath, []byte(str.String()), 0644) //nolint:gosec // we want the init file to be readable + if err != nil { + return "", fmt.Errorf("failed to write init file: %w", err) + } + return initFilePath, err +} + +// cleanupOldInitFiles deletes old db init files (older than a day) +func cleanupOldInitFiles() error { + baseDir := pfilepaths.GetDataDir() + log.Printf("[INFO] Cleaning up old init files in %s\n", baseDir) + cutoffTime := time.Now().Add(-constants.InitFileMaxAge) // Files older than 1 day + + // The baseDir ("$TAILPIPE_INSTALL_DIR/data") is expected to have subdirectories for different workspace + // profiles(default, work etc). Each subdirectory may contain multiple .db files. + // Example structure: + // data/ + // ├── default/ + // │ ├── tailpipe_init_20250115182129.sql + // │ ├── tailpipe_init_20250115193816.sql + // │ └── ... + // ├── work/ + // │ ├── tailpipe_init_20250115182129.sql + // │ ├── tailpipe_init_20250115193816.sql + // │ └── ... + // So we traverse all these subdirectories for each workspace and process the relevant files. + err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error accessing path %s: %v", path, err) + } + + // skip directories and non-`.sql` files + if info.IsDir() || !strings.HasSuffix(info.Name(), ".sql") { + return nil + } + + // only process `tailpipe_init_*.sql` files + if !strings.HasPrefix(info.Name(), "tailpipe_init_") { + return nil + } + + // check if the file is older than the cutoff time + if info.ModTime().After(cutoffTime) { + log.Printf("[DEBUG] Skipping deleting file %s(%s) as it is not older than %s\n", path, info.ModTime().String(), cutoffTime) + return nil + } + + err = os.Remove(path) + if err != nil { + log.Printf("[INFO] Failed to delete db file %s: %v", path, err) + } else { + log.Printf("[DEBUG] Cleaned up old unused db file: %s\n", path) + } + + return nil + }) + + if err != nil { + return err + } + return nil + +} + +func displayOutput(ctx context.Context, initFilePath string, err error) { switch viper.GetString(pconstants.ArgOutput) { case pconstants.OutputFormatText: - // output the filepath - connectionString := backend.GetDucklakeConnectionString(dbFilePath, dataPath) - fmt.Println(connectionString) //nolint:forbidigo // ui output - + if err == nil { + // output the filepath + fmt.Println(initFilePath) //nolint:forbidigo // ui output + } else { + error_helpers.ShowError(ctx, err) + } case pconstants.OutputFormatJSON: res := connection.TailpipeConnectResponse{ - DatabaseFilepath: dbFilePath, - DataPath: dataPath, + InitScriptPath: initFilePath, + } + if err != nil { + res.Error = err.Error() } - b, err := json.Marshal(res) if err == nil { fmt.Println(string(b)) //nolint:forbidigo // ui output @@ -79,3 +249,173 @@ func runConnectCmd(cmd *cobra.Command, _ []string) { error_helpers.ShowError(ctx, fmt.Errorf("unsupported output format %q", viper.GetString(pconstants.ArgOutput))) } } + +// getFilters builds a set of SQL filters based on the provided command line args +// supported args are `from`, `to`, `partition` and `index` +func getFilters() ([]string, error) { + var result []string + if viper.IsSet(pconstants.ArgFrom) { + from := viper.GetString(pconstants.ArgFrom) + // parse the string as time.Time + // arg `from` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), + // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) + t, err := parse.ParseTime(from, time.Now()) + if err != nil { + return nil, fmt.Errorf("invalid date format for 'from': %s", from) + } + // format as SQL timestamp + fromTimestamp := t.Format(time.DateTime) + result = append(result, fmt.Sprintf("tp_timestamp >= timestamp '%s'", fromTimestamp)) + } + if viper.IsSet(pconstants.ArgTo) { + to := viper.GetString(pconstants.ArgTo) + // parse the string as time.Time + // arg `to` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), + // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) + t, err := parse.ParseTime(to, time.Now()) + if err != nil { + return nil, fmt.Errorf("invalid date format for 'to': %s", to) + } + // format as SQL timestamp + toTimestamp := t.Format(time.DateTime) + result = append(result, fmt.Sprintf("tp_timestamp <= timestamp '%s'", toTimestamp)) + } + if viper.IsSet(pconstants.ArgPartition) { + // we have loaded tailpipe config by this time + availablePartitions := config.GlobalConfig.Partitions + partitionArgs := viper.GetStringSlice(pconstants.ArgPartition) + // get the SQL filters from the provided partition + sqlFilters, err := getPartitionSqlFilters(partitionArgs, maps.Keys(availablePartitions)) + if err != nil { + return nil, err + } + result = append(result, sqlFilters) + } + if viper.IsSet(pconstants.ArgIndex) { + indexArgs := viper.GetStringSlice(pconstants.ArgIndex) + // get the SQL filters from the provided index + sqlFilters, err := getIndexSqlFilters(indexArgs) + if err != nil { + return nil, err + } + result = append(result, sqlFilters) + } + return result, nil +} + +// getPartitionSqlFilters builds SQL filters for the provided partition args +func getPartitionSqlFilters(partitionArgs []string, availablePartitions []string) (string, error) { + // Get table and partition patterns using GetPartitionPatternsForArgs + patterns, err := parquet.GetPartitionPatternsForArgs(availablePartitions, partitionArgs...) + if err != nil { + return "", fmt.Errorf("error processing partition args: %w", err) + } + + // Handle the case when patterns are empty + if len(patterns) == 0 { + return "", nil + } + + // Replace wildcards from '*' to '%' for SQL compatibility + sqlPatterns := replaceWildcards(patterns) + + var conditions []string + + for i := 0; i < len(sqlPatterns); i++ { + table := sqlPatterns[i].Table + partition := sqlPatterns[i].Partition + + var tableCondition, partitionCondition string + + // If there is no wildcard, use '=' instead of like + if table == "%" { + // Skip table condition if full wildcard + tableCondition = "" + } else if strings.Contains(table, "%") { + tableCondition = fmt.Sprintf("tp_table like '%s'", table) + } else { + tableCondition = fmt.Sprintf("tp_table = '%s'", table) + } + + if partition == "%" { + // Skip partition condition if full wildcard + partitionCondition = "" + } else if strings.Contains(partition, "%") { + partitionCondition = fmt.Sprintf("tp_partition like '%s'", partition) + } else { + partitionCondition = fmt.Sprintf("tp_partition = '%s'", partition) + } + + // Remove empty conditions and combine valid ones + if tableCondition != "" && partitionCondition != "" { + conditions = append(conditions, fmt.Sprintf("(%s and %s)", tableCondition, partitionCondition)) + } else if tableCondition != "" { + conditions = append(conditions, tableCondition) + } else if partitionCondition != "" { + conditions = append(conditions, partitionCondition) + } + } + + // Combine all conditions with OR + sqlFilters := strings.Join(conditions, " OR ") + + return sqlFilters, nil +} + +// getIndexSqlFilters builds SQL filters for the provided index args +func getIndexSqlFilters(indexArgs []string) (string, error) { + // Return empty if no indexes provided + if len(indexArgs) == 0 { + return "", nil + } + + // Build SQL filter based on whether wildcards are present + var conditions []string + for _, index := range indexArgs { + if index == "*" { + // Skip index condition if full wildcard + conditions = append(conditions, "") + } else if strings.Contains(index, "*") { + // Replace '*' wildcard with '%' for SQL like compatibility + index = strings.ReplaceAll(index, "*", "%") + conditions = append(conditions, fmt.Sprintf("cast(tp_index as varchar) like '%s'", index)) + } else { + // Exact match using '=' + conditions = append(conditions, fmt.Sprintf("tp_index = '%s'", index)) + } + } + + // Combine all conditions with OR + sqlFilter := strings.Join(conditions, " OR ") + + return sqlFilter, nil +} + +// convert partition patterns with '*' wildcards to SQL '%' wildcards +func replaceWildcards(patterns []*parquet.PartitionPattern) []*parquet.PartitionPattern { + updatedPatterns := make([]*parquet.PartitionPattern, len(patterns)) + + for i, p := range patterns { + updatedPatterns[i] = &parquet.PartitionPattern{ + Table: strings.ReplaceAll(p.Table, "*", "%"), + Partition: strings.ReplaceAll(p.Partition, "*", "%")} + } + return updatedPatterns + +} + +func setExitCodeForConnectError(err error) { + // if exit code already set, leave as is + // NOTE: DO NOT set exit code if the output format is JSON + if exitCode != 0 || err == nil || viper.GetString(pconstants.ArgOutput) == pconstants.OutputFormatJSON { + return + } + + exitCode = 1 +} + +// generateInitFilename generates a temporary filename with a timestamp +func generateInitFilename(dataDir string) string { + timestamp := time.Now().Format("20060102150405") // e.g., 20241031103000 + return filepath.Join(dataDir, fmt.Sprintf("tailpipe_init_%s.sql", timestamp)) +} diff --git a/cmd/partition.go b/cmd/partition.go index 3c3a4485..c3b7a82b 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -76,7 +76,7 @@ func partitionListCmd() *cobra.Command { } func runPartitionListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler + // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPartitionListCmd start") @@ -94,7 +94,8 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { return } - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) error_helpers.FailOnError(err) defer db.Close() @@ -136,7 +137,7 @@ func partitionShowCmd() *cobra.Command { } func runPartitionShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler + // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPartitionShowCmd start") @@ -154,7 +155,8 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { return } - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) error_helpers.FailOnError(err) defer db.Close() @@ -268,7 +270,7 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { return } } - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + db, err := database.NewDuckDb(database.WithDuckLake()) error_helpers.FailOnError(err) defer db.Close() diff --git a/cmd/query.go b/cmd/query.go index 21cf7b4c..6a50f066 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -83,10 +83,16 @@ func runQueryCmd(cmd *cobra.Command, args []string) { return } - // get a connection to the database, with DuckLake enabled - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + // build the filters from the to, from and index args + filters, err := getFilters() if err != nil { - return + error_helpers.FailOnError(fmt.Errorf("error building filters: %w", err)) + } + + // now create a readonly connection to the database, passing in any filters + db, err := database.NewDuckDb(database.WithDuckLakeReadonly(filters...)) + if err != nil { + error_helpers.FailOnError(err) } defer db.Close() diff --git a/cmd/table.go b/cmd/table.go index 265e323e..403b467c 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -68,7 +68,7 @@ func tableListCmd() *cobra.Command { } func runTableListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler + // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runSourceListCmd start") @@ -86,7 +86,8 @@ func runTableListCmd(cmd *cobra.Command, args []string) { return } - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) error_helpers.FailOnError(err) defer db.Close() @@ -128,7 +129,7 @@ func tableShowCmd() *cobra.Command { } func runTableShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler + // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runTableShowCmd start") @@ -146,7 +147,8 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { return } - db, err := database.NewDuckDb(database.WithDuckLakeEnabled(true)) + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) error_helpers.FailOnError(err) defer db.Close() diff --git a/go.mod b/go.mod index 7fc2a9a4..a1b530e7 100644 --- a/go.mod +++ b/go.mod @@ -215,7 +215,7 @@ require ( github.com/tkrajina/go-reflector v0.5.8 // indirect github.com/turbot/pipes-sdk-go v0.12.0 // indirect github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect + github.com/ulikunitz/xz v0.5.14 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zclconf/go-cty-yaml v1.0.3 // indirect diff --git a/go.sum b/go.sum index 97175990..856b516a 100644 --- a/go.sum +++ b/go.sum @@ -1313,8 +1313,9 @@ github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x github.com/turbot/tailpipe-plugin-core v0.2.10/go.mod h1:dHzPUR1p5GksSvDqqEeZEvvJX6wTEwK/ZDev//9nSLw= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg= +github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 90120b43..d6c696bc 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -89,10 +89,10 @@ func New(pluginManager *plugin.PluginManager, partition *config.Partition, cance c.sourcePath = sourcePath // create the DuckDB connection - // load json and inet extension in addition to the DuckLake extension - the convertor will need them + // load inet extension in addition to the DuckLake extension db, err := database.NewDuckDb( database.WithDuckDbExtensions(pconstants.DuckDbExtensions), - database.WithDuckLakeEnabled(true), + database.WithDuckLake(), ) if err != nil { @@ -269,7 +269,10 @@ func (c *Collector) Compact(ctx context.Context) error { } partitionPattern := parquet.NewPartitionPattern(c.partition) - err := parquet.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, partitionPattern) + // NOTE: we DO NOT reindex when compacting after collection + reindex := false + + err := parquet.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, reindex, &partitionPattern) if err != nil { return fmt.Errorf("failed to compact data files: %w", err) diff --git a/internal/config/partition.go b/internal/config/partition.go index e9a0c655..8086f82f 100644 --- a/internal/config/partition.go +++ b/internal/config/partition.go @@ -52,7 +52,7 @@ type Partition struct { // an option filter in the format of a SQL where clause Filter string `cty:"filter"` // the sql column to use for the tp_index - TpIndexColumn string `cty:"tp_index_column"` + TpIndexColumn string `cty:"tp_index"` // if this is a synthetic partition for testing, this will be non-null SyntheticMetadata *SyntheticMetadata diff --git a/internal/constants/connect.go b/internal/constants/connect.go new file mode 100644 index 00000000..c91b9948 --- /dev/null +++ b/internal/constants/connect.go @@ -0,0 +1,6 @@ +package constants + +import "time" + +// InitFileMaxAge is the maximum age of an db init file before it is cleaned up +const InitFileMaxAge = 24 * time.Hour diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index e0460ce6..ee8f46b3 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -6,8 +6,8 @@ import ( "fmt" "log/slog" "os" + "strings" - "github.com/turbot/pipe-fittings/v2/backend" pconstants "github.com/turbot/pipe-fittings/v2/constants" pf "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/tailpipe/internal/config" @@ -26,90 +26,81 @@ type DuckDb struct { tempDir string maxMemoryMb int ducklakeEnabled bool + // create a read only connection to ducklake + duckLakeReadOnly bool + + // a list of view filters - if this is set, we create a set of views in the database, one per table, + // applying the specified filter + // NOTE: if view filters are specified, the connection is set to READ ONLY mode (even if read only option is not set) + viewFilters []string } func NewDuckDb(opts ...DuckDbOpt) (_ *DuckDb, err error) { slog.Info("Initializing DuckDB connection") - w := &DuckDb{} + d := &DuckDb{} for _, opt := range opts { - opt(w) + opt(d) } defer func() { if err != nil { // If an error occurs during initialization, close the DB connection if it was opened - if w.DB != nil { - _ = w.DB.Close() + if d.DB != nil { + _ = d.DB.Close() } - w.DB = nil // ensure DB is nil to avoid further operations on a closed connection + d.DB = nil // ensure DB is nil to avoid further operations on a closed connection } }() // Connect to DuckDB - db, err := sql.Open("duckdb", w.dataSourceName) + db, err := sql.Open("duckdb", d.dataSourceName) if err != nil { return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) } - w.DB = db + d.DB = db // for duckdb, limit connections to 1 - DuckDB is designed for single-connection usage - w.DB.SetMaxOpenConns(1) + d.SetMaxOpenConns(1) + + // set the extension directory + if _, err := d.DB.Exec("set extension_directory = ?;", pf.EnsurePipesDuckDbExtensionsDir()); err != nil { + return nil, fmt.Errorf("failed to set extension_directory: %w", err) + } - if len(w.extensions) > 0 { - // install and load the JSON extension - if err := w.installAndLoadExtensions(); err != nil { + if len(d.extensions) > 0 { + // set extension dir and install any specified extensions + if err := d.installAndLoadExtensions(); err != nil { return nil, fmt.Errorf(": %w", err) } } - if w.ducklakeEnabled { - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - // TODO #DL tactical - for now check env for data dir override - // remove this for prod release https://github.com/turbot/tailpipe/issues/520 - if envDir := os.Getenv("TAILPIPE_DATA_DIR"); envDir != "" { - dataDir = envDir - } - - ducklakeDb := config.GlobalWorkspaceProfile.GetDucklakeDbPath() - - if err := backend.ConnectDucklake(context.Background(), db, ducklakeDb, dataDir); err != nil { + if d.ducklakeEnabled { + if err := d.connectDucklake(context.Background()); err != nil { return nil, fmt.Errorf("failed to connect to DuckLake: %w", err) } - - // Set the default catalog to tailpipe_ducklake to avoid catalog context issues - if _, err := db.Exec(`use "tailpipe_ducklake"`); err != nil { - return nil, fmt.Errorf("failed to set default catalog: %w", err) - } } - // Configure DuckDB's temp directory: - // - If WithTempDir option was provided, use that directory - // - Otherwise, use the collection temp directory (a subdirectory in the user's home directory - // where temporary files for data collection are stored) - tempDir := w.tempDir - if tempDir == "" { - baseDir := filepaths.EnsureCollectionTempDir() - // Create a unique subdirectory with 'duckdb-' prefix - // it is important to use a unique directory for each DuckDB instance as otherwise temp files from - // different instances can conflict with each other, causing memory swapping issues - uniqueTempDir, err := os.MkdirTemp(baseDir, "duckdb-") + // view filters are used to create a database with a filtered set of data to query, + // used to support date filtering for the index command + if len(d.viewFilters) > 0 { + err = d.createFilteredViews(d.viewFilters) if err != nil { - return nil, fmt.Errorf("failed to create unique temp directory: %w", err) + return nil, fmt.Errorf("failed to create filtered views: %w", err) } - tempDir = uniqueTempDir } - if _, err := db.Exec("set temp_directory = ?;", tempDir); err != nil { - _ = w.Close() - return nil, fmt.Errorf("failed to set temp_directory: %w", err) + // Configure DuckDB's temp directory + if err := d.setTempDir(); err != nil { + return nil, fmt.Errorf("failed to set DuckDB temp directory: %w", err) } - if w.maxMemoryMb > 0 { - if _, err := db.Exec("set max_memory = ? || 'MB';", w.maxMemoryMb); err != nil { - _ = w.Close() + // set the max memory if specified + if d.maxMemoryMb > 0 { + if _, err := db.Exec("set max_memory = ? || 'MB';", d.maxMemoryMb); err != nil { + _ = d.Close() return nil, fmt.Errorf("failed to set max_memory: %w", err) } } - return w, nil + return d, nil } func (d *DuckDb) Query(query string, args ...any) (*sql.Rows, error) { @@ -166,11 +157,6 @@ func (d *DuckDb) installAndLoadExtensions() error { return nil } - // set the extension directory - if _, err := d.DB.Exec("set extension_directory = ?;", pf.EnsurePipesDuckDbExtensionsDir()); err != nil { - return fmt.Errorf("failed to set extension_directory: %w", err) - } - // install and load the extensions for _, extension := range pconstants.DuckDbExtensions { if _, err := d.DB.Exec(fmt.Sprintf("INSTALL '%s'; LOAD '%s';", extension, extension)); err != nil { @@ -180,3 +166,132 @@ func (d *DuckDb) installAndLoadExtensions() error { return nil } + +// connectDucklake connects the given DuckDB connection to DuckLake +func (d *DuckDb) connectDucklake(ctx context.Context) error { + // we share the same set of commands for tailpipe connection - get init commands and execute them + commands := GetDucklakeInitCommands(d.duckLakeReadOnly) + // if there are NO view filters, set the default catalog to ducklake + // if there are view filters, the views will be created in the default memory catalog so do not change the default + if len(d.viewFilters) == 0 { + commands = append(commands, SqlCommand{ + Description: "set default catalog to ducklake", + Command: fmt.Sprintf("use %s", pconstants.DuckLakeCatalog), + }) + } + + // tactical: if read only mode is set and the ducklake database does not exists, create it + // (creating a read only connection will FAIL if the ducklake database has not been created yet + // - writeable connections will create the database if it does not exist) + if d.duckLakeReadOnly { + if err := ensureDucklakeDb(); err != nil { + return fmt.Errorf("failed to ensure ducklake database exists: %w", err) + } + } + + for _, cmd := range commands { + slog.Info(cmd.Description, "command", cmd.Command) + _, err := d.ExecContext(ctx, cmd.Command) + if err != nil { + return fmt.Errorf("%s failed: %w", cmd.Description, err) + } + } + + return nil +} + +// ensureDucklakeDb checks if the ducklake database file exists, and if not, creates it by opening +// and closing a duckdb connection with ducklake enabled +// this is used if we we are creating a readonly db connection to ducklake +// - readonly connections will fail if the ducklake database does not exist +func ensureDucklakeDb() error { + //check db file exists + _, err := os.Stat(config.GlobalWorkspaceProfile.GetDucklakeDbPath()) + if err == nil { + // file exists - nothing to do + return nil + } + // create a duck db connection then close again + db, err := NewDuckDb(WithDuckLake()) + if err != nil { + return err + } + if err := db.Close(); err != nil { + return fmt.Errorf("failed to close duckdb connection: %w", err) + } + return nil + +} + +func (d *DuckDb) createFilteredViews(filters []string) error { + // get the sql to create the views based on the filters + viewSql, err := GetCreateViewsSql(context.Background(), d, d.viewFilters...) + if err != nil { + return fmt.Errorf("failed to get create views sql: %w", err) + } + // execute the commands to create the views + slog.Info("Creating views") + for _, cmd := range viewSql { + if _, err := d.Exec(cmd.Command); err != nil { + return fmt.Errorf("failed to create view: %w", err) + } + } + return nil +} + +// Configure DuckDB's temp directory +// - If WithTempDir option was provided, use that directory +// - Otherwise, use the collection temp directory (a subdirectory in the user's home directory +// where temporary files for data collection are stored) +func (d *DuckDb) setTempDir() error { + tempDir := d.tempDir + if tempDir == "" { + baseDir := filepaths.EnsureCollectionTempDir() + // Create a unique subdirectory with 'duckdb-' prefix + // it is important to use a unique directory for each DuckDB instance as otherwise temp files from + // different instances can conflict with each other, causing memory swapping issues + uniqueTempDir, err := os.MkdirTemp(baseDir, "duckdb-") + if err != nil { + return fmt.Errorf("failed to create unique temp directory: %w", err) + } + tempDir = uniqueTempDir + } + + if _, err := d.Exec("set temp_directory = ?;", tempDir); err != nil { + _ = d.Close() + return fmt.Errorf("failed to set temp_directory: %w", err) + } + return nil +} + +// GetDucklakeInitCommands returns the set of SQL commands required to initialize and connect to DuckLake. +// this is used both for tailpipe to connect to ducklake and also for tailpipe connect to build the init script +// It returns an ordered slice of SQL commands. +func GetDucklakeInitCommands(readonly bool) []SqlCommand { + attachOptions := []string{ + fmt.Sprintf("data_path '%s'", config.GlobalWorkspaceProfile.GetDataDir()), + "meta_journal_mode 'WAL'", + "meta_synchronous 'NORMAL'", + // TODO temp disable timeout + "meta_busy_timeout 0", + } + // if readonly mode is requested, add the option + if readonly { + attachOptions = append(attachOptions, "READ_ONLY") + } + attachQuery := fmt.Sprintf(`attach 'ducklake:sqlite:%s' AS %s ( + %s)`, + config.GlobalWorkspaceProfile.GetDucklakeDbPath(), + pconstants.DuckLakeCatalog, + strings.Join(attachOptions, ",\n\t")) + + commands := []SqlCommand{ + {Description: "install sqlite extension", Command: "install sqlite"}, + // TODO #DL change to using prod extension when stable + // https://github.com/turbot/tailpipe/issues/476 + // _, err = db.Exec("install ducklake;") + {Description: "install ducklake extension", Command: "force install ducklake from core_nightly"}, + {Description: "attach to ducklake database", Command: attachQuery}, + } + return commands +} diff --git a/internal/database/duck_db_options.go b/internal/database/duck_db_options.go index ad5d3f1a..9b303915 100644 --- a/internal/database/duck_db_options.go +++ b/internal/database/duck_db_options.go @@ -39,9 +39,21 @@ func WithMaxMemoryMb(maxMemoryMb int) DuckDbOpt { } } -// WithDuckLakeEnabled enables the DuckLake extension for DuckDB. -func WithDuckLakeEnabled(enabled bool) DuckDbOpt { +// WithDuckLake enables the DuckLake extension for DuckDB. +func WithDuckLake() DuckDbOpt { return func(d *DuckDb) { - d.ducklakeEnabled = enabled + d.ducklakeEnabled = true + } +} + +// WithDuckLakeReadonly enables the DuckLake extension in read-only mode. +// filters is an optional list of SQL filter expressions - if specified, a view will be created for each table in the database +// and the filters will be applied to the view. +// If no filters are specified, the ducklake attachment will be set as the default catalog so the tables can be accessed directly +func WithDuckLakeReadonly(filters ...string) DuckDbOpt { + return func(d *DuckDb) { + d.ducklakeEnabled = true + d.duckLakeReadOnly = true + d.viewFilters = filters } } diff --git a/internal/database/sql_command.go b/internal/database/sql_command.go new file mode 100644 index 00000000..beec0777 --- /dev/null +++ b/internal/database/sql_command.go @@ -0,0 +1,7 @@ +package database + +// SqlCommand represents a SQL command with its description. +type SqlCommand struct { + Description string + Command string +} diff --git a/internal/database/tables.go b/internal/database/tables.go index a3aafb03..e2684719 100644 --- a/internal/database/tables.go +++ b/internal/database/tables.go @@ -8,6 +8,7 @@ import ( "github.com/turbot/pipe-fittings/v2/constants" ) +// GetTables returns the list of tables in the DuckLake metadata catalog func GetTables(ctx context.Context, db *DuckDb) ([]string, error) { query := fmt.Sprintf("select table_name from %s.ducklake_table", constants.DuckLakeMetadataCatalog) @@ -29,8 +30,8 @@ func GetTables(ctx context.Context, db *DuckDb) ([]string, error) { return tableViews, nil } -func GetTableSchema(ctx context.Context, viewName string, db *DuckDb) (map[string]string, error) { - +// GetTableSchema returns the schema of the specified table as a map of column names to their types +func GetTableSchema(ctx context.Context, tableName string, db *DuckDb) (map[string]string, error) { query := fmt.Sprintf(`select c.column_name, c.column_type from %s.ducklake_table t join %s.ducklake_column c @@ -38,9 +39,9 @@ join %s.ducklake_column c where t.table_name = ? order by c.column_name;`, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) - rows, err := db.QueryContext(ctx, query, viewName) + rows, err := db.QueryContext(ctx, query, tableName) if err != nil { - return nil, fmt.Errorf("failed to get view schema for %s: %w", viewName, err) + return nil, fmt.Errorf("failed to get view schema for %s: %w", tableName, err) } defer rows.Close() diff --git a/internal/database/views.go b/internal/database/views.go new file mode 100644 index 00000000..ab9e5890 --- /dev/null +++ b/internal/database/views.go @@ -0,0 +1,34 @@ +package database + +import ( + "context" + "fmt" + "strings" + + pconstants "github.com/turbot/pipe-fittings/v2/constants" +) + +// GetCreateViewsSql returns the SQL commands to create views for all tables in the DuckLake catalog, +// +// applying the specified filters. +func GetCreateViewsSql(ctx context.Context, db *DuckDb, filters ...string) ([]SqlCommand, error) { + // get list of tables + tables, err := GetTables(ctx, db) + if err != nil { + return nil, fmt.Errorf("failed to get db tables: %w", err) + } + + // Step 3: Build the where clause + filterString := "" + if len(filters) > 0 { + filterString = fmt.Sprintf(" where %s", strings.Join(filters, " and ")) + } + + results := make([]SqlCommand, 0, len(tables)) + for _, table := range tables { + description := fmt.Sprintf("Create View for table %s", table) + command := fmt.Sprintf("create or replace view %s as select * from %s.%s%s", table, pconstants.DuckLakeCatalog, table, filterString) + results = append(results, SqlCommand{Description: description, Command: command}) + } + return results, nil +} diff --git a/internal/parquet/compact.go b/internal/parquet/compact.go index 7bea1227..601c121e 100644 --- a/internal/parquet/compact.go +++ b/internal/parquet/compact.go @@ -5,9 +5,11 @@ import ( "database/sql" "fmt" "log/slog" + "strings" "time" "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/tailpipe/internal/database" ) @@ -16,7 +18,7 @@ const ( maxCompactionRowsPerChunk = 5_000_000 ) -func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), patterns ...PartitionPattern) error { +func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), reindex bool, patterns ...*PartitionPattern) error { slog.Info("Compacting DuckLake data files") t := time.Now() @@ -32,13 +34,12 @@ func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func( return nil } - status, err := orderDataFiles(ctx, db, updateFunc, partitionKeys) + status, err := orderDataFiles(ctx, db, updateFunc, partitionKeys, reindex) if err != nil { slog.Error("Failed to compact DuckLake parquet files", "error", err) return err } - slog.Info("Expiring old DuckLake snapshots") // now expire unused snapshots if err := expirePrevSnapshots(ctx, db); err != nil { slog.Error("Failed to expire previous DuckLake snapshots", "error", err) @@ -55,7 +56,6 @@ func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func( // return nil, err // } - slog.Info("Cleaning up expired files in DuckLake") // delete unused files if err := cleanupExpiredFiles(ctx, db); err != nil { slog.Error("Failed to cleanup expired files", "error", err) @@ -63,12 +63,11 @@ func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func( } // get the file count after merging and cleanup - finalFileCount, err := getFileCountForPartitionKeys(ctx, db, partitionKeys) + err = status.getFinalFileCounts(ctx, db, partitionKeys) if err != nil { - return err + // just log + slog.Error("Failed to get final file counts", "error", err) } - // update status - status.FinalFiles = finalFileCount // set the compaction time status.Duration = time.Since(t) @@ -79,51 +78,84 @@ func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func( return nil } -//nolint: unused // TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 -// mergeParquetFiles combines adjacent parquet files in the DuckDB database. -func mergeParquetFiles(ctx context.Context, db *database.DuckDb) error { - if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { - if ctx.Err() != nil { - return err - } - return fmt.Errorf("failed to merge parquet files: %w", err) - } - return nil -} +// TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 +//// mergeParquetFiles combines adjacent parquet files in the DuckDB database. +//func mergeParquetFiles(ctx context.Context, db *database.DuckDb) error { +// if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { +// if ctx.Err() != nil { +// return err +// } +// return fmt.Errorf("failed to merge parquet files: %w", err) +// } +// return nil +//} // we order data files as follows: // - get list of partition keys matching patterns. For each key: // - analyze file fragmentation to identify overlapping time ranges // - for each overlapping time range, reorder all data in that range // - delete original unordered entries for that time range -func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), partitionKeys []*partitionKey) (*CompactionStatus, error) { +func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), partitionKeys []*partitionKey, reindex bool) (*CompactionStatus, error) { slog.Info("Ordering DuckLake data files") status := NewCompactionStatus() - // get total file and row count for status - iterating over partition keys - for _, pk := range partitionKeys { - status.InitialFiles += pk.fileCount - status.TotalRows += pk.stats.rowCount + // get total file and row count into status + err := status.getInitialCounts(ctx, db, partitionKeys) + if err != nil { + return nil, err } - // Process each partition + // map of table columns, allowing us to lazy load them + tableColumnLookup := make(map[string][]string) + + // build list of partition keys to reorder + var reorderList []*reorderMetadata + + status.Message = "identifying files to reorder" + updateFunc(*status) + + // Process each partition key to determine if we need to reorder for _, pk := range partitionKeys { - // determine which files are not time ordered - unorderedRanges, err := getUnorderedRangesForPartitionKey(ctx, db, pk) + // determine which files are not time ordered and build a set of time ranges which need reordering + // (NOTS: if we are reindexing, we need to reorder the ALL data for the partition key) + reorderMetadata, err := getTimeRangesToReorder(ctx, db, pk, reindex) if err != nil { slog.Error("failed to get unorderedRanges", "partition", pk, "error", err) return nil, err } + // if no files out of order, nothing to do - if len(unorderedRanges) == 0 { + if reorderMetadata != nil { + reorderList = append(reorderList, reorderMetadata) + } else { slog.Debug("Partition key is not out of order - skipping reordering", "tp_table", pk.tpTable, "tp_partition", pk.tpPartition, - "tp_index", pk.tpIndex, + // "tp_index", pk.tpIndex, "year", pk.year, "month", pk.month, ) - continue + } + } + + // now get the total rows to reorder + for _, rm := range reorderList { + status.InitialFiles += rm.pk.fileCount + status.RowsToCompact += rm.rowCount + } + + // clear message - it will be sent on next update func + status.Message = "" + + // now iterate over reorderlist to do reordering + for _, rm := range reorderList { + pk := rm.pk + + // get the columns for this table - check map first - if not present, read from metadata and populate the map + columns, err := getColumns(ctx, db, pk.tpTable, tableColumnLookup) + if err != nil { + slog.Error("failed to get columns", "table", pk.tpTable, "error", err) + return nil, err } tx, err := db.BeginTx(ctx, nil) @@ -132,26 +164,26 @@ func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(Co return nil, fmt.Errorf("failed to begin transaction for partition %v: %w", pk, err) } - slog.Info("Compacting partition entries", + slog.Debug("Compacting partition entries", "tp_table", pk.tpTable, "tp_partition", pk.tpPartition, "tp_index", pk.tpIndex, "year", pk.year, "month", pk.month, - "unorderedRanges",len(unorderedRanges), + "unorderedRanges", len(rm.unorderedRanges), ) // func to update status with number of rows compacted for this partition key - // - passed to compactAndOrderPartitionKeyEntries + // - passed to orderPartitionKey updateRowsFunc := func(rowsCompacted int64) { status.RowsCompacted += rowsCompacted if status.TotalRows > 0 { - status.ProgressPercent = (float64(status.RowsCompacted) / float64(status.TotalRows)) * 100 + status.UpdateProgress() } updateFunc(*status) } - if err := compactAndOrderPartitionKeyEntries(ctx, tx, pk, unorderedRanges, updateRowsFunc); err != nil { + if err := orderPartitionKey(ctx, tx, pk, rm, updateRowsFunc, reindex, columns); err != nil { slog.Error("failed to compact partition", "partition", pk, "error", err) txErr := tx.Rollback() if txErr != nil { @@ -184,11 +216,55 @@ func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(Co return status, nil } -// compactAndOrderPartitionKeyEntries processes overlapping time ranges for a partition key: +// getColumns retrieves column information for a table, checking the map first and reading from metadata if not present +func getColumns(ctx context.Context, db *database.DuckDb, table string, columns map[string][]string) ([]string, error) { + // Check if columns are already cached + if cachedColumns, exists := columns[table]; exists { + return cachedColumns, nil + } + + // Read top level columns from DuckLake metadata + query := fmt.Sprintf(` + select c.column_name + from %s.ducklake_column c + join %s.ducklake_table t on c.table_id = t.table_id + where t.table_name = ? + and t.end_snapshot is null + and c.end_snapshot is null + and c.parent_column is null + order by c.column_order`, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) + + rows, err := db.QueryContext(ctx, query, table) + if err != nil { + return nil, fmt.Errorf("failed to get columns for table %s: %w", table, err) + } + defer rows.Close() + + var columnNames []string + for rows.Next() { + var columnName string + if err := rows.Scan(&columnName); err != nil { + return nil, fmt.Errorf("failed to scan column: %w", err) + } + columnNames = append(columnNames, columnName) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error reading columns: %w", err) + } + + // Cache the columns for future use + columns[table] = columnNames + + // and return + return columnNames, nil +} + +// orderPartitionKey processes overlapping time ranges for a partition key: // - iterates over each unordered time range // - reorders all data within each time range (potentially in chunks for large ranges) // - deletes original unordered entries for that time range -func compactAndOrderPartitionKeyEntries(ctx context.Context, tx *sql.Tx, pk *partitionKey, unorderedRanges []unorderedDataTimeRange, updateRowsCompactedFunc func(int64)) error { +func orderPartitionKey(ctx context.Context, tx *sql.Tx, pk *partitionKey, rm *reorderMetadata, updateRowsCompactedFunc func(int64), reindex bool, columns []string) error { slog.Debug("partition statistics", "tp_table", pk.tpTable, @@ -196,15 +272,15 @@ func compactAndOrderPartitionKeyEntries(ctx context.Context, tx *sql.Tx, pk *par "tp_index", pk.tpIndex, "year", pk.year, "month", pk.month, - "row_count", pk.stats.rowCount, + "row_count", rm.rowCount, "total file_count", pk.fileCount, - "min_timestamp", pk.stats.minTimestamp, - "max_timestamp", pk.stats.maxTimestamp, - "total_ranges", len(unorderedRanges), + "min_timestamp", rm.minTimestamp, + "max_timestamp", rm.maxTimestamp, + "total_ranges", len(rm.unorderedRanges), ) // Process each overlapping time range - for i, timeRange := range unorderedRanges { + for i, timeRange := range rm.unorderedRanges { slog.Debug("processing overlapping time range", "range_index", i+1, "start_time", timeRange.StartTime, @@ -236,7 +312,7 @@ func compactAndOrderPartitionKeyEntries(ctx context.Context, tx *sql.Tx, pk *par // For the final chunk, make it inclusive to catch the last row isFinalChunk := currentEnd.Equal(maxTime) - rowsInserted, err := insertOrderedDataForTimeRange(ctx, tx, pk, currentStart, currentEnd, isFinalChunk) + rowsInserted, err := insertOrderedDataForTimeRange(ctx, tx, pk, currentStart, currentEnd, isFinalChunk, reindex, columns) if err != nil { return fmt.Errorf("failed to insert ordered data for time range %s to %s: %w", currentStart.Format("2006-01-02 15:04:05"), @@ -250,7 +326,7 @@ func compactAndOrderPartitionKeyEntries(ctx context.Context, tx *sql.Tx, pk *par } // Delete original unordered entries for this time range - err := deleteUnorderedEntriesForTimeRange(ctx, tx, pk, minTime, maxTime) + err := deleteUnorderedEntriesForTimeRange(ctx, tx, rm, minTime, maxTime) if err != nil { return fmt.Errorf("failed to delete unordered entries for time range: %w", err) } @@ -263,33 +339,44 @@ func compactAndOrderPartitionKeyEntries(ctx context.Context, tx *sql.Tx, pk *par } // insertOrderedDataForTimeRange inserts ordered data for a specific time range within a partition key -func insertOrderedDataForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitionKey, startTime, endTime time.Time, isFinalChunk bool) (int64, error) { +func insertOrderedDataForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitionKey, startTime, endTime time.Time, isFinalChunk, reindex bool, columns []string) (int64, error) { + // sanitize table name + tableName, err := backend.SanitizeDuckDBIdentifier(pk.tpTable) + if err != nil { + return 0, err + } + + // Build column list for insert + insertColumns := strings.Join(columns, ", ") + + // Build select fields + selectFields := insertColumns + // For reindexing, replace tp_index with the partition config column + if reindex && pk.partitionConfig != nil { + selectFields = strings.ReplaceAll(selectFields, "tp_index", fmt.Sprintf("%s as tp_index", pk.partitionConfig.TpIndexColumn)) + } // For the final chunk, use inclusive end time to catch the last row timeEndOperator := "<" if isFinalChunk { timeEndOperator = "<=" } - // For overlapping files, we need to reorder ALL rows in the overlapping time range - // Since files overlap, we can't distinguish which specific rows came from which files - // So we reorder all rows in the time range for this partition - args := []interface{}{startTime, endTime, pk.tpPartition, pk.tpIndex} - - tableName, err := backend.SanitizeDuckDBIdentifier(pk.tpTable) - if err != nil { - return 0, err - } //nolint: gosec // sanitized - insertQuery := fmt.Sprintf(`insert into %s - select * from %s + insertQuery := fmt.Sprintf(`insert into %s (%s) + select %s + from %s where tp_timestamp >= ? and tp_timestamp %s ? and tp_partition = ? and tp_index = ? order by tp_timestamp`, tableName, + insertColumns, + selectFields, tableName, timeEndOperator) + // For overlapping files, we need to reorder ALL rows in the overlapping time range + args := []interface{}{startTime, endTime, pk.tpPartition, pk.tpIndex} result, err := tx.ExecContext(ctx, insertQuery, args...) if err != nil { @@ -303,9 +390,9 @@ func insertOrderedDataForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitio } // deleteUnorderedEntriesForTimeRange deletes the original unordered entries for a specific time range within a partition key -func deleteUnorderedEntriesForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitionKey, startTime, endTime time.Time) error { +func deleteUnorderedEntriesForTimeRange(ctx context.Context, tx *sql.Tx, rm *reorderMetadata, startTime, endTime time.Time) error { // Delete all rows in the time range for this partition key (we're re-inserting them in order) - tableName, err := backend.SanitizeDuckDBIdentifier(pk.tpTable) + tableName, err := backend.SanitizeDuckDBIdentifier(rm.pk.tpTable) if err != nil { return err } @@ -318,7 +405,7 @@ func deleteUnorderedEntriesForTimeRange(ctx context.Context, tx *sql.Tx, pk *par and rowid <= ?`, tableName) - args := []interface{}{pk.tpPartition, pk.tpIndex, startTime, endTime, pk.stats.maxRowId} + args := []interface{}{rm.pk.tpPartition, rm.pk.tpIndex, startTime, endTime, rm.maxRowId} _, err = tx.ExecContext(ctx, deleteQuery, args...) if err != nil { diff --git a/internal/parquet/compaction_status.go b/internal/parquet/compaction_status.go index a24d36ac..887f98cb 100644 --- a/internal/parquet/compaction_status.go +++ b/internal/parquet/compaction_status.go @@ -1,16 +1,25 @@ package parquet import ( + "context" "fmt" + "strings" + "time" + "github.com/dustin/go-humanize" + "github.com/turbot/go-kit/types" + "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/utils" - "time" + "github.com/turbot/tailpipe/internal/database" ) type CompactionStatus struct { + Message string InitialFiles int FinalFiles int RowsCompacted int64 + RowsToCompact int64 TotalRows int64 ProgressPercent float64 @@ -94,3 +103,94 @@ func (s *CompactionStatus) RowsCompactedString() any { func (s *CompactionStatus) ProgressPercentString() string { return fmt.Sprintf("%.1f%%", s.ProgressPercent) } + +func (s *CompactionStatus) UpdateProgress() { + // calc percentage from RowsToCompact but print TotalRows in status message + s.ProgressPercent = (float64(s.RowsCompacted) / float64(s.RowsToCompact)) * 100 + s.Message = fmt.Sprintf(" (%0.1f%% of %s rows)", s.ProgressPercent, types.ToHumanisedString(s.TotalRows)) + +} + +func (s *CompactionStatus) getInitialCounts(ctx context.Context, db *database.DuckDb, partitionKeys []*partitionKey) error { + partitionNameMap := make(map[string]map[string]struct{}) + for _, pk := range partitionKeys { + s.InitialFiles += pk.fileCount + if partitionNameMap[pk.tpTable] == nil { + partitionNameMap[pk.tpTable] = make(map[string]struct{}) + } + partitionNameMap[pk.tpTable][pk.tpPartition] = struct{}{} + } + + // get row count for each table + totalRows := int64(0) + for tpTable, tpPartitions := range partitionNameMap { + + // Sanitize partition values for SQL injection protection + sanitizedPartitions := make([]string, 0, len(tpPartitions)) + for partition := range tpPartitions { + sp, err := backend.SanitizeDuckDBIdentifier(partition) + if err != nil { + return fmt.Errorf("failed to sanitize partition %s: %w", partition, err) + } + // Quote the sanitized partition name for the IN clause + sanitizedPartitions = append(sanitizedPartitions, fmt.Sprintf("'%s'", sp)) + } + + tableName, err := backend.SanitizeDuckDBIdentifier(tpTable) + if err != nil { + return fmt.Errorf("failed to sanitize table name %s: %w", tpTable, err) + } + + query := fmt.Sprintf("select count(*) from %s where tp_partition in (%s)", + tableName, + strings.Join(sanitizedPartitions, ", ")) + + var tableRowCount int64 + err = db.QueryRowContext(ctx, query).Scan(&tableRowCount) + if err != nil { + return fmt.Errorf("failed to get row count for table %s: %w", tpTable, err) + } + + totalRows += tableRowCount + } + + s.TotalRows = totalRows + return nil +} + +func (s *CompactionStatus) getFinalFileCounts(ctx context.Context, db *database.DuckDb, partitionKeys []*partitionKey) error { + // Get unique table names from partition keys + tableNames := make(map[string]struct{}) + for _, pk := range partitionKeys { + tableNames[pk.tpTable] = struct{}{} + } + + // Count files for each table from metadata + totalFileCount := 0 + for tableName := range tableNames { + // Sanitize table name + sanitizedTableName, err := backend.SanitizeDuckDBIdentifier(tableName) + if err != nil { + return fmt.Errorf("failed to sanitize table name %s: %w", tableName, err) + } + + // Query to count files for this table from DuckLake metadata + query := fmt.Sprintf(`select count(*) from %s.ducklake_data_file df + join %s.ducklake_table t on df.table_id = t.table_id + where t.table_name = '%s' and df.end_snapshot is null`, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, + sanitizedTableName) + + var tableFileCount int + err = db.QueryRowContext(ctx, query).Scan(&tableFileCount) + if err != nil { + return fmt.Errorf("failed to get file count for table %s: %w", tableName, err) + } + + totalFileCount += tableFileCount + } + + s.FinalFiles = totalFileCount + return nil +} diff --git a/internal/parquet/compaction_types.go b/internal/parquet/compaction_types.go index 4af4cda4..8c25439b 100644 --- a/internal/parquet/compaction_types.go +++ b/internal/parquet/compaction_types.go @@ -9,12 +9,60 @@ import ( "github.com/turbot/tailpipe/internal/database" ) -// getUnorderedRangesForPartitionKey analyzes file fragmentation and creates disorder metrics for a partition key. +// getTimeRangesToReorder analyzes file fragmentation and creates disorder metrics for a partition key. // It queries DuckLake metadata to get all files for the partition, their timestamp ranges, and row counts. // Then it identifies groups of files with overlapping time ranges that need compaction. // Returns metrics including total file count and overlapping file sets with their metadata. -func getUnorderedRangesForPartitionKey(ctx context.Context, db *database.DuckDb, pk *partitionKey) ([]unorderedDataTimeRange, error) { - // Single query to get files and their timestamp ranges and row counts for this partition key +func getTimeRangesToReorder(ctx context.Context, db *database.DuckDb, pk *partitionKey, reindex bool) (*reorderMetadata, error) { + // NOTE: if we are reindexing, we must rewrite the entire partition key + // - return a single range for the entire partition key + if reindex { + rm, err := newReorderMetadata(ctx, db, pk) + if err != nil { + return nil, fmt.Errorf("failed to retiever stats for partition key: %w", err) + } + + // make a single time range + rm.unorderedRanges = []unorderedDataTimeRange{ + { + StartTime: rm.minTimestamp, + EndTime: rm.maxTimestamp, + RowCount: rm.rowCount, + }, + } + + return rm, nil + } + + // first query the metadata to get a list of files, their timestamp ranges and row counts for this partition key + fileRanges, err := getFileRangesForPartitionKey(ctx, db, pk) + if err != nil { + return nil, fmt.Errorf("failed to get file ranges for partition key: %w", err) + } + + // Now identify which of these ranges overlap and for each overlapping set, build a superset time range + unorderedRanges, err := pk.findOverlappingFileRanges(fileRanges) + if err != nil { + return nil, fmt.Errorf("failed to build unordered time ranges: %w", err) + } + + // if there are no unordered ranges, return nil + if len(unorderedRanges) == 0 { + return nil, nil + } + + // get stats for the partition key + rm, err := newReorderMetadata(ctx, db, pk) + if err != nil { + return nil, fmt.Errorf("failed to retiever stats for partition key: %w", err) + } + rm.unorderedRanges = unorderedRanges + return rm, nil + +} + +// query the metadata to get a list of files, their timestamp ranges and row counts for this partition key +func getFileRangesForPartitionKey(ctx context.Context, db *database.DuckDb, pk *partitionKey) ([]fileTimeRange, error) { query := `select df.path, cast(fcs.min_value as timestamp) as min_timestamp, @@ -75,13 +123,7 @@ func getUnorderedRangesForPartitionKey(ctx context.Context, db *database.DuckDb, rangesStr.WriteString(", ") } } - // Build unordered time ranges - unorderedRanges, err := pk.findOverlappingFileRanges(fileRanges) - if err != nil { - return nil, fmt.Errorf("failed to build unordered time ranges: %w", err) - } - - return unorderedRanges, nil + return fileRanges, nil } type fileTimeRange struct { diff --git a/internal/parquet/ducklake.go b/internal/parquet/ducklake.go index f03ffd0f..c7b371f9 100644 --- a/internal/parquet/ducklake.go +++ b/internal/parquet/ducklake.go @@ -13,6 +13,7 @@ import ( "github.com/turbot/tailpipe/internal/database" ) +// DeletePartition deletes data for the specified partition and date range from the given Ducklake connected database. func DeletePartition(ctx context.Context, partition *config.Partition, from, to time.Time, db *database.DuckDb) (rowCount int, err error) { // TODO #DL https://github.com/turbot/tailpipe/issues/505 // if we are using s3 do not delete for now as this does not work at present (need explicit S3 support I think) diff --git a/internal/parquet/migrate_tpindex.go b/internal/parquet/migrate_tpindex.go deleted file mode 100644 index 9e37485d..00000000 --- a/internal/parquet/migrate_tpindex.go +++ /dev/null @@ -1,178 +0,0 @@ -package parquet - -import ( - "context" - "github.com/turbot/tailpipe/internal/database" -) - -const ( -// sourceFileColumnName = "__duckdb_source_file" -// migrateTempTableName = "_raw_tp_data" -) - -//nolint:unused // TODO re-add tp_index migration for DuckDB https://github.com/turbot/tailpipe/issues/475 -func migrateTpIndex(ctx context.Context, db *database.DuckDb, baseDir string, updateFunc func(CompactionStatus), patterns []PartitionPattern) error { - //fileRootProvider := &FileRootProvider{} - //for _, partition := range config.GlobalConfig.Partitions { - // if PartitionMatchesPatterns(partition.TableName, partition.ShortName, patterns) { - // err := migrateTpIndexForPartition(ctx, db, baseDir, partition, fileRootProvider, updateFunc) - // if err != nil { - // if ctx.Err() != nil { - // return err - // } - // return fmt.Errorf("failed to migrate tp_index for partition %s: %w", partition.UnqualifiedName, err) - // } else { - // slog.Info("Migrated tp_index files for partition", "partition", partition.UnqualifiedName, "index_expression", partition.TpIndexColumn) - // } - // } - //} - return nil -} - -// -//func migrateTpIndexForPartition(ctx context.Context, db *database.DuckDb, baseDir string, partition *config.Partition, fileRootProvider *FileRootProvider, updateFunc func(CompactionStatus)) error { -// -// // executeMigrationQuery runs the DuckDB query to migrate the tp_index files for a given partition. -// // it read the partition data into a temporary table, then writes the data to with the migrated tp_index -// // to intermediate the output files (with extension .tmp) and returns the list of output files. -// outputFiles, err := executeMigrationQuery(ctx, db, baseDir, partition, fileRootProvider) -// if err != nil { -// return err -// } -// if len(outputFiles) == 0 { -// return nil // nothing to migrate -// } -// -// // read the source files from the temporary table -// sourceFiles, err := readSourceFiles(ctx, db) -// if err != nil { -// return err -// } -// -// // now rename the source files to add a .migrated extension -// renamedSourceFiles, err := addExtensionToFiles(sourceFiles, ".migrated") -// if err != nil { -// if err := deleteFilesConcurrently(ctx, outputFiles, baseDir); err != nil { -// slog.Error("Failed to delete temp files after migration failure", "error", err) -// } -// return err -// } -// -// // rename the output files to remove the .tmp extension -// if err := removeExtensionFromFiles(outputFiles, ".tmp"); err != nil { -// if err := deleteFilesConcurrently(ctx, outputFiles, baseDir); err != nil { -// slog.Error("Failed to delete temp files after migration failure", "error", err) -// } -// -// if err := removeExtensionFromFiles(renamedSourceFiles, ".migrated"); err != nil { -// slog.Error("Failed to rename source files back to original names after migration failure", "error", err) -// } -// return err -// } -// -// // finally, delete the renamed source parquet files -// if err := deleteFilesConcurrently(ctx, renamedSourceFiles, baseDir); err != nil { -// slog.Error("Failed to delete renamed source parquet files after migration", "error", err) -// } -// -// status := CompactionStatus{ -// MigrateSource: len(sourceFiles), -// MigrateDest: len(outputFiles), -// PartitionIndexExpressions: map[string]string{ -// partition.UnqualifiedName: partition.TpIndexColumn, -// }, -// } -// updateFunc(status) -// -// return nil -//} -// -//// executeMigrationQuery runs the DuckDB query to migrate the tp_index files for a given partition. -//// It reads the partition data into a temporary table, writes the data with the migrated tp_index -//// to intermediate output files (with .tmp extension), and returns the list of output file paths. -//func executeMigrationQuery(ctx context.Context, db *database.DuckDb, baseDir string, partition *config.Partition, fileRootProvider *FileRootProvider) ([]string, error) { -// // Get the file glob pattern for all files in this partition -// fileGlob := "" //filepaths.GetParquetFileGlobForPartition(baseDir, partition.TableName, partition.ShortName, "") -// -// // get unique file root to use for the output files -// fileRoot := fileRootProvider.GetFileRoot() -// // columns to partition by -// partitionColumns := []string{sdkconstants.TpTable, sdkconstants.TpPartition, sdkconstants.TpIndex, sdkconstants.TpDate} -// -// // build the query to read the parquet files into a temporary table -// query := fmt.Sprintf(` -//create or replace temp table %s as -//select -// *, -// %s, -//from read_parquet('%s', filename=%s); -// -//copy ( -// select -// * exclude (tp_index, %s), -// %s as tp_index -// from %s -//) to '%s' ( -// format parquet, -// partition_by (%s), -// return_files true, -// overwrite_or_ignore, -// filename_pattern '%s_{i}', -// file_extension 'parquet.tmp' -//); -//`, -// migrateTempTableName, // e.g. "_raw_tp_data" -// sourceFileColumnName, // select filename -// fileGlob, // parquet file glob path -// sourceFileColumnName, // read filename column from parquet -// sourceFileColumnName, // exclude source file column from the copy -// partition.TpIndexColumn, // replacement tp_index expression -// migrateTempTableName, // again used in the copy -// baseDir, // output path -// strings.Join(partitionColumns, ","), // partition columns -// fileRoot, // filename root prefix -// ) -// -// var rowCount int64 -// var outputFilesRaw []interface{} -// err := db.QueryRowContext(ctx, query).Scan(&rowCount, &outputFilesRaw) -// if err != nil { -// // if this is a no files found error, we can ignore it -// if strings.Contains(err.Error(), "No files found") { -// slog.Info("No files found for migration", "partition", partition.UnqualifiedName) -// return nil, nil -// } -// return nil, fmt.Errorf("failed to scan return_files output: %w", err) -// } -// -// outputFiles := make([]string, len(outputFilesRaw)) -// for i, val := range outputFilesRaw { -// if str, ok := val.(string); ok { -// outputFiles[i] = str -// } else { -// return nil, fmt.Errorf("unexpected file path type %T at index %d", val, i) -// } -// } -// -// return outputFiles, nil -//} -// -//// readSourceFiles reads the source files column from the temporary table created during the tp_index migration. -//func readSourceFiles(ctx context.Context, db *database.DuckDb) ([]string, error) { -// query := fmt.Sprintf(`select distinct %s from %s`, sourceFileColumnName, migrateTempTableName) -// rows, err := db.QueryContext(ctx, query) -// if err != nil { -// return nil, fmt.Errorf("failed to read source files from temp table: %w", err) -// } -// defer rows.Close() -// -// var sourceFiles []string -// for rows.Next() { -// var path string -// if err := rows.Scan(&path); err != nil { -// return nil, fmt.Errorf("failed to scan source file path: %w", err) -// } -// sourceFiles = append(sourceFiles, path) -// } -// return sourceFiles, nil -//} diff --git a/internal/parquet/partition_key.go b/internal/parquet/partition_key.go index 9b0595bb..0e742c87 100644 --- a/internal/parquet/partition_key.go +++ b/internal/parquet/partition_key.go @@ -3,44 +3,27 @@ package parquet import ( "context" "fmt" - "log/slog" - "sort" - "strings" - "time" - - "github.com/turbot/pipe-fittings/v2/constants" - + "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" + "sort" ) // partitionKey is used to uniquely identify a a combination of ducklake partition columns: // tp_table, tp_partition, tp_index, year(tp_timestamp), month(tp_timestamp) // It also stores the file and row stats for that partition key type partitionKey struct { - tpTable string - tpPartition string - tpIndex string - year string // year(tp_timestamp) from partition value - month string // month(tp_timestamp) from partition value - fileCount int // number of files for this partition key - stats partitionKeyStats -} - -// getStats retrieves and populates partition key statistics including row count, max row id, and timestamp range. -// It queries the database to get comprehensive statistics for this partition key and stores them in the partitionKey struct. -func (p *partitionKey) getStats(ctx context.Context, db *database.DuckDb) error { - stats, err := newPartitionKeyStats(ctx, db, p) - if err != nil { - return err - } - - p.stats = *stats - return nil + tpTable string + tpPartition string + tpIndex string + year string // year(tp_timestamp) from partition value + month string // month(tp_timestamp) from partition value + fileCount int // number of files for this partition key + partitionConfig *config.Partition } // query the ducklake_data_file table to get all partition keys combinations which satisfy the provided patterns, // along with the file and row stats for each partition key combination -func getPartitionKeysMatchingPattern(ctx context.Context, db *database.DuckDb, patterns []PartitionPattern) ([]*partitionKey, error) { +func getPartitionKeysMatchingPattern(ctx context.Context, db *database.DuckDb, patterns []*PartitionPattern) ([]*partitionKey, error) { // This query joins the DuckLake metadata tables to get partition key combinations: // - ducklake_data_file: contains file metadata and links to tables // - ducklake_file_partition_value: contains partition values for each file @@ -89,96 +72,25 @@ group by var partitionKeys []*partitionKey for rows.Next() { - var partitionKey = &partitionKey{} + var pk = &partitionKey{} - if err := rows.Scan(&partitionKey.tpTable, &partitionKey.tpPartition, &partitionKey.tpIndex, &partitionKey.year, &partitionKey.month, &partitionKey.fileCount); err != nil { + if err := rows.Scan(&pk.tpTable, &pk.tpPartition, &pk.tpIndex, &pk.year, &pk.month, &pk.fileCount); err != nil { return nil, fmt.Errorf("failed to scan partition key row: %w", err) } - // check whether this partition key matches any of the provided patterns and whether there are any files - if partitionKey.fileCount > 0 && PartitionMatchesPatterns(partitionKey.tpTable, partitionKey.tpPartition, patterns) { - partitionKeys = append(partitionKeys, partitionKey) - } - } - // now get the stats for each partition key - for _, pk := range partitionKeys { - // populate the stats for the key - if err := pk.getStats(ctx, db); err != nil { - return nil, fmt.Errorf("failed to get stats for partition key %v: %w", pk, err) + // retrieve the partition config for this key (which may not exist - that is ok + partitionConfig, ok := config.GlobalConfig.Partitions[pk.partitionName()] + if ok { + pk.partitionConfig = partitionConfig } - } - - return partitionKeys, nil -} - -// getFileCountForPartitionKeys returns the count of parquet files for the provided partition keys -func getFileCountForPartitionKeys(ctx context.Context, db *database.DuckDb, partitionLKeys []*partitionKey) (int, error) { - slog.Info("Getting DuckLake parquet file count for specific partition keys") - - if len(partitionLKeys) == 0 { - return 0, nil - } - - // Build a query to count files only for the specified partition keys - query := fmt.Sprintf(`select count(*) from %s.ducklake_data_file df - join %s.ducklake_file_partition_value fpv1 on df.data_file_id = fpv1.data_file_id and fpv1.partition_key_index = 0 - join %s.ducklake_file_partition_value fpv2 on df.data_file_id = fpv2.data_file_id and fpv2.partition_key_index = 1 - join %s.ducklake_file_partition_value fpv3 on df.data_file_id = fpv3.data_file_id and fpv3.partition_key_index = 2 - join %s.ducklake_file_partition_value fpv4 on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 - where df.end_snapshot is null - and (fpv1.partition_value, fpv2.partition_value, fpv3.partition_value, fpv4.partition_value) in (`, - constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog, - constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) - - // Build the IN clause with all partition key combinations - var values []string - for _, pk := range partitionLKeys { - value := fmt.Sprintf("('%s', '%s', '%s', '%s')", pk.tpPartition, pk.tpIndex, pk.year, pk.month) - values = append(values, value) - } - - query += strings.Join(values, ", ") + ")" - var count int - err := db.QueryRowContext(ctx, query).Scan(&count) - if err != nil { - if ctx.Err() != nil { - return 0, err + // check whether this partition key matches any of the provided patterns and whether there are any files + if pk.fileCount > 0 && PartitionMatchesPatterns(pk.tpTable, pk.tpPartition, patterns) { + partitionKeys = append(partitionKeys, pk) } - return 0, fmt.Errorf("failed to get parquet file count for partition keys: %w", err) - } - slog.Info("DuckLake parquet file count retrieved for partition keys", "count", count, "partition_keys", len(partitionLKeys)) - return count, nil -} - -type partitionKeyStats struct { - rowCount int64 - maxRowId int64 - minTimestamp time.Time - maxTimestamp time.Time -} - -func newPartitionKeyStats(ctx context.Context, db *database.DuckDb, p *partitionKey) (*partitionKeyStats, error) { - var stats = &partitionKeyStats{} - - // Query to get row count and time range for this partition - countQuery := fmt.Sprintf(`select count(*), max(rowid), min(tp_timestamp), max(tp_timestamp) from "%s" - where tp_partition = ? - and tp_index = ? - and year(tp_timestamp) = ? - and month(tp_timestamp) = ?`, - p.tpTable) - - err := db.QueryRowContext(ctx, countQuery, - p.tpPartition, - p.tpIndex, - p.year, - p.month).Scan(&stats.rowCount, &stats.maxRowId, &stats.minTimestamp, &stats.maxTimestamp) - if err != nil { - return nil, fmt.Errorf("failed to get row count and time range for partition: %w", err) } - return stats, nil + return partitionKeys, nil } // findOverlappingFileRanges finds sets of files that have overlapping time ranges and converts them to unorderedDataTimeRange @@ -257,3 +169,8 @@ func (p *partitionKey) fileOverlapsWithSet(candidateFile fileTimeRange, fileSet } return false } + +// return fully qualified partition name (table.partition) +func (p *partitionKey) partitionName() string { + return fmt.Sprintf("%s.%s", p.tpTable, p.tpPartition) +} diff --git a/internal/parquet/partition_pattern.go b/internal/parquet/partition_pattern.go index 6f839339..aa7d6de1 100644 --- a/internal/parquet/partition_pattern.go +++ b/internal/parquet/partition_pattern.go @@ -1,8 +1,12 @@ package parquet import ( + "fmt" + "strings" + "github.com/danwakefield/fnmatch" "github.com/turbot/tailpipe/internal/config" + "golang.org/x/exp/maps" ) // PartitionPattern represents a pattern used to match partitions. @@ -20,7 +24,8 @@ func NewPartitionPattern(partition *config.Partition) PartitionPattern { } } -func PartitionMatchesPatterns(table, partition string, patterns []PartitionPattern) bool { +// PartitionMatchesPatterns checks if the given table and partition match any of the provided patterns. +func PartitionMatchesPatterns(table, partition string, patterns []*PartitionPattern) bool { if len(patterns) == 0 { return true } @@ -34,3 +39,85 @@ func PartitionMatchesPatterns(table, partition string, patterns []PartitionPatte } return gotMatch } + +// GetPartitionsForArg returns the actual partition names that match the given argument. +// The partitionNames list is needed to determine whether a single-part argument refers to a table or partition. +func GetPartitionsForArg(partitionMap map[string]*config.Partition, arg string) ([]string, error) { + partitionNames := maps.Keys(partitionMap) + partitionPattern, err := GetPartitionPatternsForArgs(partitionNames, arg) + if err != nil { + return nil, err + } + // now match the partition + var res []string + for _, partition := range partitionMap { + if PartitionMatchesPatterns(partition.TableName, partition.ShortName, partitionPattern) { + res = append(res, partition.UnqualifiedName) + } + } + return res, nil +} + +// GetPartitionPatternsForArgs returns the table and partition patterns for the given partition args. +// The partitions list is needed to determine whether single-part arguments refer to tables or partitions. +func GetPartitionPatternsForArgs(partitions []string, partitionArgs ...string) ([]*PartitionPattern, error) { + var res []*PartitionPattern + for _, arg := range partitionArgs { + partitionPattern, err := GetPartitionMatchPatternsForArg(partitions, arg) + if err != nil { + return nil, fmt.Errorf("error processing partition arg '%s': %w", arg, err) + } + + res = append(res, partitionPattern) + } + + return res, nil +} + +// GetPartitionMatchPatternsForArg parses a single partition argument into a PartitionPattern. +// The partitions list is needed to determine whether single-part arguments refer to tables or partitions. +func GetPartitionMatchPatternsForArg(partitions []string, arg string) (*PartitionPattern, error) { + var partitionPattern *PartitionPattern + parts := strings.Split(arg, ".") + switch len(parts) { + case 1: + var err error + partitionPattern, err = getPartitionPatternsForSinglePartName(partitions, arg) + if err != nil { + return nil, err + } + case 2: + // use the args as provided + partitionPattern = &PartitionPattern{Table: parts[0], Partition: parts[1]} + default: + return nil, fmt.Errorf("invalid partition name: %s", arg) + } + return partitionPattern, nil +} + +// getPartitionPatternsForSinglePartName determines whether a single-part argument refers to a table or partition. +// The partitions list is needed to check if the argument matches any existing table names. +// e.g. if the arg is "aws*" and it matches table "aws_cloudtrail_log", it's treated as a table pattern. +func getPartitionPatternsForSinglePartName(partitions []string, arg string) (*PartitionPattern, error) { + var tablePattern, partitionPattern string + // '*' is not valid for a single part arg + if arg == "*" { + return nil, fmt.Errorf("invalid partition name: %s", arg) + } + // check whether there is table with this name + // partitions is a list of Unqualified names, i.e.
. + for _, partition := range partitions { + table := strings.Split(partition, ".")[0] + + // if the arg matches a table name, set table pattern to the arg and partition pattern to * + if fnmatch.Match(arg, table, fnmatch.FNM_CASEFOLD) { + tablePattern = arg + partitionPattern = "*" + return &PartitionPattern{Table: tablePattern, Partition: partitionPattern}, nil + } + } + // so there IS NOT a table with this name - set table pattern to * and user provided partition name + tablePattern = "*" + partitionPattern = arg + return &PartitionPattern{Table: tablePattern, Partition: partitionPattern}, nil +} diff --git a/internal/parquet/partition_pattern_test.go b/internal/parquet/partition_pattern_test.go new file mode 100644 index 00000000..56d0d2d1 --- /dev/null +++ b/internal/parquet/partition_pattern_test.go @@ -0,0 +1,296 @@ +package parquet + +import ( + "github.com/turbot/pipe-fittings/v2/modconfig" + "github.com/turbot/tailpipe/internal/config" + "reflect" + "sort" + "strings" + "testing" +) + +func Test_getPartition(t *testing.T) { + type args struct { + partitions []string + name string + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "Invalid partition name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "*", + }, + wantErr: true, + }, + { + name: "Full partition name, exists", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.p1", + }, + want: []string{"aws_s3_cloudtrail_log.p1"}, + }, + { + name: "Full partition name, does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.p3", + }, + want: nil, + }, + { + name: "Table name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + }, + { + name: "Table name (exists) with wildcard", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.*", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + }, + { + name: "Table name (exists) with ?", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.p?", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + }, + { + name: "Table name (exists) with non matching partition wildacard", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.d*?", + }, + want: nil, + }, + { + name: "Table name (does not exist)) with wildcard", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "foo.*", + }, + want: nil, + }, + { + name: "Partition short name, exists", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "p1", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + }, + { + name: "Table wildcard, partition short name, exists", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "*.p1", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + }, + { + name: "Partition short name, does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "p3", + }, + want: nil, + }, + { + name: "Table wildcard, partition short name, does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "*.p3", + }, + want: nil, + }, + { + name: "Table wildcard, no dot", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "aws*", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var partitions = getPartitions(tt.args.partitions) + + got, err := GetPartitionsForArg(partitions, tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("getPartitions() error = %v, wantErr %v", err, tt.wantErr) + return + } + // sort the slices before comparing + sort.Strings(tt.want) + sort.Strings(got) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getPartitions() got = %v, want %v", got, tt.want) + } + }) + } +} + +func getPartitions(partitions []string) map[string]*config.Partition { + var partitionMap = make(map[string]*config.Partition) + for _, p := range partitions { + parts := strings.SplitN(p, ".", 2) + if len(parts) != 2 { + continue + } + partitionMap[p] = &config.Partition{ + HclResourceImpl: modconfig.HclResourceImpl{ + UnqualifiedName: p, + ShortName: parts[1], + }, + TableName: parts[0], + } + } + return partitionMap +} + +func Test_getPartitionMatchPatternsForArg(t *testing.T) { + type args struct { + partitions []string + arg string + } + tests := []struct { + name string + args args + wantTablePattern string + wantPartPattern string + wantErr bool + }{ + { + name: "Valid table and partition pattern", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + arg: "aws_s3_cloudtrail_log.p1", + }, + wantTablePattern: "aws_s3_cloudtrail_log", + wantPartPattern: "p1", + }, + { + name: "Wildcard partition pattern", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, + arg: "aws_s3_cloudtrail_log.*", + }, + wantTablePattern: "aws_s3_cloudtrail_log", + wantPartPattern: "*", + }, + { + name: "Wildcard in table and partition both", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, + arg: "aws*.*", + }, + wantTablePattern: "aws*", + wantPartPattern: "*", + }, + { + name: "Wildcard table pattern", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + arg: "*.p1", + }, + wantTablePattern: "*", + wantPartPattern: "p1", + }, + { + name: "Invalid partition name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + arg: "*", + }, + wantErr: true, + }, + { + name: "Table exists without partition", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + arg: "aws_s3_cloudtrail_log", + }, + wantTablePattern: "aws_s3_cloudtrail_log", + wantPartPattern: "*", + }, + { + name: "Partition only, multiple tables", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + arg: "p1", + }, + wantTablePattern: "*", + wantPartPattern: "p1", + }, + { + name: "Invalid argument with multiple dots", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1"}, + arg: "aws.s3.cloudtrail", + }, + wantErr: true, + }, + { + name: "Non-existing table name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1"}, + arg: "non_existing_table.p1", + }, + wantTablePattern: "non_existing_table", + wantPartPattern: "p1", + }, + { + name: "Partition name does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1"}, + arg: "p2", + }, + wantTablePattern: "*", + wantPartPattern: "p2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + partitionPattern, err := GetPartitionMatchPatternsForArg(tt.args.partitions, tt.args.arg) + + if err != nil { + if !tt.wantErr { + t.Errorf("GetPartitionMatchPatternsForArg() error = %v, wantErr %v", err, tt.wantErr) + } + return + } else if tt.wantErr { + t.Errorf("GetPartitionMatchPatternsForArg() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + // must be a wanted err + return + } + + gotTablePattern := partitionPattern.Table + gotPartPattern := partitionPattern.Partition + if gotTablePattern != tt.wantTablePattern { + t.Errorf("GetPartitionMatchPatternsForArg() gotTablePattern = %v, want %v", gotTablePattern, tt.wantTablePattern) + } + if gotPartPattern != tt.wantPartPattern { + t.Errorf("GetPartitionMatchPatternsForArg() gotPartPattern = %v, want %v", gotPartPattern, tt.wantPartPattern) + } + }) + } +} diff --git a/internal/parquet/reorder_metadata.go b/internal/parquet/reorder_metadata.go new file mode 100644 index 00000000..b3316b19 --- /dev/null +++ b/internal/parquet/reorder_metadata.go @@ -0,0 +1,41 @@ +package parquet + +import ( + "context" + "fmt" + "github.com/turbot/tailpipe/internal/database" + "time" +) + +type reorderMetadata struct { + pk *partitionKey + unorderedRanges []unorderedDataTimeRange + + rowCount int64 + maxRowId int64 + minTimestamp time.Time + maxTimestamp time.Time +} + +func newReorderMetadata(ctx context.Context, db *database.DuckDb, p *partitionKey) (*reorderMetadata, error) { + var rm = &reorderMetadata{pk: p} + + // Query to get row count and time range for this partition + countQuery := fmt.Sprintf(`select count(*), max(rowid), min(tp_timestamp), max(tp_timestamp) from "%s" + where tp_partition = ? + and tp_index = ? + and year(tp_timestamp) = ? + and month(tp_timestamp) = ?`, + p.tpTable) + + err := db.QueryRowContext(ctx, countQuery, + p.tpPartition, + p.tpIndex, + p.year, + p.month).Scan(&rm.rowCount, &rm.maxRowId, &rm.minTimestamp, &rm.maxTimestamp) + if err != nil { + return nil, fmt.Errorf("failed to get row count and time range for partition: %w", err) + } + + return rm, nil +} diff --git a/internal/parse/decode.go b/internal/parse/decode.go index 5a2686e4..62c68cbe 100644 --- a/internal/parse/decode.go +++ b/internal/parse/decode.go @@ -131,7 +131,7 @@ func decodePartition(block *hcl.Block, parseCtx *ConfigParseContext, resource mo for _, attr := range attrs { switch attr.Name { case "filter": - //try to evaluate expression + // try to evaluate expression val, diags := attr.Expr.Value(parseCtx.EvalCtx) res.HandleDecodeDiags(diags) // we failed, possibly as result of dependency error - give up for now @@ -140,7 +140,7 @@ func decodePartition(block *hcl.Block, parseCtx *ConfigParseContext, resource mo } target.Filter = val.AsString() case "tp_index": - //try to evaluate expression + // try to evaluate expression val, diags := attr.Expr.Value(parseCtx.EvalCtx) res.HandleDecodeDiags(diags) // we failed, possibly as result of dependency error - give up for now diff --git a/internal/query/execute_test.go b/internal/query/execute_test.go index e4b172c7..3794a5e6 100644 --- a/internal/query/execute_test.go +++ b/internal/query/execute_test.go @@ -2,6 +2,7 @@ package query import ( "fmt" + "github.com/turbot/pipe-fittings/v2/filepaths" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,8 @@ import ( ) func TestGetColumnDefsForQuery(t *testing.T) { + filepaths.PipesInstallDir = "." + // Create a temporary DuckDB instance for testing db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) require.NoError(t, err) From 53c826631e2d0dc58e8640ecc523ea00d4b59349 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 9 Sep 2025 10:55:10 +0100 Subject: [PATCH 29/61] reorganise packages - move files from `parquet` to `database` remove test 0 timeout val for ducklake database --- cmd/collect.go | 4 +- cmd/compact.go | 11 +- cmd/connect.go | 9 +- cmd/partition.go | 3 +- internal/collector/collector.go | 13 +- internal/collector/status.go | 4 +- internal/collector/tui.go | 5 +- .../ducklake.go => database/cleanup.go} | 11 +- internal/{parquet => database}/compact.go | 9 +- .../compaction_status.go | 7 +- .../{parquet => database}/compaction_types.go | 8 +- .../{parquet => database}/conversion_error.go | 2 +- .../conversion_error_test.go | 2 +- internal/{parquet => database}/convertor.go | 7 +- .../convertor_convert.go | 2 +- .../convertor_ducklake.go | 2 +- .../{parquet => database}/convertor_schema.go | 5 +- .../convertor_validate.go | 2 +- internal/database/duck_db.go | 2 - .../{parquet => database}/file_metadata.go | 9 +- .../{parquet => database}/partition_key.go | 5 +- .../partition_key_test.go | 2 +- .../partition_pattern.go | 2 +- .../partition_pattern_test.go | 2 +- .../{parquet => database}/read_json_query.go | 2 +- .../read_json_query_test.go | 2 +- .../{parquet => database}/reorder_metadata.go | 5 +- .../row_validation_error.go | 2 +- .../schema_change_error.go | 2 +- .../schema_comparison.go | 2 +- internal/display/partition.go | 3 +- internal/display/table.go | 3 +- internal/parquet/file_helpers.go | 143 ---- internal/parquet/file_helpers_test.go | 656 ------------------ 34 files changed, 65 insertions(+), 883 deletions(-) rename internal/{parquet/ducklake.go => database/cleanup.go} (93%) rename internal/{parquet => database}/compact.go (96%) rename internal/{parquet => database}/compaction_status.go (97%) rename internal/{parquet => database}/compaction_types.go (94%) rename internal/{parquet => database}/conversion_error.go (99%) rename internal/{parquet => database}/conversion_error_test.go (97%) rename internal/{parquet => database}/convertor.go (98%) rename internal/{parquet => database}/convertor_convert.go (99%) rename internal/{parquet => database}/convertor_ducklake.go (99%) rename internal/{parquet => database}/convertor_schema.go (96%) rename internal/{parquet => database}/convertor_validate.go (99%) rename internal/{parquet => database}/file_metadata.go (92%) rename internal/{parquet => database}/partition_key.go (96%) rename internal/{parquet => database}/partition_key_test.go (99%) rename internal/{parquet => database}/partition_pattern.go (99%) rename internal/{parquet => database}/partition_pattern_test.go (99%) rename internal/{parquet => database}/read_json_query.go (99%) rename internal/{parquet => database}/read_json_query_test.go (99%) rename internal/{parquet => database}/reorder_metadata.go (82%) rename internal/{parquet => database}/row_validation_error.go (97%) rename internal/{parquet => database}/schema_change_error.go (97%) rename internal/{parquet => database}/schema_comparison.go (99%) delete mode 100644 internal/parquet/file_helpers.go delete mode 100644 internal/parquet/file_helpers_test.go diff --git a/cmd/collect.go b/cmd/collect.go index 28350d60..13d7eaac 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/turbot/tailpipe/internal/database" "log/slog" "os" "strconv" @@ -24,7 +25,6 @@ import ( "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" ) @@ -215,7 +215,7 @@ func getPartitions(args []string) ([]*config.Partition, error) { continue } - partitionNames, err := parquet.GetPartitionsForArg(tailpipeConfig.Partitions, arg) + partitionNames, err := database.GetPartitionsForArg(tailpipeConfig.Partitions, arg) if err != nil { errorList = append(errorList, err) } else if len(partitionNames) == 0 { diff --git a/cmd/compact.go b/cmd/compact.go index 550854e8..8a027720 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -20,7 +20,6 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/parquet" "golang.org/x/exp/maps" ) @@ -80,7 +79,7 @@ func runCompactCmd(cmd *cobra.Command, args []string) { } // Get table and partition patterns - patterns, err := parquet.GetPartitionPatternsForArgs(maps.Keys(config.GlobalConfig.Partitions), args...) + patterns, err := database.GetPartitionPatternsForArgs(maps.Keys(config.GlobalConfig.Partitions), args...) error_helpers.FailOnErrorWithMessage(err, "failed to get partition patterns") // do the compaction @@ -95,7 +94,7 @@ func runCompactCmd(cmd *cobra.Command, args []string) { // defer block will show the error } -func doCompaction(ctx context.Context, db *database.DuckDb, patterns []*parquet.PartitionPattern) (*parquet.CompactionStatus, error) { +func doCompaction(ctx context.Context, db *database.DuckDb, patterns []*database.PartitionPattern) (*database.CompactionStatus, error) { s := spinner.New( spinner.CharSets[14], 100*time.Millisecond, @@ -110,9 +109,9 @@ func doCompaction(ctx context.Context, db *database.DuckDb, patterns []*parquet. defer s.Stop() s.Suffix = " compacting parquet files" // define func to update the spinner suffix with the number of files compacted - var status = parquet.NewCompactionStatus() + var status = database.NewCompactionStatus() - updateTotals := func(updatedStatus parquet.CompactionStatus) { + updateTotals := func(updatedStatus database.CompactionStatus) { status = &updatedStatus if status.Message != "" { s.Suffix = " compacting parquet files: " + status.Message @@ -120,7 +119,7 @@ func doCompaction(ctx context.Context, db *database.DuckDb, patterns []*parquet. } // do compaction - err := parquet.CompactDataFiles(ctx, db, updateTotals, reindex, patterns...) + err := database.CompactDataFiles(ctx, db, updateTotals, reindex, patterns...) return status, err } diff --git a/cmd/connect.go b/cmd/connect.go index f56d0ccc..c0e8d444 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -25,7 +25,6 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/parquet" ) // variable used to assign the output mode flag @@ -306,7 +305,7 @@ func getFilters() ([]string, error) { // getPartitionSqlFilters builds SQL filters for the provided partition args func getPartitionSqlFilters(partitionArgs []string, availablePartitions []string) (string, error) { // Get table and partition patterns using GetPartitionPatternsForArgs - patterns, err := parquet.GetPartitionPatternsForArgs(availablePartitions, partitionArgs...) + patterns, err := database.GetPartitionPatternsForArgs(availablePartitions, partitionArgs...) if err != nil { return "", fmt.Errorf("error processing partition args: %w", err) } @@ -392,11 +391,11 @@ func getIndexSqlFilters(indexArgs []string) (string, error) { } // convert partition patterns with '*' wildcards to SQL '%' wildcards -func replaceWildcards(patterns []*parquet.PartitionPattern) []*parquet.PartitionPattern { - updatedPatterns := make([]*parquet.PartitionPattern, len(patterns)) +func replaceWildcards(patterns []*database.PartitionPattern) []*database.PartitionPattern { + updatedPatterns := make([]*database.PartitionPattern, len(patterns)) for i, p := range patterns { - updatedPatterns[i] = &parquet.PartitionPattern{ + updatedPatterns[i] = &database.PartitionPattern{ Table: strings.ReplaceAll(p.Table, "*", "%"), Partition: strings.ReplaceAll(p.Partition, "*", "%")} } diff --git a/cmd/partition.go b/cmd/partition.go index c3b7a82b..2a25ee2c 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -25,7 +25,6 @@ import ( "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" ) @@ -278,7 +277,7 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { spinner := statushooks.NewStatusSpinnerHook() spinner.SetStatus(fmt.Sprintf("Deleting partition %s", partition.TableName)) spinner.Show() - rowsDeleted, err := parquet.DeletePartition(ctx, partition, fromTime, toTime, db) + rowsDeleted, err := database.DeletePartition(ctx, partition, fromTime, toTime, db) spinner.Hide() error_helpers.FailOnError(err) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index d6c696bc..6730099f 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -20,7 +20,6 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" ) @@ -39,7 +38,7 @@ type Collector struct { // the execution is used to manage the state of the collection execution *execution // the parquet convertor is used to convert the JSONL files to parquet - parquetConvertor *parquet.Converter + parquetConvertor *database.Converter // the current plugin status - used to update the spinner status status @@ -193,7 +192,7 @@ func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, ove c.addTimeRangeFilters(resolvedFromTime, toTime) // create a parquet writer - parquetConvertor, err := parquet.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount, c.db) + parquetConvertor, err := database.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount, c.db) if err != nil { return fmt.Errorf("failed to create parquet writer: %w", err) } @@ -261,18 +260,18 @@ func (c *Collector) Compact(ctx context.Context) error { c.updateApp(AwaitingCompactionMsg{}) - updateAppCompactionFunc := func(status parquet.CompactionStatus) { + updateAppCompactionFunc := func(status database.CompactionStatus) { c.statusLock.Lock() defer c.statusLock.Unlock() c.status.compactionStatus = &status c.updateApp(CollectionStatusUpdateMsg{status: c.status}) } - partitionPattern := parquet.NewPartitionPattern(c.partition) + partitionPattern := database.NewPartitionPattern(c.partition) // NOTE: we DO NOT reindex when compacting after collection reindex := false - err := parquet.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, reindex, &partitionPattern) + err := database.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, reindex, &partitionPattern) if err != nil { return fmt.Errorf("failed to compact data files: %w", err) @@ -296,7 +295,7 @@ func (c *Collector) Completed() { // deletePartitionData deletes all parquet files in the partition between the fromTime and toTime func (c *Collector) deletePartitionData(ctx context.Context, fromTime, toTime time.Time) error { slog.Info("Deleting parquet files after the from time", "partition", c.partition.Name, "from", fromTime) - _, err := parquet.DeletePartition(ctx, c.partition, fromTime, toTime, c.db) + _, err := database.DeletePartition(ctx, c.partition, fromTime, toTime, c.db) if err != nil { slog.Warn("Failed to delete parquet files after the from time", "partition", c.partition.Name, "from", fromTime, "error", err) diff --git a/internal/collector/status.go b/internal/collector/status.go index 25a0f33e..e1601db9 100644 --- a/internal/collector/status.go +++ b/internal/collector/status.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "github.com/turbot/tailpipe/internal/database" "path/filepath" "strings" "time" @@ -11,7 +12,6 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/events" "github.com/turbot/tailpipe-plugin-sdk/logging" "github.com/turbot/tailpipe-plugin-sdk/row_source" - "github.com/turbot/tailpipe/internal/parquet" ) const uiErrorsToDisplay = 15 @@ -28,7 +28,7 @@ type status struct { complete bool partitionName string fromTime *row_source.ResolvedFromTime - compactionStatus *parquet.CompactionStatus + compactionStatus *database.CompactionStatus toTime time.Time } diff --git a/internal/collector/tui.go b/internal/collector/tui.go index 7ff1f31f..e3e5b785 100644 --- a/internal/collector/tui.go +++ b/internal/collector/tui.go @@ -1,12 +1,11 @@ package collector import ( + "github.com/turbot/tailpipe/internal/database" "strings" "time" tea "github.com/charmbracelet/bubbletea" - - "github.com/turbot/tailpipe/internal/parquet" ) type collectionModel struct { @@ -64,7 +63,7 @@ func (c collectionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return c, nil case AwaitingCompactionMsg: // this doesn't do anything useful except trigger a view update with file compaction placeholder - cs := parquet.CompactionStatus{} + cs := database.CompactionStatus{} c.status.compactionStatus = &cs return c, nil case tickMsg: diff --git a/internal/parquet/ducklake.go b/internal/database/cleanup.go similarity index 93% rename from internal/parquet/ducklake.go rename to internal/database/cleanup.go index c7b371f9..a34743f7 100644 --- a/internal/parquet/ducklake.go +++ b/internal/database/cleanup.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "context" @@ -10,11 +10,10 @@ import ( "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/database" ) // DeletePartition deletes data for the specified partition and date range from the given Ducklake connected database. -func DeletePartition(ctx context.Context, partition *config.Partition, from, to time.Time, db *database.DuckDb) (rowCount int, err error) { +func DeletePartition(ctx context.Context, partition *config.Partition, from, to time.Time, db *DuckDb) (rowCount int, err error) { // TODO #DL https://github.com/turbot/tailpipe/issues/505 // if we are using s3 do not delete for now as this does not work at present (need explicit S3 support I think) // remove before release https://github.com/turbot/tailpipe/issues/520 @@ -66,7 +65,7 @@ func DeletePartition(ctx context.Context, partition *config.Partition, from, to } // DucklakeCleanup performs removes old snapshots deletes expired and unused parquet files from the DuckDB database. -func DucklakeCleanup(ctx context.Context, db *database.DuckDb) error { +func DucklakeCleanup(ctx context.Context, db *DuckDb) error { slog.Info("Cleaning up DuckLake snapshots and expired files") // now clean old snapshots if err := expirePrevSnapshots(ctx, db); err != nil { @@ -84,7 +83,7 @@ func DucklakeCleanup(ctx context.Context, db *database.DuckDb) error { // However we do not need (currently) take advantage of this ducklake functionality, so we can remove all but the latest snapshot // To do this we get the date of the most recent snapshot and then expire all snapshots older than that date. // We then call ducklake_cleanup to remove the expired files. -func expirePrevSnapshots(ctx context.Context, db *database.DuckDb) error { +func expirePrevSnapshots(ctx context.Context, db *DuckDb) error { slog.Info("Expiring old DuckLake snapshots") defer slog.Info("DuckLake snapshot expiration complete") @@ -127,7 +126,7 @@ func expirePrevSnapshots(ctx context.Context, db *database.DuckDb) error { } // cleanupExpiredFiles deletes and files marked as expired in the ducklake system. -func cleanupExpiredFiles(ctx context.Context, db *database.DuckDb) error { +func cleanupExpiredFiles(ctx context.Context, db *DuckDb) error { slog.Info("Cleaning up expired files in DuckLake") defer slog.Info("DuckLake expired files cleanup complete") diff --git a/internal/parquet/compact.go b/internal/database/compact.go similarity index 96% rename from internal/parquet/compact.go rename to internal/database/compact.go index 601c121e..a9e9c7b4 100644 --- a/internal/parquet/compact.go +++ b/internal/database/compact.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "context" @@ -10,7 +10,6 @@ import ( "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/tailpipe/internal/database" ) const ( @@ -18,7 +17,7 @@ const ( maxCompactionRowsPerChunk = 5_000_000 ) -func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), reindex bool, patterns ...*PartitionPattern) error { +func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(CompactionStatus), reindex bool, patterns ...*PartitionPattern) error { slog.Info("Compacting DuckLake data files") t := time.Now() @@ -95,7 +94,7 @@ func CompactDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func( // - analyze file fragmentation to identify overlapping time ranges // - for each overlapping time range, reorder all data in that range // - delete original unordered entries for that time range -func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(CompactionStatus), partitionKeys []*partitionKey, reindex bool) (*CompactionStatus, error) { +func orderDataFiles(ctx context.Context, db *DuckDb, updateFunc func(CompactionStatus), partitionKeys []*partitionKey, reindex bool) (*CompactionStatus, error) { slog.Info("Ordering DuckLake data files") status := NewCompactionStatus() @@ -217,7 +216,7 @@ func orderDataFiles(ctx context.Context, db *database.DuckDb, updateFunc func(Co } // getColumns retrieves column information for a table, checking the map first and reading from metadata if not present -func getColumns(ctx context.Context, db *database.DuckDb, table string, columns map[string][]string) ([]string, error) { +func getColumns(ctx context.Context, db *DuckDb, table string, columns map[string][]string) ([]string, error) { // Check if columns are already cached if cachedColumns, exists := columns[table]; exists { return cachedColumns, nil diff --git a/internal/parquet/compaction_status.go b/internal/database/compaction_status.go similarity index 97% rename from internal/parquet/compaction_status.go rename to internal/database/compaction_status.go index 887f98cb..5b345f31 100644 --- a/internal/parquet/compaction_status.go +++ b/internal/database/compaction_status.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "context" @@ -11,7 +11,6 @@ import ( "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/utils" - "github.com/turbot/tailpipe/internal/database" ) type CompactionStatus struct { @@ -111,7 +110,7 @@ func (s *CompactionStatus) UpdateProgress() { } -func (s *CompactionStatus) getInitialCounts(ctx context.Context, db *database.DuckDb, partitionKeys []*partitionKey) error { +func (s *CompactionStatus) getInitialCounts(ctx context.Context, db *DuckDb, partitionKeys []*partitionKey) error { partitionNameMap := make(map[string]map[string]struct{}) for _, pk := range partitionKeys { s.InitialFiles += pk.fileCount @@ -158,7 +157,7 @@ func (s *CompactionStatus) getInitialCounts(ctx context.Context, db *database.Du return nil } -func (s *CompactionStatus) getFinalFileCounts(ctx context.Context, db *database.DuckDb, partitionKeys []*partitionKey) error { +func (s *CompactionStatus) getFinalFileCounts(ctx context.Context, db *DuckDb, partitionKeys []*partitionKey) error { // Get unique table names from partition keys tableNames := make(map[string]struct{}) for _, pk := range partitionKeys { diff --git a/internal/parquet/compaction_types.go b/internal/database/compaction_types.go similarity index 94% rename from internal/parquet/compaction_types.go rename to internal/database/compaction_types.go index 8c25439b..aee3b479 100644 --- a/internal/parquet/compaction_types.go +++ b/internal/database/compaction_types.go @@ -1,19 +1,17 @@ -package parquet +package database import ( "context" "fmt" "strings" "time" - - "github.com/turbot/tailpipe/internal/database" ) // getTimeRangesToReorder analyzes file fragmentation and creates disorder metrics for a partition key. // It queries DuckLake metadata to get all files for the partition, their timestamp ranges, and row counts. // Then it identifies groups of files with overlapping time ranges that need compaction. // Returns metrics including total file count and overlapping file sets with their metadata. -func getTimeRangesToReorder(ctx context.Context, db *database.DuckDb, pk *partitionKey, reindex bool) (*reorderMetadata, error) { +func getTimeRangesToReorder(ctx context.Context, db *DuckDb, pk *partitionKey, reindex bool) (*reorderMetadata, error) { // NOTE: if we are reindexing, we must rewrite the entire partition key // - return a single range for the entire partition key if reindex { @@ -62,7 +60,7 @@ func getTimeRangesToReorder(ctx context.Context, db *database.DuckDb, pk *partit } // query the metadata to get a list of files, their timestamp ranges and row counts for this partition key -func getFileRangesForPartitionKey(ctx context.Context, db *database.DuckDb, pk *partitionKey) ([]fileTimeRange, error) { +func getFileRangesForPartitionKey(ctx context.Context, db *DuckDb, pk *partitionKey) ([]fileTimeRange, error) { query := `select df.path, cast(fcs.min_value as timestamp) as min_timestamp, diff --git a/internal/parquet/conversion_error.go b/internal/database/conversion_error.go similarity index 99% rename from internal/parquet/conversion_error.go rename to internal/database/conversion_error.go index 2e8e50a4..df06efd2 100644 --- a/internal/parquet/conversion_error.go +++ b/internal/database/conversion_error.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "bytes" diff --git a/internal/parquet/conversion_error_test.go b/internal/database/conversion_error_test.go similarity index 97% rename from internal/parquet/conversion_error_test.go rename to internal/database/conversion_error_test.go index be3766f2..04e022b2 100644 --- a/internal/parquet/conversion_error_test.go +++ b/internal/database/conversion_error_test.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "errors" diff --git a/internal/parquet/convertor.go b/internal/database/convertor.go similarity index 98% rename from internal/parquet/convertor.go rename to internal/database/convertor.go index cfe1bc3f..211e50c1 100644 --- a/internal/parquet/convertor.go +++ b/internal/database/convertor.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "context" @@ -10,7 +10,6 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/database" ) // TODO #DL @@ -78,10 +77,10 @@ type Converter struct { statusFunc func(int64, int64, ...error) // the DuckDB database connection - this must have a ducklake attachment - db *database.DuckDb + db *DuckDb } -func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executionId string, partition *config.Partition, sourceDir string, tableSchema *schema.TableSchema, statusFunc func(int64, int64, ...error), db *database.DuckDb) (*Converter, error) { +func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executionId string, partition *config.Partition, sourceDir string, tableSchema *schema.TableSchema, statusFunc func(int64, int64, ...error), db *DuckDb) (*Converter, error) { // get the data dir - this will already have been created by the config loader destDir := config.GlobalWorkspaceProfile.GetDataDir() diff --git a/internal/parquet/convertor_convert.go b/internal/database/convertor_convert.go similarity index 99% rename from internal/parquet/convertor_convert.go rename to internal/database/convertor_convert.go index a47821bb..d2cd3b7d 100644 --- a/internal/parquet/convertor_convert.go +++ b/internal/database/convertor_convert.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "errors" diff --git a/internal/parquet/convertor_ducklake.go b/internal/database/convertor_ducklake.go similarity index 99% rename from internal/parquet/convertor_ducklake.go rename to internal/database/convertor_ducklake.go index 28e3b8a1..3ed7e892 100644 --- a/internal/parquet/convertor_ducklake.go +++ b/internal/database/convertor_ducklake.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/parquet/convertor_schema.go b/internal/database/convertor_schema.go similarity index 96% rename from internal/parquet/convertor_schema.go rename to internal/database/convertor_schema.go index 11e10278..929d0795 100644 --- a/internal/parquet/convertor_schema.go +++ b/internal/database/convertor_schema.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "encoding/json" @@ -7,7 +7,6 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe-plugin-sdk/table" - "github.com/turbot/tailpipe/internal/database" ) // populate the ConversionSchema @@ -98,7 +97,7 @@ func (w *Converter) inferSchemaForJSONLFileWithJSONStructure(filePath string) (* return res, nil } -func (w *Converter) inferSchemaForJSONLFileWithDescribe(db *database.DuckDb, filePath string) (*schema.TableSchema, error) { +func (w *Converter) inferSchemaForJSONLFileWithDescribe(db *DuckDb, filePath string) (*schema.TableSchema, error) { // Use DuckDB to describe the schema of the JSONL file query := `SELECT column_name, column_type FROM (DESCRIBE (SELECT * FROM read_json_auto(?)))` diff --git a/internal/parquet/convertor_validate.go b/internal/database/convertor_validate.go similarity index 99% rename from internal/parquet/convertor_validate.go rename to internal/database/convertor_validate.go index ab7d6769..d2f4d3f1 100644 --- a/internal/parquet/convertor_validate.go +++ b/internal/database/convertor_validate.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index ee8f46b3..f35756c3 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -272,8 +272,6 @@ func GetDucklakeInitCommands(readonly bool) []SqlCommand { fmt.Sprintf("data_path '%s'", config.GlobalWorkspaceProfile.GetDataDir()), "meta_journal_mode 'WAL'", "meta_synchronous 'NORMAL'", - // TODO temp disable timeout - "meta_busy_timeout 0", } // if readonly mode is requested, add the option if readonly { diff --git a/internal/parquet/file_metadata.go b/internal/database/file_metadata.go similarity index 92% rename from internal/parquet/file_metadata.go rename to internal/database/file_metadata.go index 165521e4..6d48665c 100644 --- a/internal/parquet/file_metadata.go +++ b/internal/database/file_metadata.go @@ -1,11 +1,10 @@ -package parquet +package database import ( "context" "fmt" "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/tailpipe/internal/database" ) // FileMetadata represents the result of a file metadata query @@ -16,7 +15,7 @@ type FileMetadata struct { } // TableExists checks if a table exists in the DuckLake metadata tables -func TableExists(ctx context.Context, tableName string, db *database.DuckDb) (bool, error) { +func TableExists(ctx context.Context, tableName string, db *DuckDb) (bool, error) { query := fmt.Sprintf(`select count(*) from %s.ducklake_table where table_name = ?`, constants.DuckLakeMetadataCatalog) var count int64 @@ -29,7 +28,7 @@ func TableExists(ctx context.Context, tableName string, db *database.DuckDb) (bo } // GetTableFileMetadata gets file metadata for a specific table from DuckLake metadata tables -func GetTableFileMetadata(ctx context.Context, tableName string, db *database.DuckDb) (*FileMetadata, error) { +func GetTableFileMetadata(ctx context.Context, tableName string, db *DuckDb) (*FileMetadata, error) { // first see if the table exists exists, err := TableExists(ctx, tableName, db) if err != nil { @@ -66,7 +65,7 @@ where tp.table_name = ? and f.end_snapshot is null`, } // GetPartitionFileMetadata gets file metadata for a specific partition from DuckLake metadata tables -func GetPartitionFileMetadata(ctx context.Context, tableName, partitionName string, db *database.DuckDb) (*FileMetadata, error) { +func GetPartitionFileMetadata(ctx context.Context, tableName, partitionName string, db *DuckDb) (*FileMetadata, error) { // first see if the table exists exists, err := TableExists(ctx, tableName, db) if err != nil { diff --git a/internal/parquet/partition_key.go b/internal/database/partition_key.go similarity index 96% rename from internal/parquet/partition_key.go rename to internal/database/partition_key.go index 0e742c87..053ad58d 100644 --- a/internal/parquet/partition_key.go +++ b/internal/database/partition_key.go @@ -1,10 +1,9 @@ -package parquet +package database import ( "context" "fmt" "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/database" "sort" ) @@ -23,7 +22,7 @@ type partitionKey struct { // query the ducklake_data_file table to get all partition keys combinations which satisfy the provided patterns, // along with the file and row stats for each partition key combination -func getPartitionKeysMatchingPattern(ctx context.Context, db *database.DuckDb, patterns []*PartitionPattern) ([]*partitionKey, error) { +func getPartitionKeysMatchingPattern(ctx context.Context, db *DuckDb, patterns []*PartitionPattern) ([]*partitionKey, error) { // This query joins the DuckLake metadata tables to get partition key combinations: // - ducklake_data_file: contains file metadata and links to tables // - ducklake_file_partition_value: contains partition values for each file diff --git a/internal/parquet/partition_key_test.go b/internal/database/partition_key_test.go similarity index 99% rename from internal/parquet/partition_key_test.go rename to internal/database/partition_key_test.go index 1774b0c7..f995aa55 100644 --- a/internal/parquet/partition_key_test.go +++ b/internal/database/partition_key_test.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "testing" diff --git a/internal/parquet/partition_pattern.go b/internal/database/partition_pattern.go similarity index 99% rename from internal/parquet/partition_pattern.go rename to internal/database/partition_pattern.go index aa7d6de1..ddaae33d 100644 --- a/internal/parquet/partition_pattern.go +++ b/internal/database/partition_pattern.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/parquet/partition_pattern_test.go b/internal/database/partition_pattern_test.go similarity index 99% rename from internal/parquet/partition_pattern_test.go rename to internal/database/partition_pattern_test.go index 56d0d2d1..81158c1d 100644 --- a/internal/parquet/partition_pattern_test.go +++ b/internal/database/partition_pattern_test.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "github.com/turbot/pipe-fittings/v2/modconfig" diff --git a/internal/parquet/read_json_query.go b/internal/database/read_json_query.go similarity index 99% rename from internal/parquet/read_json_query.go rename to internal/database/read_json_query.go index b1031998..cf7e4c78 100644 --- a/internal/parquet/read_json_query.go +++ b/internal/database/read_json_query.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/parquet/read_json_query_test.go b/internal/database/read_json_query_test.go similarity index 99% rename from internal/parquet/read_json_query_test.go rename to internal/database/read_json_query_test.go index 420b924a..f70d2a96 100644 --- a/internal/parquet/read_json_query_test.go +++ b/internal/database/read_json_query_test.go @@ -1,4 +1,4 @@ -package parquet +package database import ( _ "github.com/marcboeker/go-duckdb/v2" diff --git a/internal/parquet/reorder_metadata.go b/internal/database/reorder_metadata.go similarity index 82% rename from internal/parquet/reorder_metadata.go rename to internal/database/reorder_metadata.go index b3316b19..67139a50 100644 --- a/internal/parquet/reorder_metadata.go +++ b/internal/database/reorder_metadata.go @@ -1,9 +1,8 @@ -package parquet +package database import ( "context" "fmt" - "github.com/turbot/tailpipe/internal/database" "time" ) @@ -17,7 +16,7 @@ type reorderMetadata struct { maxTimestamp time.Time } -func newReorderMetadata(ctx context.Context, db *database.DuckDb, p *partitionKey) (*reorderMetadata, error) { +func newReorderMetadata(ctx context.Context, db *DuckDb, p *partitionKey) (*reorderMetadata, error) { var rm = &reorderMetadata{pk: p} // Query to get row count and time range for this partition diff --git a/internal/parquet/row_validation_error.go b/internal/database/row_validation_error.go similarity index 97% rename from internal/parquet/row_validation_error.go rename to internal/database/row_validation_error.go index ad886c1b..e9f5f4b6 100644 --- a/internal/parquet/row_validation_error.go +++ b/internal/database/row_validation_error.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/parquet/schema_change_error.go b/internal/database/schema_change_error.go similarity index 97% rename from internal/parquet/schema_change_error.go rename to internal/database/schema_change_error.go index f6fb1574..e4a3c965 100644 --- a/internal/parquet/schema_change_error.go +++ b/internal/database/schema_change_error.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/parquet/schema_comparison.go b/internal/database/schema_comparison.go similarity index 99% rename from internal/parquet/schema_comparison.go rename to internal/database/schema_comparison.go index 0b7c2588..3e46e05e 100644 --- a/internal/parquet/schema_comparison.go +++ b/internal/database/schema_comparison.go @@ -1,4 +1,4 @@ -package parquet +package database import ( "fmt" diff --git a/internal/display/partition.go b/internal/display/partition.go index 935aefeb..da38938c 100644 --- a/internal/display/partition.go +++ b/internal/display/partition.go @@ -6,7 +6,6 @@ import ( "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/parquet" ) // PartitionResource represents a partition resource and is used for list/show commands @@ -95,7 +94,7 @@ func GetPartitionResource(ctx context.Context, p *config.Partition, db *database func (r *PartitionResource) setFileInformation(ctx context.Context, db *database.DuckDb) error { // Get file metadata using shared function - metadata, err := parquet.GetPartitionFileMetadata(ctx, r.table, r.partition, db) + metadata, err := database.GetPartitionFileMetadata(ctx, r.table, r.partition, db) if err != nil { return fmt.Errorf("unable to obtain file metadata: %w", err) } diff --git a/internal/display/table.go b/internal/display/table.go index 4ac948fb..28bfbd5b 100644 --- a/internal/display/table.go +++ b/internal/display/table.go @@ -14,7 +14,6 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" ) @@ -213,7 +212,7 @@ func (r *TableResource) setPartitions() { func (r *TableResource) setFileInformation(ctx context.Context, db *database.DuckDb) error { // Get file metadata using shared function - metadata, err := parquet.GetTableFileMetadata(ctx, r.Name, db) + metadata, err := database.GetTableFileMetadata(ctx, r.Name, db) if err != nil { return fmt.Errorf("unable to obtain file metadata: %w", err) } diff --git a/internal/parquet/file_helpers.go b/internal/parquet/file_helpers.go deleted file mode 100644 index 4b903d4f..00000000 --- a/internal/parquet/file_helpers.go +++ /dev/null @@ -1,143 +0,0 @@ -package parquet - -import ( - "context" - "fmt" - "golang.org/x/sync/semaphore" - "log/slog" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - - "github.com/turbot/pipe-fittings/v2/utils" -) - -// addExtensionToFiles renames all given files to add a the provided extension -func addExtensionToFiles(fileNames []string, suffix string) ([]string, error) { - if len(fileNames) == 0 { - return []string{}, nil - } - var renamedFiles []string - for _, file := range fileNames { - // Check if file exists - if _, err := os.Stat(file); os.IsNotExist(err) { - return nil, fmt.Errorf("file does not exist: %s", file) - } - - newFile := file + suffix - renamedFiles = append(renamedFiles, newFile) - if err := os.Rename(file, newFile); err != nil { - // try to rename all files we have already renamed back to their original names - for _, renamedFile := range renamedFiles { - // remove the .compacted extension - originalFile := strings.TrimSuffix(renamedFile, ".compacted") - if err := os.Rename(renamedFile, originalFile); err != nil { - slog.Warn("Failed to rename parquet file back to original name", "file", renamedFile, "error", err) - } - } - return nil, fmt.Errorf("failed to rename parquet file %s to %s: %w", file, newFile, err) - } - } - return renamedFiles, nil -} - -// removeExtensionFromFiles renames all given files to remove the provided extension -func removeExtensionFromFiles(fileNames []string, suffix string) error { - var renamedFiles []string - for _, file := range fileNames { - if !strings.HasSuffix(file, suffix) { - continue // skip files that do not have the suffix - } - newFile := strings.TrimSuffix(file, suffix) - renamedFiles = append(renamedFiles, newFile) - if err := os.Rename(file, newFile); err != nil { - // try to rename all files we have already renamed back to their original names - for _, renamedFile := range renamedFiles { - if err := os.Rename(renamedFile, renamedFile+suffix); err != nil { - slog.Warn("Failed to rename parquet file back to original name", "file", renamedFile, "error", err) - } - } - return fmt.Errorf("failed to rename parquet file %s to %s: %w", file, newFile, err) - } - } - return nil -} - -// deleteFilesConcurrently deletes the given parquet files concurrently, ensuring that empty parent directories are -// also cleaned recursively up to the baseDir. -func deleteFilesConcurrently(ctx context.Context, parquetFiles []string, baseDir string) error { - const maxConcurrentDeletions = 5 - sem := semaphore.NewWeighted(int64(maxConcurrentDeletions)) - var wg sync.WaitGroup - var failures int32 - - dirSet := make(map[string]struct{}) - var dirMu sync.Mutex - - for _, file := range parquetFiles { - wg.Add(1) - go func(file string) { - defer wg.Done() - - if err := sem.Acquire(ctx, 1); err != nil { - atomic.AddInt32(&failures, 1) - return - } - defer sem.Release(1) - - if err := os.Remove(file); err != nil { - atomic.AddInt32(&failures, 1) - return - } - - // Collect parent directory for cleanup - parentDir := filepath.Dir(file) - dirMu.Lock() - dirSet[parentDir] = struct{}{} - dirMu.Unlock() - }(file) - } - - wg.Wait() - - // Recursively delete empty parent dirs up to baseDir - for startDir := range dirSet { - deleteEmptyParentsUpTo(startDir, baseDir) - } - - if atomic.LoadInt32(&failures) > 0 { - return fmt.Errorf("failed to delete %d parquet %s", failures, utils.Pluralize("file", int(failures))) - } - return nil -} - -// deleteEmptyParentsUpTo deletes empty directories upward from startDir up to (but not including) baseDir. -func deleteEmptyParentsUpTo(startDir, baseDir string) { - baseDirAbs, err := filepath.Abs(baseDir) - if err != nil { - return // fail-safe: don't recurse without a valid base - } - current := startDir - - for { - currentAbs, err := filepath.Abs(current) - if err != nil { - return - } - - // Stop if we've reached or passed the baseDir - if currentAbs == baseDirAbs || !strings.HasPrefix(currentAbs, baseDirAbs) { - return - } - - entries, err := os.ReadDir(current) - if err != nil || len(entries) > 0 { - return // non-empty or inaccessible - } - - _ = os.Remove(current) // delete and continue upward - current = filepath.Dir(current) - } -} diff --git a/internal/parquet/file_helpers_test.go b/internal/parquet/file_helpers_test.go deleted file mode 100644 index e8b0f125..00000000 --- a/internal/parquet/file_helpers_test.go +++ /dev/null @@ -1,656 +0,0 @@ -package parquet - -import ( - "context" - "fmt" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_addExtensionToFiles(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "add_extension_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - type args struct { - fileNames []string - suffix string - } - tests := []struct { - name string - args args - want []string - wantErr assert.ErrorAssertionFunc - }{ - { - name: "add extension to single file", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test1.parquet")}, - suffix: ".compacted", - }, - want: []string{filepath.Join(tempDir, "test1.parquet.compacted")}, - wantErr: assert.NoError, - }, - { - name: "add extension to multiple files", - args: args{ - fileNames: []string{ - filepath.Join(tempDir, "test1.parquet"), - filepath.Join(tempDir, "test2.parquet"), - }, - suffix: ".compacted", - }, - want: []string{ - filepath.Join(tempDir, "test1.parquet.compacted"), - filepath.Join(tempDir, "test2.parquet.compacted"), - }, - wantErr: assert.NoError, - }, - { - name: "empty file list", - args: args{ - fileNames: []string{}, - suffix: ".compacted", - }, - want: []string{}, - wantErr: assert.NoError, - }, - { - name: "non-existent file", - args: args{ - fileNames: []string{filepath.Join(tempDir, "nonexistent.parquet")}, - suffix: ".compacted", - }, - want: nil, - wantErr: assert.Error, - }, - { - name: "file with spaces in name", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test file.parquet")}, - suffix: ".compacted", - }, - want: []string{filepath.Join(tempDir, "test file.parquet.compacted")}, - wantErr: assert.NoError, - }, - { - name: "file with special characters", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test@#$%.parquet")}, - suffix: ".compacted", - }, - want: []string{filepath.Join(tempDir, "test@#$%.parquet.compacted")}, - wantErr: assert.NoError, - }, - { - name: "file with multiple dots", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test.version.1.parquet")}, - suffix: ".compacted", - }, - want: []string{filepath.Join(tempDir, "test.version.1.parquet.compacted")}, - wantErr: assert.NoError, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create test files if they should exist - if reflect.ValueOf(tt.wantErr).Pointer() == reflect.ValueOf(assert.NoError).Pointer() { - for _, file := range tt.args.fileNames { - //nolint:gosec // just test code - if err := os.WriteFile(file, []byte("test data"), 0644); err != nil { - t.Fatalf("Failed to create test file %s: %v", file, err) - } - } - } - - got, err := addExtensionToFiles(tt.args.fileNames, tt.args.suffix) - if !tt.wantErr(t, err, fmt.Sprintf("addExtensionToFiles(%v, %v)", tt.args.fileNames, tt.args.suffix)) { - return - } - assert.Equalf(t, tt.want, got, "addExtensionToFiles(%v, %v)", tt.args.fileNames, tt.args.suffix) - - // Verify files exist with new names if no error - if err == nil { - for _, file := range got { - _, err := os.Stat(file) - assert.NoError(t, err, "File %s should exist", file) - } - } - }) - } -} - -func Test_deleteEmptyParentsUpTo(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "delete_empty_parents_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - type args struct { - startDir string - baseDir string - } - tests := []struct { - name string - args args - setup func() error - verify func(t *testing.T) - }{ - { - name: "delete empty parent directories", - args: args{ - startDir: filepath.Join(tempDir, "a", "b", "c"), - baseDir: tempDir, - }, - setup: func() error { - // Create directory structure - return os.MkdirAll(filepath.Join(tempDir, "a", "b", "c"), 0755) - }, - verify: func(t *testing.T) { - // Verify all parent directories are deleted - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b", "c")) - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b")) - assert.NoDirExists(t, filepath.Join(tempDir, "a")) - }, - }, - { - name: "keep non-empty parent directories", - args: args{ - startDir: filepath.Join(tempDir, "a", "b", "c"), - baseDir: tempDir, - }, - setup: func() error { - // Create directory structure - if err := os.MkdirAll(filepath.Join(tempDir, "a", "b", "c"), 0755); err != nil { - return err - } - // Create a file in one of the parent directories - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(tempDir, "a", "test.txt"), []byte("test"), 0644) - }, - verify: func(t *testing.T) { - // Verify empty directories are deleted but non-empty ones remain - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b", "c")) - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b")) - assert.DirExists(t, filepath.Join(tempDir, "a")) - assert.FileExists(t, filepath.Join(tempDir, "a", "test.txt")) - }, - }, - { - name: "start directory equals base directory", - args: args{ - startDir: tempDir, - baseDir: tempDir, - }, - setup: func() error { - return nil // No setup needed - }, - verify: func(t *testing.T) { - // Verify base directory still exists - assert.DirExists(t, tempDir) - }, - }, - { - name: "start directory outside base directory", - args: args{ - startDir: "/tmp/outside", - baseDir: tempDir, - }, - setup: func() error { - return nil // No setup needed - }, - verify: func(t *testing.T) { - // Verify base directory still exists - assert.DirExists(t, tempDir) - }, - }, - { - name: "directory with hidden files", - args: args{ - startDir: filepath.Join(tempDir, "a", "b", "c"), - baseDir: tempDir, - }, - setup: func() error { - if err := os.MkdirAll(filepath.Join(tempDir, "a", "b", "c"), 0755); err != nil { - return err - } - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(tempDir, "a", ".hidden"), []byte("test"), 0644) - }, - verify: func(t *testing.T) { - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b", "c")) - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b")) - assert.DirExists(t, filepath.Join(tempDir, "a")) - assert.FileExists(t, filepath.Join(tempDir, "a", ".hidden")) - }, - }, - { - name: "directory with subdirectories", - args: args{ - startDir: filepath.Join(tempDir, "a", "b", "c"), - baseDir: tempDir, - }, - setup: func() error { - // Create the directory structure - if err := os.MkdirAll(filepath.Join(tempDir, "a", "b", "c"), 0755); err != nil { - return err - } - // Create a file in the deepest directory - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(tempDir, "a", "b", "c", "test.txt"), []byte("test"), 0644); err != nil { - return err - } - // Remove the file to make the directory empty - if err := os.Remove(filepath.Join(tempDir, "a", "b", "c", "test.txt")); err != nil { - return err - } - return nil - }, - verify: func(t *testing.T) { - // Debug: Check if directories are empty before deletion - entries, err := os.ReadDir(filepath.Join(tempDir, "a", "b", "c")) - if err != nil { - t.Logf("Error reading c directory: %v", err) - } else { - t.Logf("Entries in c directory: %d", len(entries)) - } - - entries, err = os.ReadDir(filepath.Join(tempDir, "a", "b")) - if err != nil { - t.Logf("Error reading b directory: %v", err) - } else { - t.Logf("Entries in b directory: %d", len(entries)) - } - - entries, err = os.ReadDir(filepath.Join(tempDir, "a")) - if err != nil { - t.Logf("Error reading a directory: %v", err) - } else { - t.Logf("Entries in a directory: %d", len(entries)) - } - - // Call the function we're testing - deleteEmptyParentsUpTo(filepath.Join(tempDir, "a", "b", "c"), tempDir) - - // Verify directories are deleted in the correct order - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b", "c"), "c directory should be deleted") - assert.NoDirExists(t, filepath.Join(tempDir, "a", "b"), "b directory should be deleted") - // Relaxed assertion for 'a': only assert deletion if empty - entries, err = os.ReadDir(filepath.Join(tempDir, "a")) - if os.IsNotExist(err) { - t.Logf("'a' directory deleted as expected (was empty)") - } else if err == nil { - t.Logf("'a' directory still exists (not empty), entries: %d", len(entries)) - assert.DirExists(t, filepath.Join(tempDir, "a"), "a directory should still exist if not empty") - } else { - t.Errorf("Unexpected error reading 'a' directory: %v", err) - } - // Verify base directory still exists - assert.DirExists(t, tempDir, "base directory should still exist") - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup test environment - if err := tt.setup(); err != nil { - t.Fatalf("Failed to setup test: %v", err) - } - - // Run the function - deleteEmptyParentsUpTo(tt.args.startDir, tt.args.baseDir) - - // Verify results - tt.verify(t) - }) - } -} - -func Test_deleteFilesConcurrently(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "delete_files_concurrent_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - type args struct { - ctx context.Context - parquetFiles []string - baseDir string - } - tests := []struct { - name string - args args - setup func() error - wantErr assert.ErrorAssertionFunc - }{ - { - name: "delete multiple files successfully", - args: args{ - ctx: context.Background(), - parquetFiles: []string{ - filepath.Join(tempDir, "file1.parquet"), - filepath.Join(tempDir, "file2.parquet"), - }, - baseDir: tempDir, - }, - setup: func() error { - // Create test files - for _, file := range []string{"file1.parquet", "file2.parquet"} { - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(tempDir, file), []byte("test data"), 0644); err != nil { - return err - } - } - return nil - }, - wantErr: assert.NoError, - }, - { - name: "delete files in nested directories", - args: args{ - ctx: context.Background(), - parquetFiles: []string{ - filepath.Join(tempDir, "a", "b", "file1.parquet"), - filepath.Join(tempDir, "a", "b", "file2.parquet"), - }, - baseDir: tempDir, - }, - setup: func() error { - // Create directory structure and files - for _, file := range []string{"file1.parquet", "file2.parquet"} { - dir := filepath.Join(tempDir, "a", "b") - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(dir, file), []byte("test data"), 0644); err != nil { - return err - } - } - return nil - }, - wantErr: assert.NoError, - }, - { - name: "handle non-existent files", - args: args{ - ctx: context.Background(), - parquetFiles: []string{ - filepath.Join(tempDir, "nonexistent1.parquet"), - filepath.Join(tempDir, "nonexistent2.parquet"), - }, - baseDir: tempDir, - }, - setup: func() error { - return nil // No setup needed - }, - wantErr: assert.Error, - }, - { - name: "handle cancelled context", - args: args{ - ctx: func() context.Context { - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - return ctx - }(), - parquetFiles: []string{ - filepath.Join(tempDir, "file1.parquet"), - filepath.Join(tempDir, "file2.parquet"), - }, - baseDir: tempDir, - }, - setup: func() error { - // Create test files - for _, file := range []string{"file1.parquet", "file2.parquet"} { - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(tempDir, file), []byte("test data"), 0644); err != nil { - return err - } - } - return nil - }, - wantErr: assert.Error, - }, - { - name: "large number of files", - args: args{ - ctx: context.Background(), - parquetFiles: func() []string { - var files []string - for i := 0; i < 20; i++ { - files = append(files, filepath.Join(tempDir, fmt.Sprintf("file%d.parquet", i))) - } - return files - }(), - baseDir: tempDir, - }, - setup: func() error { - for i := 0; i < 20; i++ { - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(tempDir, fmt.Sprintf("file%d.parquet", i)), []byte("test data"), 0644); err != nil { - return err - } - } - return nil - }, - wantErr: assert.NoError, - }, - { - name: "deeply nested directories", - args: args{ - ctx: context.Background(), - parquetFiles: []string{ - filepath.Join(tempDir, "a", "b", "c", "d", "e", "file.parquet"), - }, - baseDir: tempDir, - }, - setup: func() error { - dir := filepath.Join(tempDir, "a", "b", "c", "d", "e") - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(dir, "file.parquet"), []byte("test data"), 0644) - }, - wantErr: assert.NoError, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup test environment - if err := tt.setup(); err != nil { - t.Fatalf("Failed to setup test: %v", err) - } - - // Run the function - err := deleteFilesConcurrently(tt.args.ctx, tt.args.parquetFiles, tt.args.baseDir) - tt.wantErr(t, err, fmt.Sprintf("deleteFilesConcurrently(%v, %v, %v)", tt.args.ctx, tt.args.parquetFiles, tt.args.baseDir)) - - // If no error expected, verify files are deleted - if err == nil { - for _, file := range tt.args.parquetFiles { - _, err := os.Stat(file) - assert.True(t, os.IsNotExist(err), "File %s should be deleted", file) - } - } - }) - } -} - -func Test_removeExtensionFromFiles(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "remove_extension_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - type args struct { - fileNames []string - suffix string - } - tests := []struct { - name string - args args - setup func() error - wantErr assert.ErrorAssertionFunc - }{ - { - name: "remove extension from single file", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test1.parquet.compacted")}, - suffix: ".compacted", - }, - setup: func() error { - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(tempDir, "test1.parquet.compacted"), []byte("test data"), 0644) - }, - wantErr: assert.NoError, - }, - { - name: "remove extension from multiple files", - args: args{ - fileNames: []string{ - filepath.Join(tempDir, "test1.parquet.compacted"), - filepath.Join(tempDir, "test2.parquet.compacted"), - }, - suffix: ".compacted", - }, - setup: func() error { - for _, file := range []string{"test1.parquet.compacted", "test2.parquet.compacted"} { - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(tempDir, file), []byte("test data"), 0644); err != nil { - return err - } - } - return nil - }, - wantErr: assert.NoError, - }, - { - name: "skip files without suffix", - args: args{ - fileNames: []string{ - filepath.Join(tempDir, "test1.parquet"), - filepath.Join(tempDir, "test2.parquet.compacted"), - }, - suffix: ".compacted", - }, - setup: func() error { - for _, file := range []string{"test1.parquet", "test2.parquet.compacted"} { - //nolint:gosec // just test code - if err := os.WriteFile(filepath.Join(tempDir, file), []byte("test data"), 0644); err != nil { - return err - } - } - return nil - }, - wantErr: assert.NoError, - }, - { - name: "empty file list", - args: args{ - fileNames: []string{}, - suffix: ".compacted", - }, - setup: func() error { - return nil // No setup needed - }, - wantErr: assert.NoError, - }, - { - name: "non-existent file", - args: args{ - fileNames: []string{filepath.Join(tempDir, "nonexistent.parquet.compacted")}, - suffix: ".compacted", - }, - setup: func() error { - return nil // No setup needed - }, - wantErr: assert.Error, - }, - { - name: "file with multiple extensions", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test.parquet.compacted.backup")}, - suffix: ".compacted", - }, - setup: func() error { - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(tempDir, "test.parquet.compacted.backup"), []byte("test data"), 0644) - }, - wantErr: assert.NoError, - }, - { - name: "file with spaces in name", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test file.parquet.compacted")}, - suffix: ".compacted", - }, - setup: func() error { - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(tempDir, "test file.parquet.compacted"), []byte("test data"), 0644) - }, - wantErr: assert.NoError, - }, - { - name: "file with special characters", - args: args{ - fileNames: []string{filepath.Join(tempDir, "test@#$%.parquet.compacted")}, - suffix: ".compacted", - }, - setup: func() error { - //nolint:gosec // just test code - return os.WriteFile(filepath.Join(tempDir, "test@#$%.parquet.compacted"), []byte("test data"), 0644) - }, - wantErr: assert.NoError, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup test environment - if err := tt.setup(); err != nil { - t.Fatalf("Failed to setup test: %v", err) - } - - // Run the function - err := removeExtensionFromFiles(tt.args.fileNames, tt.args.suffix) - tt.wantErr(t, err, fmt.Sprintf("removeExtensionFromFiles(%v, %v)", tt.args.fileNames, tt.args.suffix)) - - // If no error expected, verify files are renamed correctly - if err == nil { - for _, file := range tt.args.fileNames { - if !strings.HasSuffix(file, tt.args.suffix) { - continue // Skip files that don't have the suffix - } - // Check that the file with suffix no longer exists - _, err := os.Stat(file) - assert.True(t, os.IsNotExist(err), "File with suffix %s should be renamed", file) - - // Check that the file without suffix exists - newFile := strings.TrimSuffix(file, tt.args.suffix) - _, err = os.Stat(newFile) - assert.NoError(t, err, "Renamed file %s should exist", newFile) - } - } - }) - } -} From cb5a742d05df3bcf46d4fcf4df07334b4f95c7d4 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Wed, 10 Sep 2025 10:55:17 +0100 Subject: [PATCH 30/61] feat: re-add row validation. Fix getFileRangesForPartitionKeyCloses to add table constraint. #479 - conversion error handling should now work. Closes #480 - add prefix to handleConversionError - update formatting of row validation error - rename ProcessChunks to processAllChunks to simplify loop and factor out a processChunks function to process a set of chunks - simplify wait group handling - always decrement in defer of processChunks whether error or success - update schedule locking to only lock around insertion into scheduledChunks and for duration of getChunksToProcess --- cmd/compact.go | 2 +- internal/database/compaction_types.go | 2 + internal/database/conversion_error.go | 7 +- internal/database/convertor.go | 50 +----- internal/database/convertor_convert.go | 209 +++++++++++----------- internal/database/convertor_schema.go | 73 ++++---- internal/database/convertor_validate.go | 16 +- internal/database/read_json_query.go | 4 +- internal/database/row_validation_error.go | 3 +- 9 files changed, 169 insertions(+), 197 deletions(-) diff --git a/cmd/compact.go b/cmd/compact.go index 8a027720..e13bdc1b 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -29,7 +29,7 @@ import ( func compactCmd() *cobra.Command { cmd := &cobra.Command{ Use: "compact [table|table.partition] [flags]", - Args: cobra.ArbitraryArgs, + Args: cobra.MaximumNArgs(1), Run: runCompactCmd, Short: "Compact multiple parquet files per day to one per day", Long: `Compact multiple parquet files per day to one per day.`, diff --git a/internal/database/compaction_types.go b/internal/database/compaction_types.go index aee3b479..4a2f98e1 100644 --- a/internal/database/compaction_types.go +++ b/internal/database/compaction_types.go @@ -79,8 +79,10 @@ func getFileRangesForPartitionKey(ctx context.Context, db *DuckDb, pk *partition on df.table_id = t.table_id join __ducklake_metadata_tailpipe_ducklake.ducklake_file_column_statistics fcs on df.data_file_id = fcs.data_file_id + and df.table_id = fcs.table_id join __ducklake_metadata_tailpipe_ducklake.ducklake_column c on fcs.column_id = c.column_id + and fcs.table_id = c.table_id where t.table_name = ? and fpv1.partition_value = ? and fpv2.partition_value = ? diff --git a/internal/database/conversion_error.go b/internal/database/conversion_error.go index df06efd2..d4c8608a 100644 --- a/internal/database/conversion_error.go +++ b/internal/database/conversion_error.go @@ -10,10 +10,9 @@ import ( "strings" ) -// handleConversionError attempts to handle conversion errors by counting the number of lines in the file. +// handleConversionError attempts to handle conversion errors by counting the number of lines in the files. // if we fail, just return the raw error. -// TODO we need to pass an error prefix into here so we know the context https://github.com/turbot/tailpipe/issues/477 -func handleConversionError(err error, paths ...string) error { +func handleConversionError(message string, err error, paths ...string) error { logArgs := []any{ "error", err, @@ -34,7 +33,7 @@ func handleConversionError(err error, paths ...string) error { } // return wrapped error - return NewConversionError(err, rows, paths...) + return NewConversionError(fmt.Errorf("%s: %w", message, err), rows, paths...) } func countLinesForFiles(filenames ...string) (int64, error) { total := 0 diff --git a/internal/database/convertor.go b/internal/database/convertor.go index 211e50c1..6f11ba66 100644 --- a/internal/database/convertor.go +++ b/internal/database/convertor.go @@ -123,26 +123,19 @@ func (w *Converter) AddChunk(executionId string, chunk int32) error { // lock the schedule lock to ensure that we can safely add to the scheduled scheduledChunks w.scheduleLock.Lock() - defer w.scheduleLock.Unlock() - // add to scheduled scheduledChunks w.scheduledChunks = append(w.scheduledChunks, chunk) + w.scheduleLock.Unlock() + // increment the wait group to track the scheduled chunk w.wg.Add(1) // ok try to lock the process lock - that will fail if another process is running if w.processLock.TryLock() { - // so we have the process lock AND the schedule lock - // store the chunk to process - - // move the scheduled chunks to the chunks to process - // (scheduledChunks may be empty, in which case we will break out of the loop) - chunksToProcess := w.getChunksToProcess() - // and process = we now have the process lock // NOTE: process chunks will keep processing as long as there are scheduledChunks to process, including // scheduledChunks that were scheduled while we were processing - go w.processChunks(chunksToProcess) + go w.processAllChunks() } return nil @@ -151,6 +144,10 @@ func (w *Converter) AddChunk(executionId string, chunk int32) error { // getChunksToProcess returns the chunks to process, up to a maximum of maxChunksToProcess // it also trims the scheduledChunks to remove the processed chunks func (w *Converter) getChunksToProcess() []int32 { + // now determine if there are more chunks to process + w.scheduleLock.Lock() + defer w.scheduleLock.Unlock() + // TODO #DL do we even need this https://github.com/turbot/tailpipe/issues/523 const maxChunksToProcess = 2000 var chunksToProcess []int32 @@ -209,7 +206,7 @@ func (w *Converter) WaitForConversions(ctx context.Context) error { } } -//nolint:unused // we will use this once we re-add conversion error handling +// addJobErrors calls the status func with any job errors, first summing the failed rows in any conversion errors func (w *Converter) addJobErrors(errorList ...error) { var failedRowCount int64 @@ -231,34 +228,3 @@ func (w *Converter) updateRowCount(count int64) { // call the status function with the new row count w.statusFunc(atomic.LoadInt64(&w.rowCount), atomic.LoadInt64(&w.failedRowCount)) } - -// updateCompletionCount atomically increments the completion count -//func (w *Converter) updateCompletionCount(fileCount, conversionCount int32) { -// atomic.AddInt32(&w.fileCount, fileCount) -// atomic.AddInt32(&w.conversionCount, conversionCount) -//} -// -//func (w *Converter) GetCompletionCount() int32 { -// return atomic.LoadInt32(&w.fileCount) -//} - -// TODO #DL think about memory -// https://github.com/turbot/tailpipe/issues/478 - -//func (w *conversionWorker) forceMemoryRelease() error { -// // we need to flush the memory to release it - do this by setting a low memory limit then the full one -// // NOTE: do not set the memory to zero as we have temp table data -// const minMemoryMb = 64 -// -// // Set to minimum memory - note the use of ? parameter -// if _, err := w.db.Exec("set max_memory = ? || 'MB';", minMemoryMb); err != nil { -// return fmt.Errorf("memory flush failed: %w", err) -// } -// -// // Reset to configured memory limit -// if _, err := w.db.Exec("set max_memory = ? || 'MB';", w.maxMemoryMb); err != nil { -// return fmt.Errorf("memory reset failed: %w", err) -// } -// return nil -// -//} diff --git a/internal/database/convertor_convert.go b/internal/database/convertor_convert.go index d2cd3b7d..846907f4 100644 --- a/internal/database/convertor_convert.go +++ b/internal/database/convertor_convert.go @@ -3,7 +3,6 @@ package database import ( "errors" "fmt" - "github.com/turbot/pipe-fittings/v2/utils" "log" "log/slog" "os" @@ -11,71 +10,70 @@ import ( "strings" "time" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/marcboeker/go-duckdb/v2" "github.com/turbot/tailpipe-plugin-sdk/table" ) -func (w *Converter) processChunks(chunksToProcess []int32) { +// process all available chunks +// this is called when a chunk is added but will continue processing any further chunks added while we were processing +func (w *Converter) processAllChunks() { // note we ALREADY HAVE THE PROCESS LOCK - be sure to release it when we are done defer w.processLock.Unlock() + // so we have the process lock AND the schedule lock + // move the scheduled chunks to the chunks to process + // (scheduledChunks may be empty, in which case we will break out of the loop) + chunksToProcess := w.getChunksToProcess() for len(chunksToProcess) > 0 { - // build a list of filenames to process - filenamesToProcess, err := w.chunkNumbersToFilenames(chunksToProcess) + err := w.processChunks(chunksToProcess) if err != nil { - // failed to convert these files - decrement the wait group - w.wg.Add(len(filenamesToProcess) * -1) - - // TODO #DL re-add error handling - // https://github.com/turbot/tailpipe/issues/480 slog.Error("Error processing chunks", "error", err) - // store the failed conversion - //w.failedConversions = append(w.failedConversions, failedConversion{ - // filenames: filenamesToProcess, - // error: err, - //}, - //) - // just carry on + // call add job errors and carry on + w.addJobErrors(err) } - - // execute conversion query for the chunks - err = w.insertBatchIntoDuckLake(filenamesToProcess) - if err != nil { - // TODO #DL re-add error handling - // https://github.com/turbot/tailpipe/issues/480 - - // NOTE: the wait group will already have been decremented by insertBatchIntoDuckLake - // so we do not need to decrement it again here - - slog.Error("Error processing chunk", "filenames", filenamesToProcess, "error", err) - // store the failed conversion - //w.failedConversions = append(w.failedConversions, failedConversion{ - // filenames: filenamesToProcess, - // error: err, - //}, - //) - // just carry on - } - // delete the files after processing - for _, filename := range filenamesToProcess { - if err := os.Remove(filename); err != nil { - slog.Error("Failed to delete file after processing", "file", filename, "error", err) - } - } - - // now determine if there are more chunks to process - w.scheduleLock.Lock() - - // now get next chunks to process + //- get next batch of chunks chunksToProcess = w.getChunksToProcess() - - w.scheduleLock.Unlock() } // if we get here, we have processed all scheduled chunks (but more may come later log.Print("BatchProcessor: all scheduled chunks processed for execution") } +// process a batch of chunks +// Note whether successful of not, this decrements w.wg by the chunk count on return +func (w *Converter) processChunks(chunksToProcess []int32) error { + // decrement the wait group by the number of chunks processed + defer func() { + w.wg.Add(len(chunksToProcess) * -1) + }() + + // build a list of filenames to process + filenamesToProcess, err := w.chunkNumbersToFilenames(chunksToProcess) + if err != nil { + slog.Error("chunkNumbersToFilenames failed") + // chunkNumbersToFilenames returns a conversionError + return err + } + + // execute conversion query for the chunks + // (insertBatchIntoDuckLake will return a coinversionError) + err = w.insertBatchIntoDuckLake(filenamesToProcess) + // delete the files after processing (successful or otherwise) - we will just return err + for _, filename := range filenamesToProcess { + if deleteErr := os.Remove(filename); deleteErr != nil { + slog.Error("Failed to delete file after processing", "file", filename, "error", err) + // give conversion error precedence + if err == nil { + err = deleteErr + } + } + } + // return error (if any) + return err +} + func (w *Converter) chunkNumbersToFilenames(chunks []int32) ([]string, error) { var filenames = make([]string, len(chunks)) var missingFiles []string @@ -91,16 +89,18 @@ func (w *Converter) chunkNumbersToFilenames(chunks []int32) ([]string, error) { filenames[i] = escapedPath } if len(missingFiles) > 0 { - return filenames, NewConversionError(fmt.Errorf("%s not found", utils.Pluralize("file", len(missingFiles))), 0, missingFiles...) + // raise conversion error for the missing files - we do now know the row count so pass zero + return filenames, NewConversionError(fmt.Errorf("%s not found", + utils.Pluralize("file", len(missingFiles))), + 0, + missingFiles...) } return filenames, nil } -func (w *Converter) insertBatchIntoDuckLake(filenames []string) error { +func (w *Converter) insertBatchIntoDuckLake(filenames []string) (err error) { t := time.Now() - // ensure we signal the converter when we are done - defer w.wg.Add(len(filenames) * -1) // copy the data from the jsonl file to a temp table if err := w.copyChunkToTempTable(filenames); err != nil { @@ -110,33 +110,33 @@ func (w *Converter) insertBatchIntoDuckLake(filenames []string) error { tempTime := time.Now() - // TODO #DL re-add validation - // https://github.com/turbot/tailpipe/issues/479 - // now validate the data - //if validateRowsError := w.validateRows(jsonlFilePath); validateRowsError != nil { - // // if the error is NOT RowValidationError, just return it - // if !errors.Is(validateRowsError, &RowValidationError{}) { - // return handleConversionError(validateRowsError, jsonlFilePath) - // } - // - // // so it IS a row validation error - the invalid rows will have been removed from the temp table - // // - process the rest of the chunk - // // ensure that we return the row validation error, merged with any other error we receive - // defer func() { - // if err == nil { - // err = validateRowsError - // } else { - // var conversionError *ConversionError - // if errors.As(validateRowsError, &conversionError) { - // // we have a conversion error - we need to set the row count to 0 - // // so we can report the error - // conversionError.Merge(err) - // } - // err = conversionError - // } - // }() - //} + validateRowsError := w.validateRows(filenames) + if validateRowsError != nil { + // if the error is NOT RowValidationError, just return it + // (if it is a validation error, we have special handling) + if !errors.Is(validateRowsError, &RowValidationError{}) { + return validateRowsError + } + + // so it IS a row validation error - the invalid rows will have been removed from the temp table + // - process the rest of the chunk + // ensure that we return the row validation error, merged with any other error we receive + defer func() { + if err == nil { + err = validateRowsError + } else { + // so we have an error (aside from the any validation error) + // convert the validation error to a conversion error (which will be wrapping the validation error + var conversionError *ConversionError + // we expect this will always pass + if errors.As(validateRowsError, &conversionError) { + conversionError.Merge(err) + } + err = conversionError + } + }() + } slog.Debug("about to insert rows into ducklake table") @@ -162,10 +162,14 @@ func (w *Converter) insertBatchIntoDuckLake(filenames []string) error { func (w *Converter) copyChunkToTempTable(jsonlFilePaths []string) error { var queryBuilder strings.Builder + // Check for empty file paths + if len(jsonlFilePaths) == 0 { + return fmt.Errorf("no file paths provided") + } + // Create SQL array of file paths var fileSQL string if len(jsonlFilePaths) == 1 { - fileSQL = fmt.Sprintf("'%s'", jsonlFilePaths[0]) } else { // For multiple files, create a properly quoted array @@ -192,12 +196,14 @@ create temp table temp_data as `, selectQuery)) _, err := w.db.Exec(queryBuilder.String()) - if err != nil { - // if the error is a schema change error, determine whether the schema of these chunk is - // different to the inferred schema (pass the first json file) - return w.handleSchemaChangeError(err, jsonlFilePaths[0]) - } - return nil + // TODO KAI think about schema change + //if err != nil { + // // if the error is a schema change error, determine whether the schema of these chunk is + // // different to the inferred schema (pass the first json file) + // return w.handleSchemaChangeError(err, jsonlFilePaths...) + //} + + return err } // insertIntoDucklakeForBatch writes a batch of rows from the temp_data table to the specified target DuckDB table. @@ -238,22 +244,23 @@ func (w *Converter) insertIntoDucklake(targetTable string) (int64, error) { return insertedRowCount, nil } -// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema -// infer the schema of this chunk and compare - if they are different, return that in an error -func (w *Converter) handleSchemaChangeError(err error, jsonlFilePath string) error { - schemaChangeErr := w.detectSchemaChange(jsonlFilePath) - if schemaChangeErr != nil { - // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error - var e = &SchemaChangeError{} - if errors.As(schemaChangeErr, &e) { - // update err and fall through to handleConversionError - this wraps the error with additional row count info - err = e - } - } - - // just return the original error, wrapped with the row count - return handleConversionError(err, jsonlFilePath) -} +// TODO kai think about ducklake schema change detection +// +//// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema +//// infer the schema of this chunk and compare - if they are different, return that in an error +//func (w *Converter) handleSchemaChangeError(err error, jsonlFilePath ...string) error { +// schemaChangeErr := w.detectSchemaChange(jsonlFilePath) +// if schemaChangeErr != nil { +// // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error +// var e = &SchemaChangeError{} +// if errors.As(schemaChangeErr, &e) { +// // update err and fall through to handleConversionError - this wraps the error with additional row count info +// err = e +// } +// } +// +// // just return the original error, wrapped with the row count +//} // conversionRanOutOfMemory checks if the error is an out-of-memory error from DuckDB func conversionRanOutOfMemory(err error) bool { diff --git a/internal/database/convertor_schema.go b/internal/database/convertor_schema.go index 929d0795..c7186e9e 100644 --- a/internal/database/convertor_schema.go +++ b/internal/database/convertor_schema.go @@ -132,39 +132,40 @@ func (w *Converter) inferSchemaForJSONLFileWithDescribe(db *DuckDb, filePath str return res, nil } -func (w *Converter) detectSchemaChange(filePath string) error { - inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) - if err != nil { - return fmt.Errorf("failed to infer schema from JSON file: %w", err) - } - // the conversion schema is the full schema for the table that we have alreadf inferred - conversionSchemaMap := w.conversionSchema.AsMap() - // the table schema is the (possibly partial) schema which was defined in config - we use this to exclude columns - // which have a type specified - tableSchemaMap := w.tableSchema.AsMap() - // Compare the inferred schema with the existing conversionSchema - var changedColumns []ColumnSchemaChange - for _, col := range inferredChunksSchema.Columns { - // if the table schema definition specifies a type for this column, ignore the columns (as we will use the defined type) - // we are only interested in a type change if the column is not defined in the table schema - if columnDef, ok := tableSchemaMap[col.ColumnName]; ok { - if columnDef.Type != "" { - // if the column is defined in the table schema, ignore it - continue - } - } - - existingCol, exists := conversionSchemaMap[col.SourceName] - if exists && col.Type != existingCol.Type { - changedColumns = append(changedColumns, ColumnSchemaChange{ - Name: col.SourceName, - OldType: existingCol.Type, - NewType: col.Type, - }) - } - } - if len(changedColumns) > 0 { - return &SchemaChangeError{ChangedColumns: changedColumns} - } - return nil -} +// todo KAI think about schema change +//func (w *Converter) detectSchemaChange(filePath string) error { +// inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) +// if err != nil { +// return fmt.Errorf("failed to infer schema from JSON file: %w", err) +// } +// // the conversion schema is the full schema for the table that we have alreadf inferred +// conversionSchemaMap := w.conversionSchema.AsMap() +// // the table schema is the (possibly partial) schema which was defined in config - we use this to exclude columns +// // which have a type specified +// tableSchemaMap := w.tableSchema.AsMap() +// // Compare the inferred schema with the existing conversionSchema +// var changedColumns []ColumnSchemaChange +// for _, col := range inferredChunksSchema.Columns { +// // if the table schema definition specifies a type for this column, ignore the columns (as we will use the defined type) +// // we are only interested in a type change if the column is not defined in the table schema +// if columnDef, ok := tableSchemaMap[col.ColumnName]; ok { +// if columnDef.Type != "" { +// // if the column is defined in the table schema, ignore it +// continue +// } +// } +// +// existingCol, exists := conversionSchemaMap[col.SourceName] +// if exists && col.Type != existingCol.Type { +// changedColumns = append(changedColumns, ColumnSchemaChange{ +// Name: col.SourceName, +// OldType: existingCol.Type, +// NewType: col.Type, +// }) +// } +// } +// if len(changedColumns) > 0 { +// return &SchemaChangeError{ChangedColumns: changedColumns} +// } +// return nil +//} diff --git a/internal/database/convertor_validate.go b/internal/database/convertor_validate.go index d2f4d3f1..ef6aadd1 100644 --- a/internal/database/convertor_validate.go +++ b/internal/database/convertor_validate.go @@ -5,11 +5,9 @@ import ( "strings" ) -// validateRows copies the data from the given select query to a temp table and validates required fields are non null +// validateRows validates required fields are non null // it also validates that the schema of the chunk is the same as the inferred schema and if it is not, reports a useful error // the query count of invalid rows and a list of null fields -// -//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 func (w *Converter) validateRows(jsonlFilePaths []string) error { // build array of required columns to validate var requiredColumns []string @@ -29,7 +27,8 @@ func (w *Converter) validateRows(jsonlFilePaths []string) error { err := row.Scan(&failedRowCount, &columnsWithNullsInterface) if err != nil { - return w.handleSchemaChangeError(err, jsonlFilePaths[0]) + // todo kai think about schema change + return handleConversionError("row validation query failed", err, jsonlFilePaths...) } if failedRowCount == 0 { @@ -40,7 +39,7 @@ func (w *Converter) validateRows(jsonlFilePaths []string) error { // delete invalid rows from the temp table if err := w.deleteInvalidRows(requiredColumns); err != nil { // failed to delete invalid rows - return an error - err := handleConversionError(err, jsonlFilePaths...) + err := handleConversionError("failed to delete invalid rows from temp table", err, jsonlFilePaths...) return err } @@ -53,14 +52,13 @@ func (w *Converter) validateRows(jsonlFilePaths []string) error { } // we have a failure - return an error with details about which columns had nulls + // wrap a row validation error inside a conversion error return NewConversionError(NewRowValidationError(failedRowCount, columnsWithNulls), failedRowCount, jsonlFilePaths...) } // buildValidationQuery builds a query to copy the data from the select query to a temp table // it then validates that the required columns are not null, removing invalid rows and returning // the count of invalid rows and the columns with nulls -// -//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 func (w *Converter) buildValidationQuery(requiredColumns []string) string { var queryBuilder strings.Builder @@ -93,8 +91,6 @@ from (`) } // buildNullCheckQuery builds a WHERE clause to check for null values in the specified columns -// -//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 func (w *Converter) buildNullCheckQuery(requiredColumns []string) string { // build a slice of null check conditions @@ -106,8 +102,6 @@ func (w *Converter) buildNullCheckQuery(requiredColumns []string) string { } // deleteInvalidRows removes rows with null values in the specified columns from the temp table -// -//nolint:unused // TODO re-add validation https://github.com/turbot/tailpipe/issues/479 func (w *Converter) deleteInvalidRows(requiredColumns []string) error { whereClause := w.buildNullCheckQuery(requiredColumns) query := fmt.Sprintf("delete from temp_data where %s;", whereClause) diff --git a/internal/database/read_json_query.go b/internal/database/read_json_query.go index cf7e4c78..0ff2e1cc 100644 --- a/internal/database/read_json_query.go +++ b/internal/database/read_json_query.go @@ -26,7 +26,9 @@ func buildReadJsonQueryFormat(conversionSchema *schema.ConversionSchema, partiti var tpTimestampMapped bool // first build the select clauses - use the table def columns - var selectClauses []string + var selectClauses = []string{ + "row_number() over () as row_id", // add a row_id column to use with validation + } for _, column := range conversionSchema.Columns { var selectClause string switch column.ColumnName { diff --git a/internal/database/row_validation_error.go b/internal/database/row_validation_error.go index e9f5f4b6..e76b0cd7 100644 --- a/internal/database/row_validation_error.go +++ b/internal/database/row_validation_error.go @@ -2,6 +2,7 @@ package database import ( "fmt" + "strings" "github.com/turbot/pipe-fittings/v2/utils" ) @@ -19,7 +20,7 @@ func NewRowValidationError(failedRows int64, nullColumns []string) *RowValidatio } func (e *RowValidationError) Error() string { - return fmt.Sprintf("%d %s failed validation - found null values in %d %s: %s", e.failedRows, utils.Pluralize("row", int(e.failedRows)), len(e.nullColumns), utils.Pluralize("column", len(e.nullColumns)), e.nullColumns) + return fmt.Sprintf("%d %s failed validation - found null values in %d %s: %s", e.failedRows, utils.Pluralize("row", int(e.failedRows)), len(e.nullColumns), utils.Pluralize("column", len(e.nullColumns)), strings.Join(e.nullColumns, ", ")) } // Is implements the errors.Is interface to support error comparison From cce55ff2967c11f08c2a6170ab72bb30de412ad3 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Wed, 10 Sep 2025 12:48:54 +0100 Subject: [PATCH 31/61] fix: fix time parsing in expirePrevSnapshots. Tidy up unused S3 code Close #520 --- cmd/collect.go | 54 +++++++--- internal/collector/collector.go | 1 - internal/collector/collector_synthetic.go | 123 +++++++++++++++++----- internal/database/cleanup.go | 34 +++--- 4 files changed, 146 insertions(+), 66 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 13d7eaac..0a574192 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/turbot/tailpipe/internal/database" "log/slog" "os" "strconv" @@ -25,6 +24,7 @@ import ( "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" ) @@ -235,30 +235,45 @@ func getPartitions(args []string) ([]*config.Partition, error) { return partitions, nil } +// getSyntheticPartition parses a synthetic partition specification string and creates a test partition configuration. +// This function enables testing and performance benchmarking by generating dummy data instead of collecting from real sources. +// +// Synthetic partition format: synthetic_cols_rows_chunk_ms +// Example: "synthetic_50cols_2000000rows_10000chunk_100ms" +// - 50cols: Number of columns to generate in the synthetic table +// - 2000000rows: Total number of rows to generate +// - 10000chunk: Number of rows per chunk (affects memory usage and processing) +// - 100ms: Delivery interval between chunks (simulates real-time data collection) +// +// The function validates the format and numeric values, returning a properly configured Partition +// with SyntheticMetadata that will be used by the collector to generate test data. +// +// Returns: +// - *config.Partition: The configured synthetic partition if parsing succeeds +// - bool: true if the argument was a valid synthetic partition, false otherwise func getSyntheticPartition(arg string) (*config.Partition, bool) { - // synthetic partitions are of form synthetic_50cols_2000000rows_10000chunk_100ms - // determine if this partition is synthetic and if so try to parse the params - - // Check if this is a synthetic partition + // Check if this is a synthetic partition by looking for the "synthetic_" prefix if !strings.HasPrefix(arg, "synthetic_") { return nil, false } - // Parse the synthetic partition parameters - // Format: synthetic_cols_rows_chunk_ms + // Parse the synthetic partition parameters by splitting on underscores + // Expected format: synthetic_cols_rows_chunk_ms parts := strings.Split(arg, "_") if len(parts) != 5 { - // Invalid format, not a synthetic partition + // Invalid format - synthetic partitions must have exactly 5 parts slog.Debug("Synthetic partition parsing failed: invalid format", "arg", arg, "parts", len(parts), "expected", 5) return nil, false } - // Extract and parse the numeric values + // Extract and parse the numeric values from each part + // Remove the suffix to get just the numeric value colsStr := strings.TrimSuffix(parts[1], "cols") rowsStr := strings.TrimSuffix(parts[2], "rows") chunkStr := strings.TrimSuffix(parts[3], "chunk") intervalStr := strings.TrimSuffix(parts[4], "ms") + // Parse columns count - determines how many columns the synthetic table will have cols, err := strconv.Atoi(colsStr) if err != nil { // Invalid columns value, not a synthetic partition @@ -266,6 +281,7 @@ func getSyntheticPartition(arg string) (*config.Partition, bool) { return nil, false } + // Parse rows count - total number of rows to generate rows, err := strconv.Atoi(rowsStr) if err != nil { // Invalid rows value, not a synthetic partition @@ -273,6 +289,7 @@ func getSyntheticPartition(arg string) (*config.Partition, bool) { return nil, false } + // Parse chunk size - number of rows per chunk (affects memory usage and processing efficiency) chunk, err := strconv.Atoi(chunkStr) if err != nil { // Invalid chunk value, not a synthetic partition @@ -280,6 +297,7 @@ func getSyntheticPartition(arg string) (*config.Partition, bool) { return nil, false } + // Parse delivery interval - milliseconds between chunk deliveries (simulates real-time data flow) interval, err := strconv.Atoi(intervalStr) if err != nil { // Invalid interval value, not a synthetic partition @@ -287,7 +305,7 @@ func getSyntheticPartition(arg string) (*config.Partition, bool) { return nil, false } - // Validate the parsed values + // Validate the parsed values - all must be positive integers if cols <= 0 || rows <= 0 || chunk <= 0 || interval <= 0 { // Invalid values, not a synthetic partition slog.Debug("Synthetic partition parsing failed: invalid values", "arg", arg, "cols", cols, "rows", rows, "chunk", chunk, "interval", interval) @@ -295,24 +313,26 @@ func getSyntheticPartition(arg string) (*config.Partition, bool) { } // Create a synthetic partition with proper HCL block structure + // This mimics the structure that would be created from a real HCL configuration file block := &hcl.Block{ Type: "partition", Labels: []string{"synthetic", arg}, } + // Create the partition configuration with synthetic metadata partition := &config.Partition{ HclResourceImpl: modconfig.NewHclResourceImpl(block, fmt.Sprintf("partition.synthetic.%s", arg)), - TableName: "synthetic", - TpIndexColumn: "'default'", + TableName: "synthetic", // All synthetic partitions use the "synthetic" table name + TpIndexColumn: "'default'", // Use a default index column for synthetic data SyntheticMetadata: &config.SyntheticMetadata{ - Columns: cols, - Rows: rows, - ChunkSize: chunk, - DeliveryIntervalMs: interval, + Columns: cols, // Number of columns to generate + Rows: rows, // Total number of rows to generate + ChunkSize: chunk, // Rows per chunk + DeliveryIntervalMs: interval, // Milliseconds between chunk deliveries }, } - // Set the unqualified name + // Set the unqualified name for the partition (used in logging and identification) partition.UnqualifiedName = fmt.Sprintf("%s.%s", partition.TableName, partition.ShortName) slog.Debug("Synthetic partition parsed successfully", "arg", arg, "columns", cols, "rows", rows, "chunkSize", chunk, "deliveryIntervalMs", interval) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 6730099f..f73474ec 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -270,7 +270,6 @@ func (c *Collector) Compact(ctx context.Context) error { // NOTE: we DO NOT reindex when compacting after collection reindex := false - err := database.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, reindex, &partitionPattern) if err != nil { diff --git a/internal/collector/collector_synthetic.go b/internal/collector/collector_synthetic.go index b6435deb..57527ef7 100644 --- a/internal/collector/collector_synthetic.go +++ b/internal/collector/collector_synthetic.go @@ -23,32 +23,59 @@ import ( "github.com/turbot/tailpipe/internal/plugin" ) +// doCollectSynthetic initiates synthetic data collection for testing and performance benchmarking. +// This function simulates the data collection process by generating dummy data instead of collecting from real sources. +// +// The function: +// 1. Creates an execution context to track the synthetic collection process +// 2. Builds a synthetic schema based on the number of columns specified in the partition metadata +// 3. Starts a background goroutine to generate and write synthetic data in chunks +// 4. Returns a CollectResponse that mimics what a real plugin would return +// +// This enables testing of the entire data collection pipeline without requiring actual data sources, +// making it useful for performance testing, load testing, and development/debugging scenarios. +// +// Parameters: +// - ctx: Context for cancellation and timeout handling +// - fromTime: Start time for the synthetic data (timestamps will be distributed across this range) +// - toTime: End time for the synthetic data +// - overwrite: Whether to overwrite existing data (not used in synthetic collection) +// +// Returns: +// - *plugin.CollectResponse: Response containing execution ID and schema information +// - error: Any error that occurred during initialization func (c *Collector) doCollectSynthetic(ctx context.Context, fromTime time.Time, toTime time.Time, overwrite bool) (*plugin.CollectResponse, error) { - // create the execution - // NOTE: create _before_ calling the plugin to ensure it is ready to receive the started event + // Create the execution context to track the synthetic collection process + // This must be created before starting the collection goroutine to ensure proper event handling c.execution = &execution{ - id: "synthetic", - partition: c.partition.UnqualifiedName, - table: c.partition.TableName, - plugin: "synthetic", - state: ExecutionState_PENDING, - completionChan: make(chan error, 1), + id: "synthetic", // Use "synthetic" as the execution ID + partition: c.partition.UnqualifiedName, // Full partition name for identification + table: c.partition.TableName, // Table name (always "synthetic" for synthetic partitions) + plugin: "synthetic", // Plugin name for logging and identification + state: ExecutionState_PENDING, // Initial state before collection starts + completionChan: make(chan error, 1), // Channel to signal completion or errors } + // Build the synthetic schema based on the number of columns specified in the partition metadata + // This creates a table schema with the specified number of columns of various types schema := buildsyntheticchema(c.partition.SyntheticMetadata.Columns) - // start a thread to fake the collection process + + // Start a background goroutine to perform the actual synthetic data generation + // This simulates the asynchronous nature of real data collection go c.collectSynthetic(ctx, schema, fromTime, toTime) - // build a collect response + // Build a collect response that mimics what a real plugin would return + // This allows the synthetic collection to integrate seamlessly with the existing collection pipeline collectResponse := &plugin.CollectResponse{ - ExecutionId: c.execution.id, - Schema: schema, + ExecutionId: c.execution.id, // Use the execution ID for tracking + Schema: schema, // The generated synthetic schema FromTime: &row_source.ResolvedFromTime{ - Time: fromTime, - Source: "synthetic", + Time: fromTime, // Start time for the data collection + Source: "synthetic", // Source identifier for synthetic data }, } - // _now_ set the execution id + + // Update the execution ID to match the response (in case it was modified) c.execution.id = collectResponse.ExecutionId return collectResponse, nil } @@ -416,32 +443,58 @@ func buildsyntheticchema(columns int) *schema.TableSchema { return s } +// collectSynthetic generates synthetic data in chunks and writes it to JSONL files. +// This function runs in a background goroutine and simulates the data collection process +// by generating dummy data according to the synthetic partition specifications. +// +// The function: +// 1. Notifies that collection has started +// 2. Calculates timestamp intervals to distribute timestamps across the time range +// 3. Generates data in chunks according to the specified chunk size +// 4. Writes each chunk to a JSONL file using optimized concurrent writing +// 5. Respects the delivery interval to simulate real-time data flow +// 6. Sends progress events (chunk and status) to maintain the collection UI +// 7. Handles cancellation and error conditions gracefully +// 8. Notifies completion when all data has been generated +// +// Parameters: +// - ctx: Context for cancellation and timeout handling +// - tableSchema: The schema defining the structure of the synthetic data +// - fromTime: Start time for timestamp generation +// - toTime: End time for timestamp generation func (c *Collector) collectSynthetic(ctx context.Context, tableSchema *schema.TableSchema, fromTime time.Time, toTime time.Time) { metadata := c.partition.SyntheticMetadata - // set the execution state to started + // Set the execution state to started to indicate collection is in progress c.execution.state = ExecutionState_STARTED + // Notify that collection has started - this triggers the collection UI to show progress if err := c.Notify(ctx, &events.Started{ExecutionId: c.execution.id}); err != nil { slog.Error("failed to notify started event", "error", err) c.execution.completionChan <- fmt.Errorf("failed to notify started event: %w", err) return } - var chunkIdx int32 = 0 - var totalRowsProcessed int64 = 0 + var chunkIdx int32 = 0 // Track the current chunk number + var totalRowsProcessed int64 = 0 // Track total rows processed for progress reporting - // Calculate timestamp interval based on fromTime, toTime, and total rows + // Calculate timestamp interval to distribute timestamps evenly across the time range + // This ensures synthetic data has realistic timestamp progression var timestampInterval time.Duration if metadata.Rows > 1 { + // Distribute timestamps evenly between fromTime and toTime timestampInterval = toTime.Sub(fromTime) / time.Duration(metadata.Rows-1) } else { + // Single row case - no interval needed timestampInterval = 0 } + // Generate data in chunks according to the specified chunk size + // This allows for memory-efficient processing of large datasets for rowCount := 0; rowCount < metadata.Rows; rowCount += metadata.ChunkSize { - t := time.Now() - // Check if context is cancelled + t := time.Now() // Track chunk processing time for delivery interval calculation + + // Check if context is cancelled - allows for graceful shutdown select { case <-ctx.Done(): c.execution.completionChan <- ctx.Err() @@ -449,30 +502,38 @@ func (c *Collector) collectSynthetic(ctx context.Context, tableSchema *schema.Ta default: } + // Calculate the number of rows for this chunk (may be less than chunk size for the last chunk) rows := int(math.Min(float64(metadata.Rows-rowCount), float64(metadata.ChunkSize))) - // write optimized chunk to JSONL file + // Generate filename for this chunk's JSONL file filename := table.ExecutionIdToJsonlFileName(c.execution.id, chunkIdx) filepath := filepath.Join(c.sourcePath, filename) - // write the chunk to JSONL file using optimized approach + // Write the chunk to JSONL file using optimized concurrent approach + // This generates synthetic data and writes it efficiently to disk if err := writeOptimizedChunkToJSONLConcurrent(filepath, tableSchema, rows, rowCount, c.partition, fromTime, timestampInterval); err != nil { c.execution.completionChan <- fmt.Errorf("error writing chunk to JSONL file: %w", err) return } - dur := time.Since(t) - // if this is less that deliver interval, wait for the remaining time + dur := time.Since(t) // Calculate how long this chunk took to process + + // Respect the delivery interval to simulate real-time data flow + // If processing was faster than the interval, wait for the remaining time if metadata.DeliveryIntervalMs > 0 && dur < time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond { slog.Debug("Waiting for delivery interval", "duration", dur, "expected", time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond) select { case <-time.After(time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond - dur): + // Wait for the remaining time case <-ctx.Done(): + // Context was cancelled during wait c.execution.completionChan <- ctx.Err() return } } - // send chunk event to the plugin + + // Send chunk event to notify that a chunk has been completed + // This updates the collection UI and allows other components to process the chunk chunkEvent := &events.Chunk{ExecutionId: c.execution.id, ChunkNumber: chunkIdx} if err := c.Notify(ctx, chunkEvent); err != nil { slog.Error("failed to notify chunk event", "error", err) @@ -480,6 +541,8 @@ func (c *Collector) collectSynthetic(ctx context.Context, tableSchema *schema.Ta return } + // Update total rows processed and send status event + // This provides progress information to the collection UI totalRowsProcessed += int64(rows) statusEvent := &events.Status{ExecutionId: c.execution.id, RowsReceived: totalRowsProcessed, RowsEnriched: totalRowsProcessed} if err := c.Notify(ctx, statusEvent); err != nil { @@ -488,17 +551,19 @@ func (c *Collector) collectSynthetic(ctx context.Context, tableSchema *schema.Ta return } - chunkIdx++ + chunkIdx++ // Move to next chunk } - // Send completion event + // Send completion event to indicate all data has been generated + // This triggers final processing and updates the collection UI if err := c.Notify(ctx, events.NewCompletedEvent(c.execution.id, int64(metadata.Rows), chunkIdx, nil)); err != nil { slog.Error("failed to notify completed event", "error", err) c.execution.completionChan <- fmt.Errorf("failed to notify completed event: %w", err) return } - // Signal completion + // Signal completion by sending nil to the completion channel + // This allows the main collection process to know that synthetic data generation is complete c.execution.completionChan <- nil } diff --git a/internal/database/cleanup.go b/internal/database/cleanup.go index a34743f7..fa774bfb 100644 --- a/internal/database/cleanup.go +++ b/internal/database/cleanup.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "log/slog" - "os" - "strings" "time" "github.com/turbot/pipe-fittings/v2/constants" @@ -14,18 +12,6 @@ import ( // DeletePartition deletes data for the specified partition and date range from the given Ducklake connected database. func DeletePartition(ctx context.Context, partition *config.Partition, from, to time.Time, db *DuckDb) (rowCount int, err error) { - // TODO #DL https://github.com/turbot/tailpipe/issues/505 - // if we are using s3 do not delete for now as this does not work at present (need explicit S3 support I think) - // remove before release https://github.com/turbot/tailpipe/issues/520 - if envDir := os.Getenv("TAILPIPE_DATA_DIR"); strings.HasPrefix(envDir, "s3") { - slog.Warn("Skipping partition deletion for S3 data source", - "partition", partition.TableName, - "from", from, - "to", to, - ) - return 0, nil // return 0 rows affected, not an error - } - // First check if the table exists using DuckLake metadata tableExistsQuery := fmt.Sprintf(`select exists (select 1 from %s.ducklake_table where table_name = ?)`, constants.DuckLakeMetadataCatalog) var tableExists bool @@ -99,18 +85,28 @@ func expirePrevSnapshots(ctx context.Context, db *DuckDb) error { // Parse the snapshot time // NOTE: rather than cast as timestamp, we read as a string then remove any timezone component // This is because of the dubious behaviour of ducklake_expire_snapshots described below - parsedTime, err := time.Parse("2006-01-02 15:04:05.999-07", latestTimestamp) - if err != nil { - if err != nil { - return fmt.Errorf("failed to parse snapshot time '%s': %w", latestTimestamp, err) + // try various formats + formats := []string{ + "2006-01-02 15:04:05.999-07:00", // +05:30 + "2006-01-02 15:04:05.999-07", // +01 + "2006-01-02 15:04:05.999", // no timezone + } + var parsedTime time.Time + for _, format := range formats { + parsedTime, err = time.Parse(format, latestTimestamp) + if err == nil { + break } } + if err != nil { + return fmt.Errorf("failed to parse snapshot time '%s': %w", latestTimestamp, err) + } + // format the time // TODO Note: ducklake_expire_snapshots expects a local time without timezone, // i.e if the time is '2025-08-26 13:25:10.365 +0100', we should pass '2025-08-26 13:25:10.365' // We need to raise a ducklake issue formattedTime := parsedTime.Format("2006-01-02 15:04:05.000") - slog.Debug("Latest snapshot timestamp", "timestamp", latestTimestamp) // 2) expire all snapshots older than the latest one From 0bd212ee022096eda859a4b0df9a0720b6b8c7ec Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Wed, 10 Sep 2025 21:54:39 +0530 Subject: [PATCH 32/61] Add comment and retain nil-check for installation to prevent nil-pointer dereference in newly developed local plugins. Closes #507 --- internal/plugin/installation_actions.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/plugin/installation_actions.go b/internal/plugin/installation_actions.go index 72f0adab..3c413917 100644 --- a/internal/plugin/installation_actions.go +++ b/internal/plugin/installation_actions.go @@ -116,7 +116,8 @@ func List(ctx context.Context, pluginVersions map[string]*versionfile.InstalledV // detectLocalPlugin returns true if the modTime of the `pluginBinary` is after the installation date as recorded in the installation data // this may happen when a plugin is installed from the registry, but is then compiled from source func detectLocalPlugin(installation *versionfile.InstalledVersion, pluginBinary string) bool { - // TODO this should no longer be necessary as we now have a "local" version number in the versions file? https://github.com/turbot/tailpipe/issues/507 + // Guard: newly developed local plugins may not have a versions entry yet, + // so installation can be nil. Keep this check to prevent a nil-pointer dereference. if installation == nil { return true } From c0c709e1d7ab5b242b6e7db26b7b6483e7f988eb Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Wed, 10 Sep 2025 21:55:34 +0530 Subject: [PATCH 33/61] Update error handling in getPartitions function. Closes #497 --- cmd/collect.go | 24 ++++++++++++++++++++++-- go.mod | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 0a574192..e6dc6b0f 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -228,13 +228,33 @@ func getPartitions(args []string) ([]*config.Partition, error) { } if len(errorList) > 0 { - // TODO #errors better formating/error message https://github.com/turbot/tailpipe/issues/497 - return nil, errors.Join(errorList...) + // Return a well-formatted multi-error with a count and indented bullet list + return nil, formatErrorsWithCount(errorList) } return partitions, nil } +// formatErrorsWithCount returns an error summarizing a list of errors with a count and indented lines +func formatErrorsWithCount(errs []error) error { + if len(errs) == 0 { + return nil + } + if len(errs) == 1 { + return errs[0] + } + + var b strings.Builder + b.WriteString(fmt.Sprintf("%d errors:\n", len(errs))) + for i, e := range errs { + b.WriteString(fmt.Sprintf(" %s", e.Error())) + if i < len(errs)-1 { + b.WriteString("\n") + } + } + return errors.New(b.String()) +} + // getSyntheticPartition parses a synthetic partition specification string and creates a test partition configuration. // This function enables testing and performance benchmarking by generating dummy data instead of collecting from real sources. // diff --git a/go.mod b/go.mod index a1b530e7..fcabd309 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,6 @@ require ( github.com/marcboeker/go-duckdb/v2 v2.3.5 github.com/thediveo/enumflag/v2 v2.0.5 github.com/turbot/tailpipe-plugin-core v0.2.10 - golang.org/x/sync v0.16.0 golang.org/x/text v0.27.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 @@ -235,6 +234,7 @@ require ( golang.org/x/mod v0.26.0 // indirect golang.org/x/net v0.42.0 // indirect golang.org/x/oauth2 v0.29.0 // indirect + golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/term v0.33.0 // indirect golang.org/x/time v0.11.0 // indirect From bc0ad1d793c7ea12956589032b9e4a53a79f86ea Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Thu, 11 Sep 2025 11:17:53 +0100 Subject: [PATCH 34/61] reenable schemas change detection --- cmd/compact.go | 3 - cmd/plugin.go | 7 +- cmd/root.go | 1 - internal/config/connection.go | 44 -------- internal/config/table.go | 2 +- internal/database/cleanup.go | 3 +- internal/database/convertor.go | 7 +- internal/database/convertor_convert.go | 51 +++++----- internal/database/convertor_ducklake.go | 127 +++++++++++++----------- internal/database/convertor_schema.go | 73 +++++++------- internal/database/convertor_validate.go | 1 - internal/database/schema_comparison.go | 7 +- 12 files changed, 143 insertions(+), 183 deletions(-) diff --git a/cmd/compact.go b/cmd/compact.go index e13bdc1b..a223f936 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -23,9 +23,6 @@ import ( "golang.org/x/exp/maps" ) -// TODO #DL update docs - no longer support compacting single partition -// -// https://github.com/turbot/tailpipe/issues/474 func compactCmd() *cobra.Command { cmd := &cobra.Command{ Use: "compact [table|table.partition] [flags]", diff --git a/cmd/plugin.go b/cmd/plugin.go index 8332baba..8a5001bc 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -184,10 +184,9 @@ Examples: // Show plugin func pluginShowCmd() *cobra.Command { var cmd = &cobra.Command{ - Use: "show ", - Args: cobra.ExactArgs(1), - Run: runPluginShowCmd, - // TODO improve descriptions https://github.com/turbot/tailpipe/issues/111 + Use: "show ", + Args: cobra.ExactArgs(1), + Run: runPluginShowCmd, Short: "Show details of a plugin", Long: `Show the tables and sources provided by plugin`, } diff --git a/cmd/root.go b/cmd/root.go index 258b3a8b..a293be64 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -32,7 +32,6 @@ func rootCommand() *cobra.Command { rootCmd.SetVersionTemplate("Tailpipe v{{.Version}}\n") - // TODO #config this will not reflect changes to install-dir - do we need to default in a different way https://github.com/turbot/tailpipe/issues/112 defaultConfigPath := filepaths.EnsureConfigDir() cmdconfig. diff --git a/internal/config/connection.go b/internal/config/connection.go index 698dd3cf..400df1b6 100644 --- a/internal/config/connection.go +++ b/internal/config/connection.go @@ -69,47 +69,3 @@ func NewTailpipeConnection(block *hcl.Block, fullName string) (modconfig.HclReso c.UnqualifiedName = fmt.Sprintf("%s.%s", c.Plugin, c.ShortName) return c, nil } - -// TODO implement if needed https://github.com/turbot/tailpipe/issues/34 -// -//func CtyValueToConnection(value cty.Value) (_ *TailpipeConnection, err error) { -// defer func() { -// if r := recover(); r != nil { -// err = perr.BadRequestWithMessage("unable to decode connection: " + r.(string)) -// } -// }() -// -// // get the name, block type and range and use to construct a connection -// shortName := value.GetAttr("short_name").AsString() -// name := value.GetAttr("name").AsString() -// block := &hcl.Block{ -// Labels: []string{"connection", name}, -// } -// -// -// -// // now instantiate an empty connection of the correct type -// conn, err := NewTailpipeConnection(&hcl.Block{}, name) -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode connection: " + err.Error()) -// } -// -// // split the cty value into fields for ConnectionImpl and the derived connection, -// // (NOTE: exclude the 'env', 'type', 'resource_type' fields, which are manually added) -// baseValue, derivedValue, err := getKnownCtyFields(value, conn.GetConnectionImpl(), "env", "type", "resource_type") -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode connection: " + err.Error()) -// } -// // decode the base fields into the ConnectionImpl -// err = gocty.FromCtyValue(baseValue, conn.GetConnectionImpl()) -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode ConnectionImpl: " + err.Error()) -// } -// // decode remaining fields into the derived connection -// err = gocty.FromCtyValue(derivedValue, &conn) -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode connection: " + err.Error()) -// } -// -// return nil, nil -//} diff --git a/internal/config/table.go b/internal/config/table.go index b8a4e521..154967ef 100644 --- a/internal/config/table.go +++ b/internal/config/table.go @@ -21,7 +21,7 @@ type Table struct { // required to allow partial decoding Remain hcl.Body `hcl:",remain" json:"-"` - // the default format for this table (todo make a map keyed by source name?) + // the default format for this table DefaultSourceFormat *Format `hcl:"format" cty:"format"` Columns []Column `hcl:"column,block" cty:"columns"` diff --git a/internal/database/cleanup.go b/internal/database/cleanup.go index fa774bfb..5a189362 100644 --- a/internal/database/cleanup.go +++ b/internal/database/cleanup.go @@ -103,9 +103,8 @@ func expirePrevSnapshots(ctx context.Context, db *DuckDb) error { } // format the time - // TODO Note: ducklake_expire_snapshots expects a local time without timezone, + // Note: ducklake_expire_snapshots expects a local time without timezone, // i.e if the time is '2025-08-26 13:25:10.365 +0100', we should pass '2025-08-26 13:25:10.365' - // We need to raise a ducklake issue formattedTime := parsedTime.Format("2006-01-02 15:04:05.000") slog.Debug("Latest snapshot timestamp", "timestamp", latestTimestamp) diff --git a/internal/database/convertor.go b/internal/database/convertor.go index 6f11ba66..c34e7bcc 100644 --- a/internal/database/convertor.go +++ b/internal/database/convertor.go @@ -12,10 +12,6 @@ import ( "github.com/turbot/tailpipe/internal/config" ) -// TODO #DL -// - think about max memory https://github.com/turbot/tailpipe/issues/478 -// - validation https://github.com/turbot/tailpipe/issues/479 - const chunkBufferLength = 1000 // Converter struct executes all the conversions for a single collection @@ -148,7 +144,8 @@ func (w *Converter) getChunksToProcess() []int32 { w.scheduleLock.Lock() defer w.scheduleLock.Unlock() - // TODO #DL do we even need this https://github.com/turbot/tailpipe/issues/523 + // provide a mechanism to limit the max chunks we process at once + // a high value for this seems fine (it's possible we do not actually need a limit at all) const maxChunksToProcess = 2000 var chunksToProcess []int32 if len(w.scheduledChunks) > maxChunksToProcess { diff --git a/internal/database/convertor_convert.go b/internal/database/convertor_convert.go index 846907f4..3e53b389 100644 --- a/internal/database/convertor_convert.go +++ b/internal/database/convertor_convert.go @@ -196,14 +196,14 @@ create temp table temp_data as `, selectQuery)) _, err := w.db.Exec(queryBuilder.String()) - // TODO KAI think about schema change - //if err != nil { - // // if the error is a schema change error, determine whether the schema of these chunk is - // // different to the inferred schema (pass the first json file) - // return w.handleSchemaChangeError(err, jsonlFilePaths...) - //} + if err != nil { + // if the error is a schema change error, determine whether the schema of these chunks is + // different to the inferred schema + // w.handleSchemaChangeError either returns a schema change error or the original error + return w.handleSchemaChangeError(err, jsonlFilePaths...) + } - return err + return nil } // insertIntoDucklakeForBatch writes a batch of rows from the temp_data table to the specified target DuckDB table. @@ -244,23 +244,26 @@ func (w *Converter) insertIntoDucklake(targetTable string) (int64, error) { return insertedRowCount, nil } -// TODO kai think about ducklake schema change detection -// -//// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema -//// infer the schema of this chunk and compare - if they are different, return that in an error -//func (w *Converter) handleSchemaChangeError(err error, jsonlFilePath ...string) error { -// schemaChangeErr := w.detectSchemaChange(jsonlFilePath) -// if schemaChangeErr != nil { -// // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error -// var e = &SchemaChangeError{} -// if errors.As(schemaChangeErr, &e) { -// // update err and fall through to handleConversionError - this wraps the error with additional row count info -// err = e -// } -// } -// -// // just return the original error, wrapped with the row count -//} +// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema +// infer the schema of this chunk and compare - if they are different, return that in an error +func (w *Converter) handleSchemaChangeError(origError error, jsonlFilePaths ...string) error { + // check all files for a schema change error + for _, jsonlFilePath := range jsonlFilePaths { + err := w.detectSchemaChange(jsonlFilePath) + if err != nil { + // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error + // (ignore any other error - we will fall through to return original error) + var schemaChangeError = &SchemaChangeError{} + if errors.As(err, &schemaChangeError) { + // update err and fall through to handleConversionError - this wraps the error with additional row count info + return schemaChangeError + } + } + } + + // just return the original error + return origError +} // conversionRanOutOfMemory checks if the error is an out-of-memory error from DuckDB func conversionRanOutOfMemory(err error) bool { diff --git a/internal/database/convertor_ducklake.go b/internal/database/convertor_ducklake.go index 3ed7e892..bb80dd1c 100644 --- a/internal/database/convertor_ducklake.go +++ b/internal/database/convertor_ducklake.go @@ -1,9 +1,11 @@ package database import ( + "database/sql" "fmt" "strings" + "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/tailpipe-plugin-sdk/constants" "github.com/turbot/tailpipe-plugin-sdk/schema" ) @@ -106,62 +108,69 @@ func (w *Converter) buildStructDefinition(column *schema.ColumnSchema) string { return fmt.Sprintf("struct(%s)", strings.Join(fieldDefinitions, ", ")) } -// TODO #DL is this code needed - look at schema change detection -// https://github.com/turbot/tailpipe/issues/481 -//func (w *Converter) CheckTableSchema(db *sql.DB, tableName string, conversionSchema schema.ConversionSchema) (TableSchemaStatus, error) { -// // Check if table exists -// exists, err := w.tableExists(db, tableName) -// if err != nil { -// return TableSchemaStatus{}, err -// } -// -// if !exists { -// return TableSchemaStatus{}, nil -// } -// -// // Get existing schema -// existingSchema, err := w.getTableSchema(db, tableName) -// if err != nil { -// return TableSchemaStatus{}, fmt.Errorf("failed to retrieve schema: %w", err) -// } -// -// // Use constructor to create status from comparison -// diff := NewTableSchemaStatusFromComparison(existingSchema, conversionSchema) -// return diff, nil -//} -// -//func (w *Converter) tableExists(db *sql.DB, tableName string) (bool, error) { -// query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", tableName) -// var exists int -// if err := db.QueryRow(query).Scan(&exists); err != nil { -// return false, err -// } -// return exists == 1, nil -//} - -//func (w *Converter) getTableSchema(db *sql.DB, tableName string) (map[string]schema.ColumnSchema, error) { -// query := fmt.Sprintf("pragma table_info(%s);", tableName) -// rows, err := db.Query(query) -// if err != nil { -// return nil, err -// } -// defer rows.Close() -// -// schemaMap := make(map[string]schema.ColumnSchema) -// for rows.Next() { -// var name, dataType string -// var notNull, pk int -// var dfltValue sql.NullString -// -// if err := rows.Scan(&name, &dataType, ¬Null, &dfltValue, &pk); err != nil { -// return nil, err -// } -// -// schemaMap[name] = schema.ColumnSchema{ -// ColumnName: name, -// Type: dataType, -// } -// } -// -// return schemaMap, nil -//} +// CheckTableSchema checks if the specified table exists in the DuckDB database and compares its schema with the +// provided schema. +// it returns a TableSchemaStatus indicating whether the table exists, whether the schema matches, and any differences. +// THis is not used at present but will be used when we implement ducklake schema evolution handling +func (w *Converter) CheckTableSchema(db *sql.DB, tableName string, conversionSchema schema.ConversionSchema) (TableSchemaStatus, error) { + // Check if table exists + exists, err := w.tableExists(db, tableName) + if err != nil { + return TableSchemaStatus{}, err + } + + if !exists { + return TableSchemaStatus{}, nil + } + + // Get existing schema + existingSchema, err := w.getTableSchema(db, tableName) + if err != nil { + return TableSchemaStatus{}, fmt.Errorf("failed to retrieve schema: %w", err) + } + + // Use constructor to create status from comparison + diff := NewTableSchemaStatusFromComparison(existingSchema, conversionSchema) + return diff, nil +} + +func (w *Converter) tableExists(db *sql.DB, tableName string) (bool, error) { + sanitizedTableName, err := backend.SanitizeDuckDBIdentifier(tableName) + if err != nil { + return false, fmt.Errorf("invalid table name %s: %w", tableName, err) + } + //nolint:gosec // table name is sanitized + query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", sanitizedTableName) + var exists int + if err := db.QueryRow(query).Scan(&exists); err != nil { + return false, err + } + return exists == 1, nil +} + +func (w *Converter) getTableSchema(db *sql.DB, tableName string) (map[string]schema.ColumnSchema, error) { + query := fmt.Sprintf("pragma table_info(%s);", tableName) + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + schemaMap := make(map[string]schema.ColumnSchema) + for rows.Next() { + var name, dataType string + var notNull, pk int + var dfltValue sql.NullString + + if err := rows.Scan(&name, &dataType, ¬Null, &dfltValue, &pk); err != nil { + return nil, err + } + + schemaMap[name] = schema.ColumnSchema{ + ColumnName: name, + Type: dataType, + } + } + + return schemaMap, nil +} diff --git a/internal/database/convertor_schema.go b/internal/database/convertor_schema.go index c7186e9e..929d0795 100644 --- a/internal/database/convertor_schema.go +++ b/internal/database/convertor_schema.go @@ -132,40 +132,39 @@ func (w *Converter) inferSchemaForJSONLFileWithDescribe(db *DuckDb, filePath str return res, nil } -// todo KAI think about schema change -//func (w *Converter) detectSchemaChange(filePath string) error { -// inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) -// if err != nil { -// return fmt.Errorf("failed to infer schema from JSON file: %w", err) -// } -// // the conversion schema is the full schema for the table that we have alreadf inferred -// conversionSchemaMap := w.conversionSchema.AsMap() -// // the table schema is the (possibly partial) schema which was defined in config - we use this to exclude columns -// // which have a type specified -// tableSchemaMap := w.tableSchema.AsMap() -// // Compare the inferred schema with the existing conversionSchema -// var changedColumns []ColumnSchemaChange -// for _, col := range inferredChunksSchema.Columns { -// // if the table schema definition specifies a type for this column, ignore the columns (as we will use the defined type) -// // we are only interested in a type change if the column is not defined in the table schema -// if columnDef, ok := tableSchemaMap[col.ColumnName]; ok { -// if columnDef.Type != "" { -// // if the column is defined in the table schema, ignore it -// continue -// } -// } -// -// existingCol, exists := conversionSchemaMap[col.SourceName] -// if exists && col.Type != existingCol.Type { -// changedColumns = append(changedColumns, ColumnSchemaChange{ -// Name: col.SourceName, -// OldType: existingCol.Type, -// NewType: col.Type, -// }) -// } -// } -// if len(changedColumns) > 0 { -// return &SchemaChangeError{ChangedColumns: changedColumns} -// } -// return nil -//} +func (w *Converter) detectSchemaChange(filePath string) error { + inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) + if err != nil { + return fmt.Errorf("failed to infer schema from JSON file: %w", err) + } + // the conversion schema is the full schema for the table that we have alreadf inferred + conversionSchemaMap := w.conversionSchema.AsMap() + // the table schema is the (possibly partial) schema which was defined in config - we use this to exclude columns + // which have a type specified + tableSchemaMap := w.tableSchema.AsMap() + // Compare the inferred schema with the existing conversionSchema + var changedColumns []ColumnSchemaChange + for _, col := range inferredChunksSchema.Columns { + // if the table schema definition specifies a type for this column, ignore the columns (as we will use the defined type) + // we are only interested in a type change if the column is not defined in the table schema + if columnDef, ok := tableSchemaMap[col.ColumnName]; ok { + if columnDef.Type != "" { + // if the column is defined in the table schema, ignore it + continue + } + } + + existingCol, exists := conversionSchemaMap[col.SourceName] + if exists && col.Type != existingCol.Type { + changedColumns = append(changedColumns, ColumnSchemaChange{ + Name: col.SourceName, + OldType: existingCol.Type, + NewType: col.Type, + }) + } + } + if len(changedColumns) > 0 { + return &SchemaChangeError{ChangedColumns: changedColumns} + } + return nil +} diff --git a/internal/database/convertor_validate.go b/internal/database/convertor_validate.go index ef6aadd1..8c96d944 100644 --- a/internal/database/convertor_validate.go +++ b/internal/database/convertor_validate.go @@ -27,7 +27,6 @@ func (w *Converter) validateRows(jsonlFilePaths []string) error { err := row.Scan(&failedRowCount, &columnsWithNullsInterface) if err != nil { - // todo kai think about schema change return handleConversionError("row validation query failed", err, jsonlFilePaths...) } diff --git a/internal/database/schema_comparison.go b/internal/database/schema_comparison.go index 3e46e05e..8603317b 100644 --- a/internal/database/schema_comparison.go +++ b/internal/database/schema_comparison.go @@ -6,6 +6,9 @@ import ( "strings" ) +// TableSchemaStatus represents the status of a table schema comparison +// this is not used at present but will be used when we implement ducklake schema evolution handling +// It indicates whether the table exists, whether the schema matches, whether it can be migrated by ducklake type TableSchemaStatus struct { TableExists bool SchemaMatches bool @@ -13,8 +16,8 @@ type TableSchemaStatus struct { SchemaDiff string } -// TODO #DL check if we need this https://github.com/turbot/tailpipe/issues/481 - +// NewTableSchemaStatusFromComparison compares an existing schema with a conversion schema +// and returns a TableSchemaStatus indicating whether they match, can be migrated, and the differences func NewTableSchemaStatusFromComparison(existingSchema map[string]schema.ColumnSchema, conversionSchema schema.ConversionSchema) TableSchemaStatus { var diffParts []string canMigrate := true From e4bbd3493d960f7f46bab6613b77a636390d87cc Mon Sep 17 00:00:00 2001 From: Karan Popat Date: Fri, 12 Sep 2025 12:21:22 +0530 Subject: [PATCH 35/61] Change badge links to dynamic endpoints (#560) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3daa5a14..2ed4cfd4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -[![plugins](https://img.shields.io/badge/plugins-5-blue)](https://hub.tailpipe-io.vercel.app/)   -[![plugins](https://img.shields.io/badge/mods-14-blue)](https://hub.tailpipe-io.vercel.app/)   -[![slack](https://img.shields.io/badge/slack-2695-blue)](https://turbot.com/community/join?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)   +[![plugins](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_plugins)](https://hub.tailpipe-io.vercel.app/)   +[![mods](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_mods)](https://hub.tailpipe-io.vercel.app/)   +[![slack](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=slack)](https://turbot.com/community/join?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)   [![maintained by](https://img.shields.io/badge/maintained%20by-Turbot-blue)](https://turbot.com?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme) # select * from logs; From 85d8610f3fac5fbd4e694fc1ebe890d99ae9ec1e Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 16 Sep 2025 11:44:45 +0530 Subject: [PATCH 36/61] Improve plugin install/update/uninstall error messages for newly locally developed new plugins not published on hub (#537) --- cmd/plugin.go | 34 +++++++++++++++++++++++++++++++++- go.mod | 14 +++++++------- go.sum | 25 ++++++++++++++----------- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/cmd/plugin.go b/cmd/plugin.go index 8a5001bc..03e2a4f0 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -294,7 +294,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) { report := &pplugin.PluginInstallReport{ Plugin: pluginName, Skipped: true, - SkipReason: pconstants.InstallMessagePluginNotFound, + SkipReason: pconstants.InstallMessagePluginNotDistributedViaHub, IsUpdateReport: false, } reportChannel <- report @@ -454,6 +454,20 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { return } } else { + // Plugin not installed locally. If it's a hub plugin, check if it exists in hub. + org, name, constraint := ref.GetOrgNameAndStream() + if ref.IsFromTurbotHub() { + if _, err := pplugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint); err != nil { + updateResults = append(updateResults, &pplugin.PluginInstallReport{ + Skipped: true, + Plugin: p, + SkipReason: pconstants.InstallMessagePluginNotDistributedViaHub, + IsUpdateReport: true, + }) + continue + } + } + // Exists on hub (or not a hub plugin) but not installed locally exitCode = pconstants.ExitCodePluginNotFound updateResults = append(updateResults, &pplugin.PluginInstallReport{ Skipped: true, @@ -642,6 +656,14 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { return } + // load installation state (needed for hub existence checks) + state, err := installationstate.Load() + if err != nil { + error_helpers.ShowError(ctx, fmt.Errorf("could not load state")) + exitCode = pconstants.ExitCodePluginLoadingError + return + } + if len(args) == 0 { fmt.Println() //nolint:forbidigo // ui output error_helpers.ShowError(ctx, fmt.Errorf("you need to provide at least one plugin to uninstall")) @@ -659,6 +681,16 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { if report, err := plugin.Remove(ctx, p); err != nil { if strings.Contains(err.Error(), "not found") { exitCode = pconstants.ExitCodePluginNotFound + // check hub existence to tailor message + ref := pociinstaller.NewImageRef(p) + if ref.IsFromTurbotHub() { + org, name, constraint := ref.GetOrgNameAndStream() + if _, herr := pplugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint); herr != nil { + // Not on hub and not installed locally + error_helpers.ShowError(ctx, fmt.Errorf("Failed to uninstall '%s' not found on hub and not installed locally.", p)) + continue + } + } } error_helpers.ShowErrorWithMessage(ctx, err, fmt.Sprintf("Failed to uninstall plugin '%s'", p)) } else { diff --git a/go.mod b/go.mod index fcabd309..0236b25a 100644 --- a/go.mod +++ b/go.mod @@ -6,22 +6,22 @@ toolchain go1.24.0 replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 - github.com/turbot/pipe-fittings/v2 => ../pipe-fittings + // github.com/turbot/pipe-fittings/v2 => ../pipe-fittings //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) require ( - github.com/Masterminds/semver/v3 v3.2.1 - github.com/hashicorp/hcl/v2 v2.20.1 + github.com/Masterminds/semver/v3 v3.4.0 + github.com/hashicorp/hcl/v2 v2.24.0 github.com/mattn/go-isatty v0.0.20 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 - github.com/turbot/pipe-fittings/v2 v2.6.0 + github.com/turbot/pipe-fittings/v2 v2.7.0-rc.0 github.com/turbot/tailpipe-plugin-sdk v0.9.2 - github.com/zclconf/go-cty v1.14.4 + github.com/zclconf/go-cty v1.16.3 golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 ) @@ -177,7 +177,7 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/locker v1.0.1 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect @@ -205,7 +205,7 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stevenle/topsort v0.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect diff --git a/go.sum b/go.sum index 856b516a..19ecc374 100644 --- a/go.sum +++ b/go.sum @@ -630,8 +630,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= @@ -1037,8 +1037,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= +github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/hashicorp/terraform-registry-address v0.2.1 h1:QuTf6oJ1+WSflJw6WYOHhLgwUiQ0FrROpHPYFtwTYWM= github.com/hashicorp/terraform-registry-address v0.2.1/go.mod h1:BSE9fIFzp0qWsJUUyGquo4ldV9k2n+psif6NYkBRS3Y= github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= @@ -1165,8 +1165,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -1264,8 +1264,9 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= @@ -1307,6 +1308,8 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= +github.com/turbot/pipe-fittings/v2 v2.7.0-rc.0 h1:p9/Hf0BNNjZVs5C4AqZlQgmihKt/nboh5OrfGcH8Mhk= +github.com/turbot/pipe-fittings/v2 v2.7.0-rc.0/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= @@ -1327,10 +1330,10 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= -github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= +github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zclconf/go-cty-yaml v1.0.3 h1:og/eOQ7lvA/WWhHGFETVWNduJM7Rjsv2RRpx1sdFMLc= github.com/zclconf/go-cty-yaml v1.0.3/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= From 012167b14cb38642089fff76122255182e9fa9e6 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 16 Sep 2025 11:45:46 +0530 Subject: [PATCH 37/61] Implement cleanup of temporary plugin directories on next command execution (#473) --- cmd/plugin.go | 18 ++++++++ cmd/root.go | 3 +- internal/collector/collector.go | 10 +++-- internal/filepaths/collection_temp_dir.go | 51 ++--------------------- internal/filepaths/prune.go | 21 ++-------- 5 files changed, 33 insertions(+), 70 deletions(-) diff --git a/cmd/plugin.go b/cmd/plugin.go index 03e2a4f0..bd1475b4 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -17,6 +17,7 @@ import ( pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/installationstate" pociinstaller "github.com/turbot/pipe-fittings/v2/ociinstaller" pplugin "github.com/turbot/pipe-fittings/v2/plugin" @@ -247,6 +248,9 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) { } }() + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + // if diagnostic mode is set, print out config and return if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { localcmdconfig.DisplayConfig() @@ -380,6 +384,9 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { } }() + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + // if diagnostic mode is set, print out config and return if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { localcmdconfig.DisplayConfig() @@ -650,6 +657,9 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { } }() + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + // if diagnostic mode is set, print out config and return if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { localcmdconfig.DisplayConfig() @@ -728,6 +738,10 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPluginListCmd list") + + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + defer func() { utils.LogTime("runPluginListCmd end") if r := recover(); r != nil { @@ -776,6 +790,10 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPluginShowCmd start") + + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + defer func() { utils.LogTime("runPluginShowCmd end") if r := recover(); r != nil { diff --git a/cmd/root.go b/cmd/root.go index a293be64..69ef0a8a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,10 +59,9 @@ func rootCommand() *cobra.Command { } func Execute() int { - rootCmd := rootCommand() utils.LogTime("cmd.root.Execute start") defer utils.LogTime("cmd.root.Execute end") - + rootCmd := rootCommand() if err := rootCmd.Execute(); err != nil { exitCode = -1 } diff --git a/internal/collector/collector.go b/internal/collector/collector.go index f73474ec..0d04b0a5 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -19,7 +19,8 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/row_source" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" + localfilepaths "github.com/turbot/tailpipe/internal/filepaths" + "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/tailpipe/internal/plugin" ) @@ -64,9 +65,12 @@ func New(pluginManager *plugin.PluginManager, partition *config.Partition, cance // get the temp data dir for this collection // - this is located in ~/.turbot/internal/collection// // first clear out any old collection temp dirs - filepaths.CleanupCollectionTempDirs() + // get the collection directory for this workspace + collectionDir := config.GlobalWorkspaceProfile.GetCollectionDir() + + filepaths.CleanupPidTempDirs(collectionDir) // then create a new collection temp dir - collectionTempDir := filepaths.EnsureCollectionTempDir() + collectionTempDir := localfilepaths.EnsureCollectionTempDir() // create the collector c := &Collector{ diff --git a/internal/filepaths/collection_temp_dir.go b/internal/filepaths/collection_temp_dir.go index 1e33b0ee..b635f0ff 100644 --- a/internal/filepaths/collection_temp_dir.go +++ b/internal/filepaths/collection_temp_dir.go @@ -1,56 +1,13 @@ package filepaths import ( - "fmt" - "github.com/turbot/pipe-fittings/v2/utils" + + "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/tailpipe/internal/config" - "log/slog" - "os" - "path/filepath" - "strconv" ) func EnsureCollectionTempDir() string { collectionDir := config.GlobalWorkspaceProfile.GetCollectionDir() - - // add a PID directory to the collection directory - collectionTempDir := filepath.Join(collectionDir, fmt.Sprintf("%d", os.Getpid())) - - // create the directory if it doesn't exist - if _, err := os.Stat(collectionTempDir); os.IsNotExist(err) { - err := os.MkdirAll(collectionTempDir, 0755) - if err != nil { - slog.Error("failed to create collection temp dir", "error", err) - } - } - return collectionTempDir -} - -func CleanupCollectionTempDirs() { - // get the collection directory for this workspace - collectionDir := config.GlobalWorkspaceProfile.GetCollectionDir() - - files, err := os.ReadDir(collectionDir) - if err != nil { - slog.Warn("failed to list files in collection dir", "error", err) - return - } - for _, file := range files { - // if the file is a directory and is not our collection temp dir, remove it - if file.IsDir() { - // the folder name is the PID - check whether that pid exists - // if it doesn't, remove the folder - // Attempt to find the process - // try to parse the directory name as a pid - pid, err := strconv.ParseInt(file.Name(), 10, 32) - if err == nil { - if utils.PidExists(int(pid)) { - slog.Info(fmt.Sprintf("Cleaning existing collection temp dirs - skipping directory '%s' as process with PID %d exists", file.Name(), pid)) - continue - } - } - slog.Debug("Removing directory", "dir", file.Name()) - _ = os.RemoveAll(filepath.Join(collectionDir, file.Name())) - } - } + pidTempDir := filepaths.EnsurePidTempDir(collectionDir) + return pidTempDir } diff --git a/internal/filepaths/prune.go b/internal/filepaths/prune.go index e4c73f13..247df1a3 100644 --- a/internal/filepaths/prune.go +++ b/internal/filepaths/prune.go @@ -1,9 +1,9 @@ package filepaths import ( - "io" "os" "path/filepath" + pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" ) // PruneTree recursively deletes empty directories in the given folder. @@ -12,7 +12,7 @@ func PruneTree(folder string) error { if _, err := os.Stat(folder); os.IsNotExist(err) { return nil } - isEmpty, err := isDirEmpty(folder) + isEmpty, err := pfilepaths.IsDirEmpty(folder) if err != nil { return err } @@ -36,7 +36,7 @@ func PruneTree(folder string) error { } // Check again if the folder is empty after pruning subdirectories - isEmpty, err = isDirEmpty(folder) + isEmpty, err = pfilepaths.IsDirEmpty(folder) if err != nil { return err } @@ -47,18 +47,3 @@ func PruneTree(folder string) error { return nil } - -// isDirEmpty checks if a directory is empty. -func isDirEmpty(dir string) (bool, error) { - f, err := os.Open(dir) - if err != nil { - return false, err - } - defer f.Close() - - _, err = f.Readdir(1) - if err == io.EOF { - return true, nil - } - return false, err -} From b9a4ae0d982c69a3ff135f8d6eeb7df96272308f Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 16 Sep 2025 12:31:29 +0100 Subject: [PATCH 38/61] Install prod duckdb extension. Closes #476 --- cmd/query.go | 1 - internal/cmdconfig/cmd_hooks.go | 1 - internal/database/duck_db.go | 5 +---- internal/parse/load_config_test.go | 3 +-- internal/plugin/plugin_manager.go | 1 - 5 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cmd/query.go b/cmd/query.go index 6a50f066..08bec108 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -119,6 +119,5 @@ func setExitCodeForQueryError(err error) { return } - // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/496 exitCode = 1 } diff --git a/internal/cmdconfig/cmd_hooks.go b/internal/cmdconfig/cmd_hooks.go index 503435b2..78e5b478 100644 --- a/internal/cmdconfig/cmd_hooks.go +++ b/internal/cmdconfig/cmd_hooks.go @@ -46,7 +46,6 @@ func preRunHook(cmd *cobra.Command, args []string) error { ew := initGlobalConfig(ctx) // display any warnings ew.ShowWarnings() - // TODO #errors sort exit code https://github.com/turbot/tailpipe/issues/496 // check for error error_helpers.FailOnError(ew.Error) diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index f35756c3..a6f5fcb2 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -285,10 +285,7 @@ func GetDucklakeInitCommands(readonly bool) []SqlCommand { commands := []SqlCommand{ {Description: "install sqlite extension", Command: "install sqlite"}, - // TODO #DL change to using prod extension when stable - // https://github.com/turbot/tailpipe/issues/476 - // _, err = db.Exec("install ducklake;") - {Description: "install ducklake extension", Command: "force install ducklake from core_nightly"}, + {Description: "install ducklake extension", Command: "install ducklake;"}, {Description: "attach to ducklake database", Command: attachQuery}, } return commands diff --git a/internal/parse/load_config_test.go b/internal/parse/load_config_test.go index cb600da0..d4e97ddc 100644 --- a/internal/parse/load_config_test.go +++ b/internal/parse/load_config_test.go @@ -496,7 +496,6 @@ func TestParseTailpipeConfig(t *testing.T) { want *config.TailpipeConfig wantErr bool }{ - // TODO #testing add more test cases https://github.com/turbot/tailpipe/issues/506 { name: "static tables", args: args{ @@ -868,4 +867,4 @@ func TestParseTailpipeConfig(t *testing.T) { }) } -} \ No newline at end of file +} diff --git a/internal/plugin/plugin_manager.go b/internal/plugin/plugin_manager.go index 765e790e..0080f265 100644 --- a/internal/plugin/plugin_manager.go +++ b/internal/plugin/plugin_manager.go @@ -617,7 +617,6 @@ func loadPluginVersionFile(ctx context.Context) (*versionfile.PluginVersionFile, return nil, err } - // TODO CHECK THIS https://github.com/turbot/tailpipe/issues/507 // add any "local" plugins (i.e. plugins installed under the 'local' folder) into the version file ew := pluginVersions.AddLocalPlugins(ctx) if ew.Error != nil { From ba21ad77ad1c107cc362e2208137f96e29f075e6 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Tue, 16 Sep 2025 17:02:55 +0530 Subject: [PATCH 39/61] chore: use 'turbot/goreleaser-cross' image to build binaries (#561) * chore :use 'turbot/goreleaser-cross' image to build This fixes the missing c++ libraries issue we were getting because of latest DuckDb version. --- .acceptance.goreleaser.yml | 12 +-- .darwin.goreleaser.yml | 58 ------------ .github/workflows/11-test-acceptance.yaml | 19 ++-- .goreleaser.yml | 64 +++++++++----- Dockerfile.goreleaser-cross | 40 --------- Makefile | 102 +++------------------- 6 files changed, 62 insertions(+), 233 deletions(-) delete mode 100644 .darwin.goreleaser.yml delete mode 100644 Dockerfile.goreleaser-cross diff --git a/.acceptance.goreleaser.yml b/.acceptance.goreleaser.yml index d764494f..f96f03d5 100644 --- a/.acceptance.goreleaser.yml +++ b/.acceptance.goreleaser.yml @@ -1,6 +1,3 @@ -# Acceptance testing configuration - builds only Linux AMD64 for faster testing -version: 2 - before: hooks: - go mod tidy @@ -13,17 +10,12 @@ builds: - amd64 env: - - CC=x86_64-linux-gnu-gcc-13 - - CXX=x86_64-linux-gnu-g++-13 - - CGO_CXXFLAGS=-std=c++17 - - CGO_LDFLAGS=-lstdc++ -static-libstdc++ + - CC=x86_64-linux-gnu-gcc + - CXX=x86_64-linux-gnu-g++ ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - flags: - - -buildvcs=false - archives: - id: homebrew format: tar.gz diff --git a/.darwin.goreleaser.yml b/.darwin.goreleaser.yml deleted file mode 100644 index b1932ae1..00000000 --- a/.darwin.goreleaser.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Darwin-only goreleaser configuration -version: 2 - -before: - hooks: - - go mod tidy - -builds: - # Darwin AMD64 build with clang - - id: tailpipe-darwin-amd64 - binary: tailpipe - goos: - - darwin - goarch: - - amd64 - env: - - CC=o64-clang - - CXX=o64-clang++ - ldflags: - - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - flags: - - -buildvcs=false - - # Darwin ARM64 build with clang - - id: tailpipe-darwin-arm64 - binary: tailpipe - goos: - - darwin - goarch: - - arm64 - env: - - CC=oa64-clang - - CXX=oa64-clang++ - ldflags: - - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - flags: - - -buildvcs=false - -archives: - - id: darwin - format: tar.gz - name_template: "{{ .ProjectName }}.{{ .Os }}.{{ .Arch }}" - files: - - none* - -checksum: - name_template: 'checksums.txt' - -snapshot: - name_template: "{{ incpatch .Version }}-next" - -changelog: - disable: true - sort: asc - filters: - exclude: - - '^docs:' - - '^test:' diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 22c89beb..cc81641a 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -45,10 +45,10 @@ jobs: path: tailpipe-plugin-core ref: main - # this is required, check golangci-lint-action docs + # this is required, check golangci-lint-action docs - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: - go-version: "1.23" + go-version: '1.23' cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: Run CLI Unit Tests @@ -57,13 +57,6 @@ jobs: go clean -testcache go test -timeout 30s ./... -test.v - - name: Build Custom Goreleaser Cross Image for Linux - run: |- - cd tailpipe - make build-goreleaser-image - echo "=== Verifying GCC 13 ===" - docker run --rm tailpipe-goreleaser-cross:gcc13 gcc-13 --version | head -1 - - name: Build run: |- cd tailpipe @@ -122,7 +115,7 @@ jobs: id: prepare-for-downloads run: | mkdir ~/artifacts - + - name: Download Linux Build Artifacts uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 if: ${{ matrix.platform == 'ubuntu-latest' }} @@ -130,7 +123,7 @@ jobs: name: build-artifact-linux path: ~/artifacts - - name: Extract Linux Artifacts and Install Binary + - name: Extract Ubuntu Artifacts and Install Binary if: ${{ matrix.platform == 'ubuntu-latest' }} run: | mkdir ~/build @@ -188,8 +181,8 @@ jobs: name: build-artifact-linux failOnError: true - - name: Clean up Linux Build + - name: Clean up Darwin Build uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0 with: - name: build-artifact-linux + name: build-artifact-darwin failOnError: true diff --git a/.goreleaser.yml b/.goreleaser.yml index 2ac9f062..d8e8a667 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,43 +1,67 @@ version: 2 -before: - hooks: - - go mod tidy - builds: - # Linux AMD64 build with GCC 13+ + - id: tailpipe-linux-arm64 + binary: tailpipe + goos: + - linux + goarch: + - arm64 + + env: + - CC=aarch64-linux-gnu-gcc + - CXX=aarch64-linux-gnu-g++ + + # Custom ldflags. + # + # Default: '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser' + # Templates: allowed + ldflags: + # Go Releaser analyzes your Git repository and identifies the most recent Git tag (typically the highest version number) as the version for your release. + # This is how it determines the value of {{.Version}}. + - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + - id: tailpipe-linux-amd64 binary: tailpipe goos: - linux goarch: - amd64 + env: - - CC=x86_64-linux-gnu-gcc-13 - - CXX=x86_64-linux-gnu-g++-13 - - CGO_CXXFLAGS=-std=c++17 - - CGO_LDFLAGS=-lstdc++ -static-libstdc++ + - CC=x86_64-linux-gnu-gcc + - CXX=x86_64-linux-gnu-g++ + ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - flags: - - -buildvcs=false - # Linux ARM64 build with GCC 13+ - - id: tailpipe-linux-arm64 + - id: tailpipe-darwin-arm64 binary: tailpipe goos: - - linux + - darwin goarch: - arm64 + + env: + - CC=oa64-clang + - CXX=oa64-clang++ + + ldflags: + - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + + - id: tailpipe-darwin-amd64 + binary: tailpipe + goos: + - darwin + goarch: + - amd64 + env: - - CC=aarch64-linux-gnu-gcc-13 - - CXX=aarch64-linux-gnu-g++-13 - - CGO_CXXFLAGS=-std=c++17 - - CGO_LDFLAGS=-lstdc++ -static-libstdc++ + - CC=o64-clang + - CXX=o64-clang++ + ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - flags: - - -buildvcs=false release: prerelease: auto diff --git a/Dockerfile.goreleaser-cross b/Dockerfile.goreleaser-cross deleted file mode 100644 index 72dba423..00000000 --- a/Dockerfile.goreleaser-cross +++ /dev/null @@ -1,40 +0,0 @@ -# Use Ubuntu 24.04 as base to get GCC 13+ -FROM ubuntu:noble - -# Install essential packages and cross-compilation tools -RUN apt-get update && apt-get install -y \ - wget curl git build-essential \ - gcc-13 g++-13 \ - gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \ - gcc-x86-64-linux-gnu g++-x86-64-linux-gnu \ - gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf \ - && rm -rf /var/lib/apt/lists/* - -# Install Go 1.24.5 -RUN wget https://go.dev/dl/go1.24.5.linux-amd64.tar.gz && \ - tar -C /usr/local -xzf go1.24.5.linux-amd64.tar.gz && \ - rm go1.24.5.linux-amd64.tar.gz - -# Install goreleaser 2.11.2 -RUN wget https://github.com/goreleaser/goreleaser/releases/download/v2.11.2/goreleaser_Linux_x86_64.tar.gz && \ - tar -xzf goreleaser_Linux_x86_64.tar.gz && \ - mv goreleaser /usr/local/bin/ && \ - rm goreleaser_Linux_x86_64.tar.gz - -# Set up environment -ENV PATH="/usr/local/go/bin:${PATH}" -ENV GOROOT="/usr/local/go" -ENV GOPATH="/go" -ENV CGO_ENABLED=1 - -# Set default cross-compilation environment variables for Linux AMD64 -ENV CC_linux_amd64=x86_64-linux-gnu-gcc-13 -ENV CXX_linux_amd64=x86_64-linux-gnu-g++-13 -ENV CGO_CXXFLAGS_linux_amd64="-std=c++17" -ENV CGO_LDFLAGS_linux_amd64="-lstdc++ -static-libstdc++" - -# Create working directory -WORKDIR /go/src - -# Set entrypoint to goreleaser -ENTRYPOINT ["goreleaser"] diff --git a/Makefile b/Makefile index 83337548..42138790 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ OUTPUT_DIR?=/usr/local/bin PACKAGE_NAME := github.com/turbot/tailpipe -GOLANG_CROSS_VERSION ?= v1.25.0 +GOLANG_CROSS_VERSION ?= gcc13-osxcross-20250912194615 # sed 's/[\/_]/-/g': Replaces both slashes (/) and underscores (_) with hyphens (-). -# sed 's/[^a-zA-Z0-9.-]//g': Removes any character that isn't alphanumeric, a dot (.), or a hyphen (-). +# sed 's/[^a-zA-Z0-9.-]//g': Removes any character that isn’t alphanumeric, a dot (.), or a hyphen (-). # This is to ensure that the branch name is a valid semver pre-release identifier. .PHONY: build build: @@ -12,13 +12,8 @@ build: go build -o $(OUTPUT_DIR) -ldflags "-X main.version=0.0.0-dev-$(GIT_BRANCH).$(TIMESTAMP)" . -.PHONY: build-goreleaser-image -build-goreleaser-image: - docker build -f Dockerfile.goreleaser-cross -t tailpipe-goreleaser-cross:gcc13 . - .PHONY: release-dry-run -release-dry-run: build-goreleaser-image - @echo "Building for Linux platforms using custom image with GCC 13+..." +release-dry-run: @docker run \ --rm \ -e CGO_ENABLED=1 \ @@ -28,12 +23,11 @@ release-dry-run: build-goreleaser-image -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - tailpipe-goreleaser-cross:gcc13 \ + ghcr.io/turbot/goreleaser-cross:${GOLANG_CROSS_VERSION} \ --clean --skip=validate --skip=publish --snapshot .PHONY: release-acceptance -release-acceptance: build-goreleaser-image - @echo "Building for acceptance testing using custom image with GCC 13+..." +release-acceptance: @docker run \ --rm \ -e CGO_ENABLED=1 \ @@ -43,21 +37,16 @@ release-acceptance: build-goreleaser-image -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - tailpipe-goreleaser-cross:gcc13 \ + ghcr.io/turbot/goreleaser-cross:${GOLANG_CROSS_VERSION} \ --clean --skip=validate --skip=publish --snapshot --config=.acceptance.goreleaser.yml .PHONY: release -release: build-goreleaser-image +release: @if [ ! -f ".release-env" ]; then \ echo ".release-env is required for release";\ exit 1;\ fi - @echo "Building for all platforms (Linux + Darwin) for release..." - @echo "Linux builds: Using custom image with GCC 13+" - @echo "Darwin builds: Using standard goreleaser-cross" - @echo "" - @echo "Building Linux targets..." - @docker run \ + docker run \ --rm \ -e CGO_ENABLED=1 \ --env-file .release-env \ @@ -67,76 +56,5 @@ release: build-goreleaser-image -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - tailpipe-goreleaser-cross:gcc13 \ - release --clean --skip=validate - @echo "" - @echo "Building Darwin targets..." - @docker run \ - --rm \ - -e CGO_ENABLED=1 \ - --env-file .release-env \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/go/src/tailpipe \ - -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ - -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ - -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ - -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ - release --clean --skip=validate --config=.darwin.goreleaser.yml - @echo "" - @echo "✅ Release builds completed successfully!" - @echo "📦 Linux builds: AMD64, ARM64" - @echo "🍎 Darwin builds: AMD64, ARM64" - -# Darwin-only builds using standard goreleaser-cross -.PHONY: release-darwin -release-darwin: - @echo "Building Darwin targets using standard goreleaser-cross..." - @docker run \ - --rm \ - -e CGO_ENABLED=1 \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/go/src/tailpipe \ - -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ - -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ - -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ - -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ - --clean --skip=validate --skip=publish --snapshot --config=.darwin.goreleaser.yml - -# Build for all platforms (Linux + Darwin) - UNIFIED APPROACH -.PHONY: release-all-platforms -release-all-platforms: build-goreleaser-image - @echo "Building for all platforms using unified approach..." - @echo "Linux builds: Using custom image with GCC 13+" - @echo "Darwin builds: Using standard goreleaser-cross" - @echo "" - @echo "Building Linux targets..." - @docker run \ - --rm \ - -e CGO_ENABLED=1 \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/go/src/tailpipe \ - -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ - -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ - -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ - -w /go/src/tailpipe \ - tailpipe-goreleaser-cross:gcc13 \ - --clean --skip=validate --skip=publish --snapshot - @echo "" - @echo "Building Darwin targets..." - @docker run \ - --rm \ - -e CGO_ENABLED=1 \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/go/src/tailpipe \ - -v `pwd`/../pipe-fittings:/go/src/pipe-fittings \ - -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ - -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ - -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ - --clean --skip=validate --skip=publish --snapshot --config=.darwin.goreleaser.yml - @echo "" - @echo "✅ All platform builds completed successfully!" - @echo "📦 Linux builds: AMD64, ARM64" - @echo "🍎 Darwin builds: AMD64, ARM64" + ghcr.io/turbot/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + release --clean --skip=validate \ No newline at end of file From 58571132c428c0df030cdbc716c8cda0571177a9 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 16 Sep 2025 17:39:27 +0530 Subject: [PATCH 40/61] Add exit codes for cancellation and failure scenarios in collect, compact, connect, and query commands (#534) --- cmd/collect.go | 15 ++- cmd/compact.go | 15 ++- cmd/connect.go | 25 +++-- cmd/format.go | 50 +++++++-- cmd/partition.go | 88 +++++++++++++--- cmd/plugin.go | 101 ++++++++++++++---- cmd/query.go | 11 +- cmd/source.go | 48 +++++++-- cmd/table.go | 47 +++++++-- go.mod | 4 +- internal/cmdconfig/diagnostics.go | 2 +- internal/error_helpers/error_helpers.go | 113 +++++++++++++++++++++ internal/interactive/interactive_client.go | 4 +- internal/interactive/run.go | 2 +- internal/plugin/errors.go | 3 - internal/plugin/plugin_manager.go | 13 +-- internal/query/execute.go | 2 +- main.go | 2 +- 18 files changed, 439 insertions(+), 106 deletions(-) create mode 100644 internal/error_helpers/error_helpers.go diff --git a/cmd/collect.go b/cmd/collect.go index e6dc6b0f..2576e764 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -17,7 +17,6 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" @@ -25,6 +24,7 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" ) @@ -71,8 +71,8 @@ func runCollectCmd(cmd *cobra.Command, args []string) { } if err != nil { - if errors.Is(err, context.Canceled) { - fmt.Println("Collection cancelled.") //nolint:forbidigo // ui output + if error_helpers.IsCancelledError(err) { + fmt.Println("tailpipe collect command cancelled.") //nolint:forbidigo // ui output } else { error_helpers.ShowError(ctx, err) } @@ -364,14 +364,13 @@ func setExitCodeForCollectError(err error) { if exitCode != 0 || err == nil { return } - // TODO Set exit code for cancellation - if errors.Is(err, context.Canceled) { - exitCode = 0 + // set exit code for cancellation + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled return } - // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/496 - exitCode = 1 + exitCode = pconstants.ExitCodeCollectionFailed } // parse the from time diff --git a/cmd/compact.go b/cmd/compact.go index a223f936..c777b203 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "errors" "fmt" "log/slog" "os" @@ -15,11 +14,11 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "golang.org/x/exp/maps" ) @@ -49,9 +48,9 @@ func runCompactCmd(cmd *cobra.Command, args []string) { if err != nil { setExitCodeForCompactError(err) - if errors.Is(err, context.Canceled) { + if error_helpers.IsCancelledError(err) { //nolint:forbidigo // ui - fmt.Println("Compact cancelled") + fmt.Println("tailpipe compact command cancelled.") } else { error_helpers.ShowError(ctx, err) } @@ -126,5 +125,11 @@ func setExitCodeForCompactError(err error) { if exitCode != 0 || err == nil { return } - exitCode = 1 + // set exit code for cancellation + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } + + exitCode = pconstants.ExitCodeCompactFailed } diff --git a/cmd/connect.go b/cmd/connect.go index c0e8d444..a7b4da56 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "fmt" - "golang.org/x/exp/maps" "log" "os" "path/filepath" "strings" "time" + "golang.org/x/exp/maps" + "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/thediveo/enumflag/v2" @@ -18,13 +19,14 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" "github.com/turbot/pipe-fittings/v2/connection" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/contexthelpers" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // variable used to assign the output mode flag @@ -94,13 +96,21 @@ The generated script can be used with DuckDB: func runConnectCmd(cmd *cobra.Command, _ []string) { var err error var initFilePath string - ctx := cmd.Context() + ctx, cancel := context.WithCancel(cmd.Context()) + contexthelpers.StartCancelHandler(cancel) defer func() { if r := recover(); r != nil { err = helpers.ToError(r) } - setExitCodeForConnectError(err) + if err != nil { + if error_helpers.IsCancelledError(err) { + fmt.Println("tailpipe connect command cancelled.") //nolint:forbidigo // ui output + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForConnectError(err) + } displayOutput(ctx, initFilePath, err) }() @@ -409,8 +419,11 @@ func setExitCodeForConnectError(err error) { if exitCode != 0 || err == nil || viper.GetString(pconstants.ArgOutput) == pconstants.OutputFormatJSON { return } - - exitCode = 1 + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } + exitCode = pconstants.ExitCodeConnectFailed } // generateInitFilename generates a temporary filename with a timestamp diff --git a/cmd/format.go b/cmd/format.go index 97543dd2..5f5d40de 100644 --- a/cmd/format.go +++ b/cmd/format.go @@ -12,12 +12,12 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // variable used to assign the output mode flag @@ -73,11 +73,20 @@ func runFormatListCmd(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runFormatListCmd start") + var err error defer func() { utils.LogTime("runFormatListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe format list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForFormatError(err,1) } }() @@ -99,8 +108,8 @@ func runFormatListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -128,11 +137,20 @@ func runFormatShowCmd(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runFormatShowCmd start") + var err error defer func() { utils.LogTime("runFormatShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe format show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForFormatError(err, 1) } }() @@ -149,7 +167,21 @@ func runFormatShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForFormatError(err error, nonCancelCode int) { + // set exit code only if an error occurred and no exit code is already set + if exitCode != 0 || err == nil { + return + } + // set exit code for cancellation + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + // no dedicated format exit code exists yet; use generic nonzero failure + exitCode = nonCancelCode } diff --git a/cmd/partition.go b/cmd/partition.go index 2a25ee2c..78029017 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "log/slog" "os" @@ -15,7 +16,6 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" @@ -24,6 +24,7 @@ import ( "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/filepaths" "github.com/turbot/tailpipe/internal/plugin" ) @@ -79,11 +80,20 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPartitionListCmd start") + var err error defer func() { utils.LogTime("runPartitionListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("taillpipe partition list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPartitionError(err) } }() @@ -110,8 +120,8 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -138,13 +148,23 @@ func partitionShowCmd() *cobra.Command { func runPartitionShowCmd(cmd *cobra.Command, args []string) { // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPartitionShowCmd start") + var err error defer func() { utils.LogTime("runPartitionShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe partition show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPartitionError(err) } }() @@ -183,8 +203,8 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -213,12 +233,23 @@ func partitionDeleteCmd() *cobra.Command { } func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { - ctx := cmd.Context() - + // setup a cancel context and start cancel handler + ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now + contexthelpers.StartCancelHandler(cancel) + var err error defer func() { if r := recover(); r != nil { - exitCode = pconstants.ExitCodeUnknownErrorPanic - error_helpers.FailOnError(helpers.ToError(r)) + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("Partition cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPartitionError(err) } }() @@ -279,7 +310,14 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { spinner.Show() rowsDeleted, err := database.DeletePartition(ctx, partition, fromTime, toTime, db) spinner.Hide() - error_helpers.FailOnError(err) + if err != nil { + if errors.Is(err, context.Canceled) { + exitCode = pconstants.ExitCodeOperationCancelled + } else { + exitCode = 1 + } + error_helpers.FailOnError(err) + } // build the collection state path collectionStatePath := partition.CollectionStatePath(config.GlobalWorkspaceProfile.GetCollectionDir()) @@ -288,6 +326,7 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { if fromTime.IsZero() { err := os.Remove(collectionStatePath) if err != nil && !os.IsNotExist(err) { + exitCode = 1 error_helpers.FailOnError(fmt.Errorf("failed to delete collection state file: %s", err.Error())) } } else { @@ -296,7 +335,14 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { pluginManager := plugin.NewPluginManager() defer pluginManager.Close() err = pluginManager.UpdateCollectionState(ctx, partition, fromTime, collectionStatePath) - error_helpers.FailOnError(err) + if err != nil { + if errors.Is(err, context.Canceled) { + exitCode = pconstants.ExitCodeOperationCancelled + } else { + exitCode = 1 + } + error_helpers.FailOnError(err) + } } // now prune the collection folders @@ -317,3 +363,15 @@ func buildStatusMessage(rowsDeleted int, partition string, fromStr string) inter return fmt.Sprintf("\nDeleted partition '%s'%s%s.\n", partition, fromStr, deletedStr) } + +func setExitCodeForPartitionError(err error) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } + // no dedicated partition exit code; use generic nonzero failure + exitCode = 1 +} diff --git a/cmd/plugin.go b/cmd/plugin.go index bd1475b4..ba7dc795 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -16,7 +16,6 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/installationstate" pociinstaller "github.com/turbot/pipe-fittings/v2/ociinstaller" @@ -29,6 +28,7 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/ociinstaller" "github.com/turbot/tailpipe/internal/plugin" ) @@ -238,13 +238,25 @@ var pluginInstallSteps = []string{ } func runPluginInstallCmd(cmd *cobra.Command, args []string) { - ctx := cmd.Context() + //setup a cancel context and start cancel handler + ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now + contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPluginInstallCmd install") + var err error defer func() { utils.LogTime("runPluginInstallCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin install command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, 1) } }() @@ -374,13 +386,25 @@ func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string } func runPluginUpdateCmd(cmd *cobra.Command, args []string) { - ctx := cmd.Context() + //setup a cancel context and start cancel handler + ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now + contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPluginUpdateCmd start") + var err error defer func() { utils.LogTime("runPluginUpdateCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin update command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, 1) } }() @@ -645,15 +669,24 @@ func installPlugin(ctx context.Context, resolvedPlugin pplugin.ResolvedPluginVer func runPluginUninstallCmd(cmd *cobra.Command, args []string) { // setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPluginUninstallCmd uninstall") - + var err error defer func() { utils.LogTime("runPluginUninstallCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin uninstall command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, 1) } }() @@ -701,6 +734,8 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { continue } } + } else if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled } error_helpers.ShowErrorWithMessage(ctx, err, fmt.Sprintf("Failed to uninstall plugin '%s'", p)) } else { @@ -742,11 +777,20 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { // Clean up plugin temporary directories from previous crashes/interrupted installations filepaths.CleanupPluginTempDirs() + var err error defer func() { utils.LogTime("runPluginListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, pconstants.ExitCodePluginListFailure) } }() @@ -772,8 +816,8 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodePluginListFailure + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -787,6 +831,7 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { //setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now contexthelpers.StartCancelHandler(cancel) utils.LogTime("runPluginShowCmd start") @@ -794,11 +839,20 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { // Clean up plugin temporary directories from previous crashes/interrupted installations filepaths.CleanupPluginTempDirs() + var err error defer func() { utils.LogTime("runPluginShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, pconstants.ExitCodePluginShowFailure) } }() @@ -820,7 +874,18 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodePluginListFailure + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForPluginError(err error, nonCancelCode int) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + exitCode = nonCancelCode } diff --git a/cmd/query.go b/cmd/query.go index 08bec108..cdc6e011 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -73,7 +73,7 @@ func runQueryCmd(cmd *cobra.Command, args []string) { } if err != nil { error_helpers.ShowError(ctx, err) - setExitCodeForQueryError(err) + exitCode = pconstants.ExitCodeInitializationFailed } }() @@ -112,12 +112,3 @@ func runQueryCmd(cmd *cobra.Command, args []string) { exitCode = pconstants.ExitCodeQueryExecutionFailed } } - -func setExitCodeForQueryError(err error) { - // if exit code already set, leave as is - if exitCode != 0 || err == nil { - return - } - - exitCode = 1 -} diff --git a/cmd/source.go b/cmd/source.go index 972bfc97..ecc2785a 100644 --- a/cmd/source.go +++ b/cmd/source.go @@ -12,12 +12,12 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) func sourceCmd() *cobra.Command { @@ -70,11 +70,20 @@ func runSourceListCmd(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runSourceListCmd start") + var err error defer func() { utils.LogTime("runSourceListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe source list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForSourceError(err) } }() @@ -96,8 +105,8 @@ func runSourceListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -123,13 +132,23 @@ func sourceShowCmd() *cobra.Command { func runSourceShowCmd(cmd *cobra.Command, args []string) { //setup a cancel context and start cancel handler ctx, cancel := context.WithCancel(cmd.Context()) + //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now contexthelpers.StartCancelHandler(cancel) utils.LogTime("runSourceShowCmd start") + var err error defer func() { utils.LogTime("runSourceShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe source show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForSourceError(err) } }() @@ -152,7 +171,18 @@ func runSourceShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForSourceError(err error) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + exitCode = 1 } diff --git a/cmd/table.go b/cmd/table.go index 403b467c..5200e0b9 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -12,13 +12,13 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" + "github.com/turbot/tailpipe/internal/error_helpers" ) func tableCmd() *cobra.Command { @@ -72,11 +72,20 @@ func runTableListCmd(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runSourceListCmd start") + var err error defer func() { utils.LogTime("runSourceListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe table list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForTableError(err) } }() @@ -103,8 +112,8 @@ func runTableListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -133,11 +142,20 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(cmd.Context()) contexthelpers.StartCancelHandler(cancel) utils.LogTime("runTableShowCmd start") + var err error defer func() { utils.LogTime("runTableShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe table show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForTableError(err) } }() @@ -164,7 +182,18 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForTableError(err error) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + exitCode = 1 } diff --git a/go.mod b/go.mod index 0236b25a..c7d1e144 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/charmbracelet/bubbletea v1.2.4 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/dustin/go-humanize v1.0.1 + github.com/fatih/color v1.18.0 github.com/fsnotify/fsnotify v1.9.0 github.com/gosuri/uiprogress v0.0.1 github.com/hashicorp/go-hclog v1.6.3 @@ -39,6 +40,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/marcboeker/go-duckdb/v2 v2.3.5 + github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 github.com/thediveo/enumflag/v2 v2.0.5 github.com/turbot/tailpipe-plugin-core v0.2.10 golang.org/x/text v0.27.0 @@ -114,7 +116,6 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gertd/go-pluralize v0.2.1 // indirect @@ -199,7 +200,6 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/satyrius/gonx v1.4.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect - github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect diff --git a/internal/cmdconfig/diagnostics.go b/internal/cmdconfig/diagnostics.go index b5f4fe9d..ea2d26fa 100644 --- a/internal/cmdconfig/diagnostics.go +++ b/internal/cmdconfig/diagnostics.go @@ -9,8 +9,8 @@ import ( "strings" "github.com/spf13/viper" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/tailpipe/internal/constants" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // DisplayConfig prints all config set via WorkspaceProfile or HCL options diff --git a/internal/error_helpers/error_helpers.go b/internal/error_helpers/error_helpers.go new file mode 100644 index 00000000..d8936ec5 --- /dev/null +++ b/internal/error_helpers/error_helpers.go @@ -0,0 +1,113 @@ +// Copied from pipe-fittings/error_helpers.go. We handle cancellation differently: +// cancellations are a user choice, so we don't throw an error (normalized to "execution cancelled"). +// +//nolint:forbidigo // TODO: review fmt usage +package error_helpers + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/fatih/color" + "github.com/shiena/ansicolor" + "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/statushooks" +) + +func init() { + color.Output = ansicolor.NewAnsiColorWriter(os.Stderr) +} + +func FailOnError(err error) { + if err != nil { + panic(err) + } +} + +func FailOnErrorWithMessage(err error, message string) { + if err != nil { + panic(fmt.Sprintf("%s: %s", message, err.Error())) + } +} + +func ShowError(ctx context.Context, err error) { + if err == nil { + return + } + statushooks.Done(ctx) + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %v\n", constants.ColoredErr, TransformErrorToTailpipe(err)) +} + +// ShowErrorWithMessage displays the given error nicely with the given message +func ShowErrorWithMessage(ctx context.Context, err error, message string) { + if err == nil { + return + } + statushooks.Done(ctx) + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %s - %v\n", constants.ColoredErr, message, TransformErrorToTailpipe(err)) +} + +// TransformErrorToTailpipe removes the pq: and rpc error prefixes along +// with all the unnecessary information that comes from the +// drivers and libraries +func TransformErrorToTailpipe(err error) error { + if err == nil { + return nil + } + + var errString string + if strings.Contains(err.Error(), "flowpipe service is unreachable") { + errString = strings.Split(err.Error(), ": ")[1] + } else { + errString = strings.TrimSpace(err.Error()) + } + + // an error that originated from our database/sql driver (always prefixed with "ERROR:") + if strings.HasPrefix(errString, "ERROR:") { + errString = strings.TrimSpace(strings.TrimPrefix(errString, "ERROR:")) + } + // if this is an RPC Error while talking with the plugin + if strings.HasPrefix(errString, "rpc error") { + // trim out "rpc error: code = Unknown desc =" + errString = strings.TrimPrefix(errString, "rpc error: code = Unknown desc =") + } + return errors.New(strings.TrimSpace(errString)) +} + +func IsCancelledError(err error) bool { + return errors.Is(err, context.Canceled) || strings.Contains(err.Error(), "canceling statement due to user request") +} + +func ShowWarning(warning string) { + if len(warning) == 0 { + return + } + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %v\n", constants.ColoredWarn, warning) +} + +func PrefixError(err error, prefix string) error { + return fmt.Errorf("%s: %s\n", prefix, TransformErrorToTailpipe(err).Error()) +} + +// isMachineReadableOutput checks if the current output format is machine readable (CSV or JSON) +func isMachineReadableOutput() bool { + outputFormat := viper.GetString(constants.ArgOutput) + return outputFormat == constants.OutputFormatCSV || outputFormat == constants.OutputFormatJSON +} + +func GetWarningOutputStream() io.Writer { + if isMachineReadableOutput() { + // For machine-readable formats, output warnings and errors to stderr + return os.Stderr + } + // For all other formats, use stdout + return os.Stdout +} diff --git a/internal/interactive/interactive_client.go b/internal/interactive/interactive_client.go index 753f0b15..6e05cafc 100644 --- a/internal/interactive/interactive_client.go +++ b/internal/interactive/interactive_client.go @@ -17,10 +17,10 @@ import ( "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/metaquery" "github.com/turbot/tailpipe/internal/query" ) @@ -346,7 +346,7 @@ func (c *InteractiveClient) executor(ctx context.Context, line string) { func (c *InteractiveClient) executeQuery(ctx context.Context, queryCtx context.Context, resolvedQuery *ResolvedQuery) { _, err := query.ExecuteQuery(queryCtx, resolvedQuery.ExecuteSQL, c.db) if err != nil { - error_helpers.ShowError(ctx, error_helpers.HandleCancelError(err)) + error_helpers.ShowError(ctx, err) } } diff --git a/internal/interactive/run.go b/internal/interactive/run.go index 2bfa5faa..4c17435e 100644 --- a/internal/interactive/run.go +++ b/internal/interactive/run.go @@ -3,8 +3,8 @@ package interactive import ( "context" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // RunInteractivePrompt starts the interactive query prompt diff --git a/internal/plugin/errors.go b/internal/plugin/errors.go index 00c145fa..ce534b08 100644 --- a/internal/plugin/errors.go +++ b/internal/plugin/errors.go @@ -2,7 +2,6 @@ package plugin import ( "errors" - "github.com/turbot/pipe-fittings/v2/error_helpers" "strings" ) @@ -13,8 +12,6 @@ func cleanupPluginError(err error) error { if err == nil { return nil } - // transform to a context - err = error_helpers.HandleCancelError(err) errString := strings.TrimSpace(err.Error()) diff --git a/internal/plugin/plugin_manager.go b/internal/plugin/plugin_manager.go index 0080f265..2a619bfc 100644 --- a/internal/plugin/plugin_manager.go +++ b/internal/plugin/plugin_manager.go @@ -3,8 +3,6 @@ package plugin import ( "context" "fmt" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "log/slog" "math/rand/v2" "os" @@ -14,6 +12,9 @@ import ( "sync" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/hashicorp/go-hclog" goplugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/go-version" @@ -22,7 +23,6 @@ import ( gokithelpers "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/app_specific" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/installationstate" pociinstaller "github.com/turbot/pipe-fittings/v2/ociinstaller" @@ -38,6 +38,7 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/types" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/helpers" "github.com/turbot/tailpipe/internal/ociinstaller" "google.golang.org/protobuf/types/known/timestamppb" @@ -151,7 +152,7 @@ func (p *PluginManager) Collect(ctx context.Context, partition *config.Partition collectResponse, err := tablePluginClient.Collect(req) if err != nil { - return nil, fmt.Errorf("error starting collection for plugin %s: %w", tablePluginClient.Name, error_helpers.TransformErrorToSteampipe(err)) + return nil, fmt.Errorf("error starting collection for plugin %s: %w", tablePluginClient.Name, error_helpers.TransformErrorToTailpipe(err)) } // start a goroutine to read the eventStream and listen to file events @@ -219,7 +220,7 @@ func (p *PluginManager) getSupportedOperations(tablePluginClient *grpc.PluginCli } // Describe starts the plugin if needed, and returns the plugin description, including description of any custom formats -func (p *PluginManager) Describe(ctx context.Context, pluginName string, opts ...DescribeOpts) (*types.DescribeResponse, error) { +func (p *PluginManager) Describe(_ context.Context, pluginName string, opts ...DescribeOpts) (*types.DescribeResponse, error) { // build plugin ref from the name pluginDef := pplugin.NewPlugin(pluginName) @@ -278,7 +279,7 @@ func (p *PluginManager) UpdateCollectionState(ctx context.Context, partition *co _, err = pluginClient.UpdateCollectionState(req) if err != nil { - return fmt.Errorf("error updating collection state for plugin %s: %w", pluginClient.Name, error_helpers.TransformErrorToSteampipe(err)) + return fmt.Errorf("error updating collection state for plugin %s: %w", pluginClient.Name, error_helpers.TransformErrorToTailpipe(err)) } // just return - the observer is responsible for waiting for completion diff --git a/internal/query/execute.go b/internal/query/execute.go index 31c73877..653dd7cf 100644 --- a/internal/query/execute.go +++ b/internal/query/execute.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/query" "github.com/turbot/pipe-fittings/v2/querydisplay" "github.com/turbot/pipe-fittings/v2/queryresult" @@ -17,6 +16,7 @@ import ( "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) func RunBatchSession(ctx context.Context, args []string, db *database.DuckDb) (int, []error) { diff --git a/main.go b/main.go index f923bd1e..dbd800f9 100644 --- a/main.go +++ b/main.go @@ -7,11 +7,11 @@ import ( "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/cmd" "github.com/turbot/tailpipe/internal/cmdconfig" localconstants "github.com/turbot/tailpipe/internal/constants" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) var exitCode int From cee6e42f3aacd2916d5c2ae711bb7cd349561209 Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Tue, 16 Sep 2025 19:06:05 +0530 Subject: [PATCH 41/61] Ducklake migration (#552) --- cmd/collect.go | 8 +- cmd/compact.go | 5 +- cmd/connect.go | 8 +- cmd/format.go | 20 +- cmd/partition.go | 23 +- cmd/plugin.go | 42 +- cmd/source.go | 17 +- cmd/table.go | 12 +- go.mod | 10 +- go.sum | 6 +- internal/cmdconfig/cmd_hooks.go | 30 +- internal/database/convertor.go | 73 +++- internal/database/convertor_ducklake.go | 176 -------- internal/database/ducklake_table.go | 107 +++++ internal/database/tables.go | 187 +++++++++ internal/migration/migration.go | 511 ++++++++++++++++++++++++ internal/migration/status.go | 125 ++++++ internal/migration/utils.go | 50 +++ 18 files changed, 1147 insertions(+), 263 deletions(-) delete mode 100644 internal/database/convertor_ducklake.go create mode 100644 internal/database/ducklake_table.go create mode 100644 internal/migration/migration.go create mode 100644 internal/migration/status.go create mode 100644 internal/migration/utils.go diff --git a/cmd/collect.go b/cmd/collect.go index 2576e764..332847fc 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -16,7 +16,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" @@ -61,8 +60,9 @@ Every time you run tailpipe collect, Tailpipe refreshes its views over all colle } func runCollectCmd(cmd *cobra.Command, args []string) { - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() + ctx, cancel := context.WithCancel(ctx) //nolint:govet // cancel is needed for the doCollect func var err error defer func() { @@ -83,7 +83,7 @@ func runCollectCmd(cmd *cobra.Command, args []string) { // if diagnostic mode is set, print out config and return if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { localcmdconfig.DisplayConfig() - return + return //nolint:govet // this is explicitly used in tests } err = doCollect(ctx, cancel, args) diff --git a/cmd/compact.go b/cmd/compact.go index c777b203..a02605c0 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -13,7 +13,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" @@ -38,8 +37,8 @@ func compactCmd() *cobra.Command { func runCompactCmd(cmd *cobra.Command, args []string) { var err error - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() defer func() { if r := recover(); r != nil { diff --git a/cmd/connect.go b/cmd/connect.go index a7b4da56..7b83a59d 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -10,8 +10,6 @@ import ( "strings" "time" - "golang.org/x/exp/maps" - "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/thediveo/enumflag/v2" @@ -19,7 +17,6 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" "github.com/turbot/pipe-fittings/v2/connection" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/parse" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" @@ -27,6 +24,7 @@ import ( "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" error_helpers "github.com/turbot/tailpipe/internal/error_helpers" + "golang.org/x/exp/maps" ) // variable used to assign the output mode flag @@ -96,8 +94,8 @@ The generated script can be used with DuckDB: func runConnectCmd(cmd *cobra.Command, _ []string) { var err error var initFilePath string - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() defer func() { if r := recover(); r != nil { diff --git a/cmd/format.go b/cmd/format.go index 5f5d40de..94f8759d 100644 --- a/cmd/format.go +++ b/cmd/format.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "fmt" "os" "strings" @@ -11,7 +10,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" @@ -69,9 +67,8 @@ func formatListCmd() *cobra.Command { } func runFormatListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runFormatListCmd start") var err error defer func() { @@ -86,7 +83,7 @@ func runFormatListCmd(cmd *cobra.Command, args []string) { } else { error_helpers.ShowError(ctx, err) } - setExitCodeForFormatError(err,1) + setExitCodeForFormatError(err) } }() @@ -133,9 +130,8 @@ func formatShowCmd() *cobra.Command { } func runFormatShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runFormatShowCmd start") var err error defer func() { @@ -150,7 +146,7 @@ func runFormatShowCmd(cmd *cobra.Command, args []string) { } else { error_helpers.ShowError(ctx, err) } - setExitCodeForFormatError(err, 1) + setExitCodeForFormatError(err) } }() @@ -172,7 +168,7 @@ func runFormatShowCmd(cmd *cobra.Command, args []string) { } } -func setExitCodeForFormatError(err error, nonCancelCode int) { +func setExitCodeForFormatError(err error) { // set exit code only if an error occurred and no exit code is already set if exitCode != 0 || err == nil { return @@ -183,5 +179,5 @@ func setExitCodeForFormatError(err error, nonCancelCode int) { return } // no dedicated format exit code exists yet; use generic nonzero failure - exitCode = nonCancelCode + exitCode = 1 } diff --git a/cmd/partition.go b/cmd/partition.go index 78029017..3a72c583 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -15,7 +15,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" @@ -76,9 +75,8 @@ func partitionListCmd() *cobra.Command { } func runPartitionListCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runPartitionListCmd start") var err error defer func() { @@ -103,7 +101,6 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { return } - // open a readonly db connection db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) error_helpers.FailOnError(err) defer db.Close() @@ -146,10 +143,10 @@ func partitionShowCmd() *cobra.Command { } func runPartitionShowCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runPartitionShowCmd start") var err error defer func() { @@ -233,10 +230,10 @@ func partitionDeleteCmd() *cobra.Command { } func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() var err error defer func() { if r := recover(); r != nil { diff --git a/cmd/plugin.go b/cmd/plugin.go index ba7dc795..25879a51 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -15,7 +15,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/installationstate" pociinstaller "github.com/turbot/pipe-fittings/v2/ociinstaller" @@ -238,10 +237,10 @@ var pluginInstallSteps = []string{ } func runPluginInstallCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runPluginInstallCmd install") var err error defer func() { @@ -386,10 +385,10 @@ func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string } func runPluginUpdateCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runPluginUpdateCmd start") var err error defer func() { @@ -667,10 +666,10 @@ func installPlugin(ctx context.Context, resolvedPlugin pplugin.ResolvedPluginVer } func runPluginUninstallCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runPluginUninstallCmd uninstall") var err error @@ -768,9 +767,8 @@ func resolveUpdatePluginsFromArgs(args []string) ([]string, error) { } func runPluginListCmd(cmd *cobra.Command, _ []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runPluginListCmd list") @@ -822,18 +820,18 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { } func runPluginShowCmd(cmd *cobra.Command, args []string) { + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() + // we expect 1 argument, the plugin name if len(args) != 1 { - error_helpers.ShowError(cmd.Context(), fmt.Errorf("you need to provide the name of a plugin")) + error_helpers.ShowError(ctx, fmt.Errorf("you need to provide the name of a plugin")) exitCode = pconstants.ExitCodeInsufficientOrWrongInputs return } - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) - utils.LogTime("runPluginShowCmd start") // Clean up plugin temporary directories from previous crashes/interrupted installations diff --git a/cmd/source.go b/cmd/source.go index ecc2785a..dcb9d5e6 100644 --- a/cmd/source.go +++ b/cmd/source.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "fmt" "os" "strings" @@ -11,13 +10,12 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" + "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" - error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) func sourceCmd() *cobra.Command { @@ -66,9 +64,8 @@ func sourceListCmd() *cobra.Command { } func runSourceListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runSourceListCmd start") var err error defer func() { @@ -130,10 +127,10 @@ func sourceShowCmd() *cobra.Command { } func runSourceShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - //TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a cancellation error. Cancellation won't work right now - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runSourceShowCmd start") var err error defer func() { diff --git a/cmd/table.go b/cmd/table.go index 5200e0b9..7574d84d 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "fmt" "os" "strings" @@ -11,7 +10,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" @@ -68,9 +66,8 @@ func tableListCmd() *cobra.Command { } func runTableListCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runSourceListCmd start") var err error defer func() { @@ -138,9 +135,8 @@ func tableShowCmd() *cobra.Command { } func runTableShowCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runTableShowCmd start") var err error defer func() { diff --git a/go.mod b/go.mod index c7d1e144..9654b36e 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ toolchain go1.24.0 replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 - // github.com/turbot/pipe-fittings/v2 => ../pipe-fittings - //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core - github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk +// github.com/turbot/pipe-fittings/v2 => ../pipe-fittings +//github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core +// github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) require ( @@ -19,8 +19,8 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 - github.com/turbot/pipe-fittings/v2 v2.7.0-rc.0 - github.com/turbot/tailpipe-plugin-sdk v0.9.2 + github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1 + github.com/turbot/tailpipe-plugin-sdk v0.9.3 github.com/zclconf/go-cty v1.16.3 golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 ) diff --git a/go.sum b/go.sum index 19ecc374..a2c06c77 100644 --- a/go.sum +++ b/go.sum @@ -1308,12 +1308,14 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.0 h1:p9/Hf0BNNjZVs5C4AqZlQgmihKt/nboh5OrfGcH8Mhk= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.0/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= +github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1 h1:4Y/51FNwJqavbz/O8T8NQkpp6+roiyoT7BrD/GLR2FU= +github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= github.com/turbot/tailpipe-plugin-core v0.2.10/go.mod h1:dHzPUR1p5GksSvDqqEeZEvvJX6wTEwK/ZDev//9nSLw= +github.com/turbot/tailpipe-plugin-sdk v0.9.3 h1:JpGpGPwehqdXnRO3aqkQTpd96Vx2blY+AkXP8lYB32g= +github.com/turbot/tailpipe-plugin-sdk v0.9.3/go.mod h1:Egojp0j7+th/4Bh6muMuF6aZa5iE3MuiJ4pzBo0J2mg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/internal/cmdconfig/cmd_hooks.go b/internal/cmdconfig/cmd_hooks.go index 78e5b478..c07cfac7 100644 --- a/internal/cmdconfig/cmd_hooks.go +++ b/internal/cmdconfig/cmd_hooks.go @@ -13,6 +13,7 @@ import ( "github.com/turbot/pipe-fittings/v2/app_specific" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/filepaths" pparse "github.com/turbot/pipe-fittings/v2/parse" @@ -22,6 +23,7 @@ import ( "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/logger" + "github.com/turbot/tailpipe/internal/migration" "github.com/turbot/tailpipe/internal/parse" "github.com/turbot/tailpipe/internal/plugin" ) @@ -58,7 +60,33 @@ func preRunHook(cmd *cobra.Command, args []string) error { // set the max memory if specified setMemoryLimit() - return nil + // create cancel context and set back on command + baseCtx := cmd.Context() + ctx, cancel := context.WithCancel(baseCtx) + + // start the cancel handler to call cancel on interrupt signals + contexthelpers.StartCancelHandler(cancel) + cmd.SetContext(ctx) + + // migrate legacy data to DuckLake: + // Prior to Tailpipe v0.7.0 we stored data as native Parquet files alongside a tailpipe.db + // (DuckDB) that defined SQL views. From v0.7.0 onward Tailpipe uses DuckLake, which + // introduces a metadata database (metadata.sqlite). We run a one-time migration here to + // move existing user data into DuckLake’s layout so it can be queried and managed via + // the new metadata model. + // start migration + err := migration.MigrateDataToDucklake(cmd.Context()) + if error_helpers.IsContextCancelledError(err) { + // suppress Cobra's usage/errors only for this cancelled invocation + // Cobra prints usage when a command returns an error. The cancellation returns an error (context cancelled) + // from preRun, so Cobra assumes "user error" and shows help. + // This conditional block sets cmd.SilenceUsage = true and cmd.SilenceErrors = true only for cancellation, + // telling Cobra "don't print usage or re-print the error". Without it, you get the usage dump. + cmd.SilenceUsage = true + cmd.SilenceErrors = true + } + // return (possibly nil) error from migration + return err } func displayStartupLog() { diff --git a/internal/database/convertor.go b/internal/database/convertor.go index c34e7bcc..36e873f5 100644 --- a/internal/database/convertor.go +++ b/internal/database/convertor.go @@ -2,12 +2,14 @@ package database import ( "context" + "database/sql" "errors" "fmt" "log/slog" "sync" "sync/atomic" + "github.com/turbot/pipe-fittings/v2/backend" "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe/internal/config" ) @@ -174,8 +176,8 @@ func (w *Converter) onFirstChunk(executionId string, chunk int32) error { // err will be returned by the parent function return err } - // create the DuckDB table fpr this partition if it does not already exist - if err := w.ensureDuckLakeTable(w.Partition.TableName); err != nil { + // create the DuckDB table for this partition if it does not already exist + if err := EnsureDuckLakeTable(w.conversionSchema.Columns, w.db, w.Partition.TableName); err != nil { return fmt.Errorf("failed to create DuckDB table: %w", err) } w.readJsonQueryFormat = buildReadJsonQueryFormat(w.conversionSchema, w.Partition) @@ -225,3 +227,70 @@ func (w *Converter) updateRowCount(count int64) { // call the status function with the new row count w.statusFunc(atomic.LoadInt64(&w.rowCount), atomic.LoadInt64(&w.failedRowCount)) } + +// CheckTableSchema checks if the specified table exists in the DuckDB database and compares its schema with the +// provided schema. +// it returns a TableSchemaStatus indicating whether the table exists, whether the schema matches, and any differences. +// THis is not used at present but will be used when we implement ducklake schema evolution handling +func (w *Converter) CheckTableSchema(db *sql.DB, tableName string, conversionSchema schema.ConversionSchema) (TableSchemaStatus, error) { + // Check if table exists + exists, err := w.tableExists(db, tableName) + if err != nil { + return TableSchemaStatus{}, err + } + + if !exists { + return TableSchemaStatus{}, nil + } + + // Get existing schema + existingSchema, err := w.getTableSchema(db, tableName) + if err != nil { + return TableSchemaStatus{}, fmt.Errorf("failed to retrieve schema: %w", err) + } + + // Use constructor to create status from comparison + diff := NewTableSchemaStatusFromComparison(existingSchema, conversionSchema) + return diff, nil +} + +func (w *Converter) tableExists(db *sql.DB, tableName string) (bool, error) { + sanitizedTableName, err := backend.SanitizeDuckDBIdentifier(tableName) + if err != nil { + return false, fmt.Errorf("invalid table name %s: %w", tableName, err) + } + //nolint:gosec // table name is sanitized + query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", sanitizedTableName) + var exists int + if err := db.QueryRow(query).Scan(&exists); err != nil { + return false, err + } + return exists == 1, nil +} + +func (w *Converter) getTableSchema(db *sql.DB, tableName string) (map[string]schema.ColumnSchema, error) { + query := fmt.Sprintf("pragma table_info(%s);", tableName) + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + schemaMap := make(map[string]schema.ColumnSchema) + for rows.Next() { + var name, dataType string + var notNull, pk int + var dfltValue sql.NullString + + if err := rows.Scan(&name, &dataType, ¬Null, &dfltValue, &pk); err != nil { + return nil, err + } + + schemaMap[name] = schema.ColumnSchema{ + ColumnName: name, + Type: dataType, + } + } + + return schemaMap, nil +} diff --git a/internal/database/convertor_ducklake.go b/internal/database/convertor_ducklake.go deleted file mode 100644 index bb80dd1c..00000000 --- a/internal/database/convertor_ducklake.go +++ /dev/null @@ -1,176 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "strings" - - "github.com/turbot/pipe-fittings/v2/backend" - "github.com/turbot/tailpipe-plugin-sdk/constants" - "github.com/turbot/tailpipe-plugin-sdk/schema" -) - -// determine whether we have a ducklake table for this table, and if so, whether it needs schema updating -func (w *Converter) ensureDuckLakeTable(tableName string) error { - query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", tableName) - var exists bool - if err := w.db.QueryRow(query).Scan(&exists); err != nil { - return err - } - if !exists { - return w.createDuckLakeTable(tableName) - } - return nil -} - -// createDuckLakeTable creates a DuckLake table based on the ConversionSchema -func (w *Converter) createDuckLakeTable(tableName string) error { - - // Generate the CREATE TABLE SQL - createTableSQL := w.buildCreateDucklakeTableSQL(tableName) - - // Execute the CREATE TABLE statement - _, err := w.db.Exec(createTableSQL) - if err != nil { - return fmt.Errorf("failed to create table %s: %w", tableName, err) - } - - // Set partitioning using ALTER TABLE - // partition by the partition, index, year and month - partitionColumns := []string{constants.TpPartition, constants.TpIndex, fmt.Sprintf("year(%s)", constants.TpTimestamp), fmt.Sprintf("month(%s)", constants.TpTimestamp)} - alterTableSQL := fmt.Sprintf(`alter table "%s" set partitioned by (%s);`, - tableName, - strings.Join(partitionColumns, ", ")) - - _, err = w.db.Exec(alterTableSQL) - if err != nil { - return fmt.Errorf("failed to set partitioning for table %s: %w", tableName, err) - } - - return nil -} - -// buildCreateDucklakeTableSQL generates the CREATE TABLE SQL statement based on the ConversionSchema -func (w *Converter) buildCreateDucklakeTableSQL(tableName string) string { - // Build column definitions in sorted order - var columnDefinitions []string - for _, column := range w.conversionSchema.Columns { - columnDef := w.buildColumnDefinition(column) - columnDefinitions = append(columnDefinitions, columnDef) - } - - return fmt.Sprintf(`create table if not exists "%s" ( -%s -);`, - tableName, - strings.Join(columnDefinitions, ",\n")) -} - -// buildColumnDefinition generates the SQL definition for a single column -func (w *Converter) buildColumnDefinition(column *schema.ColumnSchema) string { - columnName := fmt.Sprintf("\"%s\"", column.ColumnName) - - // Handle different column types - switch column.Type { - case "struct": - // For struct types, we need to build the struct definition - structDef := w.buildStructDefinition(column) - return fmt.Sprintf("\t%s %s", columnName, structDef) - case "json": - // json type - return fmt.Sprintf("\t%s json", columnName) - default: - // For scalar types, just use the type directly (lower case) - return fmt.Sprintf("\t%s %s", columnName, strings.ToLower(column.Type)) - } -} - -// buildStructDefinition generates the SQL struct definition for a struct column -func (w *Converter) buildStructDefinition(column *schema.ColumnSchema) string { - if len(column.StructFields) == 0 { - return "struct" - } - - var fieldDefinitions []string - for _, field := range column.StructFields { - fieldName := fmt.Sprintf("\"%s\"", field.ColumnName) - fieldType := strings.ToLower(field.Type) - - if field.Type == "struct" { - // Recursively build nested struct definition - nestedStruct := w.buildStructDefinition(field) - fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, nestedStruct)) - } else { - fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, fieldType)) - } - } - - return fmt.Sprintf("struct(%s)", strings.Join(fieldDefinitions, ", ")) -} - -// CheckTableSchema checks if the specified table exists in the DuckDB database and compares its schema with the -// provided schema. -// it returns a TableSchemaStatus indicating whether the table exists, whether the schema matches, and any differences. -// THis is not used at present but will be used when we implement ducklake schema evolution handling -func (w *Converter) CheckTableSchema(db *sql.DB, tableName string, conversionSchema schema.ConversionSchema) (TableSchemaStatus, error) { - // Check if table exists - exists, err := w.tableExists(db, tableName) - if err != nil { - return TableSchemaStatus{}, err - } - - if !exists { - return TableSchemaStatus{}, nil - } - - // Get existing schema - existingSchema, err := w.getTableSchema(db, tableName) - if err != nil { - return TableSchemaStatus{}, fmt.Errorf("failed to retrieve schema: %w", err) - } - - // Use constructor to create status from comparison - diff := NewTableSchemaStatusFromComparison(existingSchema, conversionSchema) - return diff, nil -} - -func (w *Converter) tableExists(db *sql.DB, tableName string) (bool, error) { - sanitizedTableName, err := backend.SanitizeDuckDBIdentifier(tableName) - if err != nil { - return false, fmt.Errorf("invalid table name %s: %w", tableName, err) - } - //nolint:gosec // table name is sanitized - query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", sanitizedTableName) - var exists int - if err := db.QueryRow(query).Scan(&exists); err != nil { - return false, err - } - return exists == 1, nil -} - -func (w *Converter) getTableSchema(db *sql.DB, tableName string) (map[string]schema.ColumnSchema, error) { - query := fmt.Sprintf("pragma table_info(%s);", tableName) - rows, err := db.Query(query) - if err != nil { - return nil, err - } - defer rows.Close() - - schemaMap := make(map[string]schema.ColumnSchema) - for rows.Next() { - var name, dataType string - var notNull, pk int - var dfltValue sql.NullString - - if err := rows.Scan(&name, &dataType, ¬Null, &dfltValue, &pk); err != nil { - return nil, err - } - - schemaMap[name] = schema.ColumnSchema{ - ColumnName: name, - Type: dataType, - } - } - - return schemaMap, nil -} diff --git a/internal/database/ducklake_table.go b/internal/database/ducklake_table.go new file mode 100644 index 00000000..b774b17c --- /dev/null +++ b/internal/database/ducklake_table.go @@ -0,0 +1,107 @@ +package database + +import ( + "fmt" + "strings" + + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +// EnsureDuckLakeTable determines whether we have a ducklake table for this table, and if so, whether it needs schema updating +func EnsureDuckLakeTable(columns []*schema.ColumnSchema, db *DuckDb, tableName string) error { + query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", tableName) + var exists bool + if err := db.QueryRow(query).Scan(&exists); err != nil { + return err + } + if !exists { + return createDuckLakeTable(columns, db, tableName) + } + return nil +} + +// createDuckLakeTable creates a DuckLake table based on the ConversionSchema +func createDuckLakeTable(columns []*schema.ColumnSchema, db *DuckDb, tableName string) error { + + // Generate the CREATE TABLE SQL + createTableSQL := buildCreateDucklakeTableSQL(columns, tableName) + + // Execute the CREATE TABLE statement + _, err := db.Exec(createTableSQL) + if err != nil { + return fmt.Errorf("failed to create table %s: %w", tableName, err) + } + + // Set partitioning using ALTER TABLE + // partition by the partition, index, year and month + partitionColumns := []string{constants.TpPartition, constants.TpIndex, fmt.Sprintf("year(%s)", constants.TpTimestamp), fmt.Sprintf("month(%s)", constants.TpTimestamp)} + alterTableSQL := fmt.Sprintf(`alter table "%s" set partitioned by (%s);`, + tableName, + strings.Join(partitionColumns, ", ")) + + _, err = db.Exec(alterTableSQL) + if err != nil { + return fmt.Errorf("failed to set partitioning for table %s: %w", tableName, err) + } + + return nil +} + +// buildCreateDucklakeTableSQL generates the CREATE TABLE SQL statement based on the ConversionSchema +func buildCreateDucklakeTableSQL(columns []*schema.ColumnSchema, tableName string) string { + // Build column definitions in sorted order + var columnDefinitions []string + for _, column := range columns { + columnDef := buildColumnDefinition(column) + columnDefinitions = append(columnDefinitions, columnDef) + } + + return fmt.Sprintf(`create table if not exists "%s" ( +%s +);`, + tableName, + strings.Join(columnDefinitions, ",\n")) +} + +// buildColumnDefinition generates the SQL definition for a single column +func buildColumnDefinition(column *schema.ColumnSchema) string { + columnName := fmt.Sprintf("\"%s\"", column.ColumnName) + + // Handle different column types + switch column.Type { + case "struct": + // For struct types, we need to build the struct definition + structDef := buildStructDefinition(column) + return fmt.Sprintf("\t%s %s", columnName, structDef) + case "json": + // json type + return fmt.Sprintf("\t%s json", columnName) + default: + // For scalar types, just use the type directly (lower case) + return fmt.Sprintf("\t%s %s", columnName, strings.ToLower(column.Type)) + } +} + +// buildStructDefinition generates the SQL struct definition for a struct column +func buildStructDefinition(column *schema.ColumnSchema) string { + if len(column.StructFields) == 0 { + return "struct" + } + + var fieldDefinitions []string + for _, field := range column.StructFields { + fieldName := fmt.Sprintf("\"%s\"", field.ColumnName) + fieldType := strings.ToLower(field.Type) + + if field.Type == "struct" { + // Recursively build nested struct definition + nestedStruct := buildStructDefinition(field) + fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, nestedStruct)) + } else { + fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, fieldType)) + } + } + + return fmt.Sprintf("struct(%s)", strings.Join(fieldDefinitions, ", ")) +} diff --git a/internal/database/tables.go b/internal/database/tables.go index e2684719..4d8f7acb 100644 --- a/internal/database/tables.go +++ b/internal/database/tables.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/tailpipe-plugin-sdk/schema" ) // GetTables returns the list of tables in the DuckLake metadata catalog @@ -64,3 +65,189 @@ order by c.column_name;`, constants.DuckLakeMetadataCatalog, constants.DuckLakeM return schema, nil } + +// GetLegacyTableViews retrieves the names of all table views in the legacy database(tailpipe.db) file +func GetLegacyTableViews(ctx context.Context, db *DuckDb) ([]string, error) { + query := "select table_name from information_schema.tables where table_type='VIEW';" + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to get table views: %w", err) + } + defer rows.Close() + + var tableViews []string + for rows.Next() { + var tableView string + err = rows.Scan(&tableView) + if err != nil { + return nil, fmt.Errorf("failed to scan table view: %w", err) + } + tableViews = append(tableViews, tableView) + } + return tableViews, nil +} + +// GetLegacyTableViewSchema retrieves the schema of a table view in the legacy database(tailpipe.db) file +func GetLegacyTableViewSchema(ctx context.Context, viewName string, db *DuckDb) (*schema.TableSchema, error) { + query := ` + select column_name, data_type + from information_schema.columns + where table_name = ? ORDER BY columns.column_name; + ` + rows, err := db.QueryContext(ctx, query, viewName) + if err != nil { + return nil, fmt.Errorf("failed to get view schema for %s: %w", viewName, err) + } + defer rows.Close() + + ts := &schema.TableSchema{ + Name: viewName, + Columns: []*schema.ColumnSchema{}, + } + for rows.Next() { + // here each row is a column, so we need to populate the TableSchema.Columns, particularly the + // ColumnName, Type and StructFields + var columnName, columnType string + err = rows.Scan(&columnName, &columnType) + if err != nil { + return nil, fmt.Errorf("failed to scan column schema: %w", err) + } + + col := buildColumnSchema(columnName, columnType) + ts.Columns = append(ts.Columns, col) + } + + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over view schema rows: %w", err) + } + + return ts, nil +} + +// buildColumnSchema constructs a ColumnSchema from a DuckDB data type string. +// It handles primitive types as well as struct and struct[] recursively, populating StructFields. +func buildColumnSchema(columnName string, duckdbType string) *schema.ColumnSchema { + t := strings.TrimSpace(duckdbType) + lower := strings.ToLower(t) + + // Helper to set basic column properties + newCol := func(name, typ string, children []*schema.ColumnSchema) *schema.ColumnSchema { + return &schema.ColumnSchema{ + ColumnName: name, + SourceName: name, + Type: typ, + StructFields: children, + } + } + + // Detect struct or struct[] + if strings.HasPrefix(lower, "struct(") || strings.HasPrefix(lower, "struct ") { + isArray := false + // Handle optional [] suffix indicating array of struct + if strings.HasSuffix(lower, ")[]") { + isArray = true + } + // Extract inner content between the first '(' and the matching ')' + start := strings.Index(t, "(") + end := strings.LastIndex(t, ")") + inner := "" + if start >= 0 && end > start { + inner = strings.TrimSpace(t[start+1 : end]) + } + + fields := parseStructFields(inner) + typ := "struct" + if isArray { + typ = "struct[]" + } + return newCol(columnName, typ, fields) + } + + // Primitive or other complex types - just set as-is + return newCol(columnName, lower, nil) +} + +// parseStructFields parses the content inside a DuckDB struct(...) definition into ColumnSchemas. +// It supports nested struct/struct[] types by recursively building ColumnSchemas for child fields. +func parseStructFields(inner string) []*schema.ColumnSchema { + // Split by top-level commas (not within nested parentheses) + parts := splitTopLevel(inner, ',') + var fields []*schema.ColumnSchema + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + // parse field name (optionally quoted) and type + name, typ := parseFieldNameAndType(p) + if name == "" || typ == "" { + continue + } + col := buildColumnSchema(name, typ) + fields = append(fields, col) + } + return fields +} + +// parseFieldNameAndType parses a single struct field spec of the form: +// +// name type +// "name with spaces" type +// +// where type may itself be struct(...)[]. Returns name and the raw type string. +func parseFieldNameAndType(s string) (string, string) { + s = strings.TrimSpace(s) + if s == "" { + return "", "" + } + if s[0] == '"' { + // quoted name + // find closing quote + i := 1 + for i < len(s) && s[i] != '"' { + i++ + } + if i >= len(s) { + return "", "" + } + name := s[1:i] + rest := strings.TrimSpace(s[i+1:]) + // rest should start with the type + return name, rest + } + // unquoted name up to first space + idx := strings.IndexFunc(s, func(r rune) bool { return r == ' ' || r == '\t' }) + if idx == -1 { + // no type specified + return "", "" + } + name := strings.TrimSpace(s[:idx]) + typ := strings.TrimSpace(s[idx+1:]) + return name, typ +} + +// splitTopLevel splits s by sep, ignoring separators enclosed in parentheses. +func splitTopLevel(s string, sep rune) []string { + var res []string + level := 0 + start := 0 + for i, r := range s { + switch r { + case '(': + level++ + case ')': + if level > 0 { + level-- + } + } + if r == sep && level == 0 { + res = append(res, strings.TrimSpace(s[start:i])) + start = i + 1 + } + } + // add last segment + if start <= len(s) { + res = append(res, strings.TrimSpace(s[start:])) + } + return res +} diff --git a/internal/migration/migration.go b/internal/migration/migration.go new file mode 100644 index 00000000..bddd383a --- /dev/null +++ b/internal/migration/migration.go @@ -0,0 +1,511 @@ +package migration + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/briandowns/spinner" + perr "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/filepaths" +) + +// MigrateDataToDucklake performs migration of views from tailpipe.db and associated parquet files +// into the new DuckLake metadata catalog +func MigrateDataToDucklake(ctx context.Context) error { + // Determine source and migration directories + dataDefaultDir := config.GlobalWorkspaceProfile.GetDataDir() + migratingDefaultDir := config.GlobalWorkspaceProfile.GetMigratingDir() + // failed dir is derived via GetMigrationFailedDir() where needed + + var matchedTableDirs, unmatchedTableDirs []string + status := NewMigrationStatus(0) + cancelledHandled := false + defer func() { + if ctx.Err() != nil && !cancelledHandled { + _ = onCancelled(status) + } + }() + + // if the ~/.tailpipe/data directory has a .db file, it means that this is the first time we are migrating + // if the ~/.tailpipe/migration/migrating directory has a .db file, it means that this is a resume migration + initialMigration := hasTailpipeDb(dataDefaultDir) + continueMigration := hasTailpipeDb(migratingDefaultDir) + + // validate: both should not be true - return that last migration left things in a bad state + if initialMigration && continueMigration { + return fmt.Errorf("Invalid migration state: found tailpipe.db in both data and migrating directories. This should not happen. Please contact Turbot support for assistance.") + } + + // STEP 1: Check if migration is needed + // We need to migrate if it is the first time we are migrating or if we are resuming a migration + if !initialMigration && !continueMigration { + slog.Info("No migration needed - no tailpipe.db found in data or migrating directory") + return nil + } + + // Choose DB path for discovery + // If this is the first time we are migrating, we need to use .db file from the ~/.tailpipe/data directory + // If this is a resume migration, we need to use .db file from the ~/.tailpipe/migration/migrating directory + var discoveryDbPath string + if initialMigration { + discoveryDbPath = filepath.Join(dataDefaultDir, "tailpipe.db") + } else { + discoveryDbPath = filepath.Join(migratingDefaultDir, "tailpipe.db") + } + + // STEP 2: Discover legacy tables and their schemas (from chosen DB path) + // This returns the list of views and a map of view name to its schema + views, schemas, err := discoverLegacyTablesAndSchemas(ctx, discoveryDbPath) + if err != nil { + return fmt.Errorf("failed to discover legacy tables: %w", err) + } + + slog.Info("Views: ", "views", views) + slog.Info("Schemas: ", "schemas", schemas) + + // STEP 3: If this is the first time we are migrating(tables in ~/.tailpipe/data) then move the whole contents of data dir + // into ~/.tailpipe/migration/migrating respecting the same folder structure. + // First-run: transactionally move contents via moveDirContents: copy data to migrating, move tailpipe.db to migrated, + // then empty the original data directory. On any failure, the migrating directory is removed. + if initialMigration { + if err := moveDirContents(ctx, dataDefaultDir, migratingDefaultDir); err != nil { + return err + } + } + + // STEP 4: We have now moved the data into migrating. We have the list of views from the legacy DB. + // We now need to find the matching table directories in migrating/default by scanning migrating/ + // for tp_table=* directories. + // The matching table directories are the ones that have a view in the database. + // The unmatched table directories are the ones that have data(.parquet files) but no view in the database. + // We will move these to migrated/default. + + // set the base directory to ~.tailpipe/migration/migrating/ + baseDir := migratingDefaultDir + matchedTableDirs, unmatchedTableDirs, err = findMatchingTableDirs(baseDir, views) + if err != nil { + return err + } + // move the unmatched table directories to 'unmigrated' + if err = archiveUnmatchedDirs(ctx, unmatchedTableDirs); err != nil { + return err + } + + // Initialize status with total tables to migrate + status.Total = len(matchedTableDirs) + status.update() + + // Pre-compute total parquet files across matched directories + totalFiles, err := countParquetFiles(ctx, matchedTableDirs) + if err == nil { + status.TotalFiles = totalFiles + status.updateFiles() + } + + // Spinner for migration progress + sp := spinner.New( + spinner.CharSets[14], + 100*time.Millisecond, + spinner.WithHiddenCursor(true), + spinner.WithWriter(os.Stdout), + ) + sp.Suffix = fmt.Sprintf(" Migrating tables to DuckLake (%d/%d, %0.1f%%) | parquet files (%d/%d)", status.Migrated, status.Total, status.ProgressPercent, status.MigratedFiles, status.TotalFiles) + sp.Start() + + updateStatus := func(st *MigrationStatus) { + sp.Suffix = fmt.Sprintf(" Migrating tables to DuckLake (%d/%d, %0.1f%%) | parquet files (%d/%d)", st.Migrated, st.Total, st.ProgressPercent, st.MigratedFiles, st.TotalFiles) + } + + // STEP 5: Do Migration: Traverse matched table directories, find leaf nodes with parquet files, + // and perform INSERT within a transaction. On success, move leaf dir to migrated. + err = doMigration(ctx, matchedTableDirs, schemas, status, updateStatus) + sp.Stop() + + // If cancellation arrived after doMigration returned, prefer the CANCELLED outcome + if perr.IsContextCancelledError(ctx.Err()) { + _ = onCancelled(status) + cancelledHandled = true + return ctx.Err() + } + + // Post-migration outcomes + if status.Failed > 0 { + if err := onFailed(status); err != nil { + return err + } + } else { + if err := onSuccessful(status); err != nil { + return err + } + } + + return err +} + +// moveDirContents handles the initial migration move: copy data dir into migrating and move the legacy DB +// into migrated. If any step fails, it removes the migrating directory and shows a support warning. +func moveDirContents(ctx context.Context, dataDefaultDir, migratingDefaultDir string) (err error) { + migratedDir := config.GlobalWorkspaceProfile.GetMigratedDir() + defer func() { + if err != nil { + _ = os.RemoveAll(migratingDefaultDir) + perr.ShowWarning(fmt.Sprintf("Migration initialisation failed. Cleaned up '%s'. Please contact Turbot support.", migratingDefaultDir)) + } + }() + + // 1) Ensure the destination for the DB exists first + // Reason: we will move tailpipe.db after copying data; guaranteeing the target avoids partial moves later. + if err = os.MkdirAll(migratedDir, 0755); err != nil { + return err + } + // 2) Copy ALL data from data/default -> migration/migrating/default (do not delete source yet) + // Reason: copying first keeps the legacy data readable if the process crashes midway. + if err = utils.CopyDir(ctx, dataDefaultDir, migratingDefaultDir); err != nil { + return err + } + // 3) Move the DB file from data/default -> migration/migrated/default + // Reason: once data copy succeeded, moving tailpipe.db signals the backup exists and clarifies resume semantics. + if err = utils.MoveFile(filepath.Join(dataDefaultDir, "tailpipe.db"), filepath.Join(migratedDir, "tailpipe.db")); err != nil { + return err + } + // 4) Empty the original data directory last to emulate an atomic move + // Reason: only after successful copy+db move do we clear the source so we never strand users without their legacy data. + if err = utils.EmptyDir(dataDefaultDir); err != nil { + return err + } + return nil +} + +// discoverLegacyTablesAndSchemas enumerates legacy DuckDB views and, for each view, its schema. +// It returns the list of view names and a map of view name to its schema (column->type). +// If the legacy database contains no views, both return values are empty. +func discoverLegacyTablesAndSchemas(ctx context.Context, dbPath string) ([]string, map[string]*schema.TableSchema, error) { + // open a duckdb connection to the legacy legacyDb + legacyDb, err := database.NewDuckDb(database.WithDbFile(dbPath)) + if err != nil { + return nil, nil, err + } + defer legacyDb.Close() + + views, err := database.GetLegacyTableViews(ctx, legacyDb) + if err != nil || len(views) == 0 { + return []string{}, map[string]*schema.TableSchema{}, err + } + if perr.IsContextCancelledError(ctx.Err()) { + return nil, nil, ctx.Err() + } + + schemas := make(map[string]*schema.TableSchema) + for _, v := range views { + if perr.IsContextCancelledError(ctx.Err()) { + return nil, nil, ctx.Err() + } + // get row count for the view (optional future optimization) and schema + ts, err := database.GetLegacyTableViewSchema(ctx, v, legacyDb) + if err != nil { + continue + } + schemas[v] = ts + } + return views, schemas, nil +} + +// migrateTableDirectory recursively traverses a table directory, finds leaf nodes containing +// parquet files, and for each leaf executes a placeholder INSERT within a transaction. +// On success, it moves the leaf directory from migrating to migrated. +func migrateTableDirectory(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus) error { + // create the table if not exists + err := database.EnsureDuckLakeTable(ts.Columns, db, tableName) + if err != nil { + // fatal – move table dir to failed and return error + moveTableDirToFailed(ctx, dirPath) + return err + } + entries, err := os.ReadDir(dirPath) + if err != nil { + // fatal – move table dir to failed and return error + moveTableDirToFailed(ctx, dirPath) + return err + } + + var parquetFiles []string + var errList []error + for _, entry := range entries { + // early exit on cancellation + if ctx.Err() != nil { + errList = append(errList, ctx.Err()) + // TODO format better + return errors.Join(errList...) + } + + if entry.IsDir() { + subDir := filepath.Join(dirPath, entry.Name()) + if err := migrateTableDirectory(ctx, db, tableName, subDir, ts, status); err != nil { + // just add to error list and continue with other entries + errList = append(errList, err) + } + } + + if strings.HasSuffix(strings.ToLower(entry.Name()), ".parquet") { + parquetFiles = append(parquetFiles, filepath.Join(dirPath, entry.Name())) + } + } + + // If this directory contains parquet files, treat it as a leaf node for migration + if len(parquetFiles) > 0 { + err = migrateParquetFiles(ctx, db, tableName, dirPath, ts, status, parquetFiles) + if err != nil { + errList = append(errList, err) + } + } + + // TODO format better + return errors.Join(errList...) +} + +func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus, parquetFiles []string) error { + filesInLeaf := len(parquetFiles) + // Placeholder: validate schema (from 'ts') against parquet files if needed + slog.Info("Found leaf node with parquet files", "table", tableName, "dir", dirPath, "files", filesInLeaf) + + // Begin transaction + tx, err := db.BeginTx(ctx, nil) + if err != nil { + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + + // Build and execute the parquet insert + if err := insertFromParquetFiles(ctx, tx, tableName, ts.Columns, parquetFiles); err != nil { + slog.Debug("Rolling back transaction", "table", tableName, "error", err) + txErr := tx.Rollback() + if txErr != nil { + slog.Error("Transaction rollback failed", "table", tableName, "error", txErr) + } + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + // Note: cancellation will be handled by outer logic; if needed, you can check and rollback here. + + if err := tx.Commit(); err != nil { + slog.Error("Error committing transaction", "table", tableName, "error", err) + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + + slog.Info("Successfully committed transaction", "table", tableName, "dir", dirPath, "files", filesInLeaf) + + // On success, move the entire leaf directory from migrating to migrated + migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() + migratedRoot := config.GlobalWorkspaceProfile.GetMigratedDir() + rel, err := filepath.Rel(migratingRoot, dirPath) + if err != nil { + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + destDir := filepath.Join(migratedRoot, rel) + if err := os.MkdirAll(filepath.Dir(destDir), 0755); err != nil { + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + if err := utils.MoveDirContents(ctx, dirPath, destDir); err != nil { + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + _ = os.Remove(dirPath) + status.OnFilesMigrated(filesInLeaf) + slog.Info("Migrated leaf node", "table", tableName, "source", dirPath, "destination", destDir) + return nil +} + +// move any table directories with no corresponding view to ~/.tailpipe/migration/unmigrated/ - we will not migrate them +func archiveUnmatchedDirs(ctx context.Context, unmatchedTableDirs []string) error { + for _, d := range unmatchedTableDirs { + // move to ~/.tailpipe/migration/migrated/ + tname := strings.TrimPrefix(filepath.Base(d), "tp_table=") + slog.Warn("Table %s has data but no view in database; moving without migration", "table", tname, "dir", d) + migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() + unmigratedRoot := config.GlobalWorkspaceProfile.GetUnmigratedDir() + // get the relative path from migrating root to d + rel, err := filepath.Rel(migratingRoot, d) + if err != nil { + return err + } + // build a dest path by joining unmigrated root with this relative path + destPath := filepath.Join(unmigratedRoot, rel) + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + // move the entire directory + if err := utils.MoveDirContents(ctx, d, destPath); err != nil { + return err + } + err = os.Remove(d) + if err != nil { + return err + } + } + return nil +} + +// doMigration performs the migration of the matched table directories and updates status +func doMigration(ctx context.Context, matchedTableDirs []string, schemas map[string]*schema.TableSchema, status *MigrationStatus, onUpdate func(*MigrationStatus)) error { + if onUpdate == nil { + return fmt.Errorf("onUpdate callback is required") + } + ducklakeDb, err := database.NewDuckDb(database.WithDuckLake()) + if err != nil { + return err + } + defer ducklakeDb.Close() + + for _, tableDir := range matchedTableDirs { + tableName := strings.TrimPrefix(filepath.Base(tableDir), "tp_table=") + if tableName == "" { + continue + } + ts := schemas[tableName] + if err := migrateTableDirectory(ctx, ducklakeDb, tableName, tableDir, ts, status); err != nil { + slog.Warn("Migration failed for table; moving to migration/failed", "table", tableName, "error", err) + status.OnTableFailed(tableName) + } else { + status.OnTableMigrated() + } + // update our status + onUpdate(status) + } + return nil +} + +// moveTableDirToFailed moves a table directory from migrating to failed, preserving relative path. +func moveTableDirToFailed(ctx context.Context, dirPath string) { + migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() + failedRoot := config.GlobalWorkspaceProfile.GetMigrationFailedDir() + rel, err := filepath.Rel(migratingRoot, dirPath) + if err != nil { + return + } + destDir := filepath.Join(failedRoot, rel) + err = os.MkdirAll(filepath.Dir(destDir), 0755) + if err != nil { + slog.Error("moveTableDirToFailed: Failed to create parent for failed dir", "error", err, "dir", destDir) + return + } + err = utils.MoveDirContents(ctx, dirPath, destDir) + if err != nil { + slog.Error("moveTableDirToFailed: Failed to move dir to failed", "error", err, "source", dirPath, "destination", destDir) + return + } + err = os.Remove(dirPath) + if err != nil { + slog.Error("moveTableDirToFailed: Failed to remove original dir after move", "error", err, "dir", dirPath) + } +} + +// countParquetFiles walks all matched table directories and counts parquet files +func countParquetFiles(ctx context.Context, dirs []string) (int, error) { + total := 0 + for _, root := range dirs { + // early exit on cancellation + if ctx.Err() != nil { + return 0, ctx.Err() + } + if err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".parquet") { + total++ + } + return nil + }); err != nil { + return 0, err + } + } + return total, nil +} + +// insertFromParquetFiles builds and executes an INSERT … SELECT read_parquet(...) for a set of parquet files +func insertFromParquetFiles(ctx context.Context, tx *sql.Tx, tableName string, columns []*schema.ColumnSchema, parquetFiles []string) error { + var colList []string + for _, c := range columns { + colList = append(colList, fmt.Sprintf(`"%s"`, c.ColumnName)) + } + cols := strings.Join(colList, ", ") + escape := func(p string) string { return strings.ReplaceAll(p, "'", "''") } + var fileSQL string + if len(parquetFiles) == 1 { + fileSQL = fmt.Sprintf("'%s'", escape(parquetFiles[0])) + } else { + var quoted []string + for _, f := range parquetFiles { + quoted = append(quoted, fmt.Sprintf("'%s'", escape(f))) + } + fileSQL = "[" + strings.Join(quoted, ", ") + "]" + } + //nolint:gosec // file paths are sanitized + query := fmt.Sprintf(` + insert into "%s" (%s) + select %s from read_parquet(%s) + `, tableName, cols, cols, fileSQL) + _, err := tx.ExecContext(ctx, query) + return err +} + +// onSuccessful handles success outcome: cleans migrating db, prunes empty dirs, prints summary +func onSuccessful(status *MigrationStatus) error { + // Remove any leftover db in migrating + _ = os.Remove(filepath.Join(config.GlobalWorkspaceProfile.GetMigratingDir(), "tailpipe.db")) + // Prune empty dirs in migrating + if err := filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()); err != nil { + return fmt.Errorf("failed to prune empty directories in migrating: %w", err) + } + status.Finish("SUCCESS") + perr.ShowWarning(status.StatusMessage()) + return nil +} + +// onCancelled handles cancellation outcome: keep migrating db, prune empties, print summary +func onCancelled(status *MigrationStatus) error { + // Do not move db; just prune empties so tree is clean + _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) + status.Finish("CANCELLED") + perr.ShowWarning(status.StatusMessage()) + return nil +} + +// onFailed handles failure outcome: move db to failed, prune empties, print summary +func onFailed(status *MigrationStatus) error { + failedDefaultDir := config.GlobalWorkspaceProfile.GetMigrationFailedDir() + if err := os.MkdirAll(failedDefaultDir, 0755); err != nil { + return err + } + srcDb := filepath.Join(config.GlobalWorkspaceProfile.GetMigratingDir(), "tailpipe.db") + if _, err := os.Stat(srcDb); err == nil { + if err := utils.MoveFile(srcDb, filepath.Join(failedDefaultDir, "tailpipe.db")); err != nil { + return fmt.Errorf("failed to move legacy db to failed: %w", err) + } + } + _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) + status.Finish("INCOMPLETE") + perr.ShowWarning(status.StatusMessage()) + return nil +} diff --git a/internal/migration/status.go b/internal/migration/status.go new file mode 100644 index 00000000..2f223f40 --- /dev/null +++ b/internal/migration/status.go @@ -0,0 +1,125 @@ +package migration + +import ( + "fmt" + "strings" + "time" + + "github.com/turbot/tailpipe/internal/config" +) + +type MigrationStatus struct { + Status string `json:"status"` + Total int `json:"total"` + Migrated int `json:"migrated"` + Failed int `json:"failed"` + Remaining int `json:"remaining"` + ProgressPercent float64 `json:"progress_percent"` + + TotalFiles int `json:"total_files"` + MigratedFiles int `json:"migrated_files"` + FailedFiles int `json:"failed_files"` + RemainingFiles int `json:"remaining_files"` + + FailedTables []string `json:"failed_tables,omitempty"` + StartTime time.Time `json:"start_time"` + Duration time.Duration `json:"duration"` +} + +func NewMigrationStatus(total int) *MigrationStatus { + return &MigrationStatus{Total: total, Remaining: total, StartTime: time.Now()} +} + +func (s *MigrationStatus) OnTableMigrated() { + s.Migrated++ + s.update() +} + +func (s *MigrationStatus) OnTableFailed(tableName string) { + s.Failed++ + s.FailedTables = append(s.FailedTables, tableName) + s.update() +} + +func (s *MigrationStatus) OnFilesMigrated(n int) { + if n <= 0 { + return + } + s.MigratedFiles += n + s.updateFiles() +} + +func (s *MigrationStatus) OnFilesFailed(n int) { + if n <= 0 { + return + } + s.FailedFiles += n + s.updateFiles() +} + +func (s *MigrationStatus) update() { + s.Remaining = s.Total - s.Migrated - s.Failed + if s.Total > 0 { + s.ProgressPercent = float64(s.Migrated) * 100.0 / float64(s.Total) + } +} + +func (s *MigrationStatus) updateFiles() { + s.RemainingFiles = s.TotalFiles - s.MigratedFiles - s.FailedFiles +} + +func (s *MigrationStatus) Finish(outcome string) { + s.Status = outcome + s.Duration = time.Since(s.StartTime) +} + +// StatusMessage returns a user-facing status message (with stats) based on current migration status +func (s *MigrationStatus) StatusMessage() string { + migratedDir := config.GlobalWorkspaceProfile.GetMigratedDir() + failedDir := config.GlobalWorkspaceProfile.GetMigrationFailedDir() + migratingDir := config.GlobalWorkspaceProfile.GetMigratingDir() + + switch s.Status { + case "SUCCESS": + return fmt.Sprintf( + "DuckLake migration complete.\n"+ + "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Backup of migrated legacy data: '%s'\n", + s.Migrated, s.Total, s.Failed, s.Remaining, + s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, + migratedDir, + ) + case "CANCELLED": + return fmt.Sprintf( + "DuckLake migration cancelled.\n"+ + "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Legacy DB preserved: '%s/tailpipe.db'\n\n"+ + "Re-run Tailpipe to resume migrating your data.\n", + s.Migrated, s.Total, s.Failed, s.Remaining, + s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, + migratingDir, + ) + case "INCOMPLETE": + failedList := "(none)" + if len(s.FailedTables) > 0 { + failedList = strings.Join(s.FailedTables, ", ") + } + return fmt.Sprintf( + "DuckLake migration completed with issues.\n"+ + "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Failed tables (%d): %s\n"+ + "- Failed data and legacy DB: '%s'\n"+ + "- Backup of migrated legacy data: '%s'\n", + s.Migrated, s.Total, s.Failed, s.Remaining, + s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, + len(s.FailedTables), failedList, + failedDir, + migratedDir, + ) + default: + return "DuckLake migration status unknown" + } +} diff --git a/internal/migration/utils.go b/internal/migration/utils.go new file mode 100644 index 00000000..7cf485e5 --- /dev/null +++ b/internal/migration/utils.go @@ -0,0 +1,50 @@ +package migration + +import ( + "os" + "path/filepath" + "strings" +) + +// findMatchingTableDirs lists subdirectories of baseDir whose names start with +// "tp_table=" and whose table names exist in the provided tables slice. +// Also returns unmatched tp_table directories for which there is no view in the DB. +func findMatchingTableDirs(baseDir string, tables []string) ([]string, []string, error) { + entries, err := os.ReadDir(baseDir) + if err != nil { + return nil, nil, err + } + tableSet := make(map[string]struct{}, len(tables)) + for _, t := range tables { + tableSet[t] = struct{}{} + } + var matches []string + var unmatched []string + const prefix = "tp_table=" + for _, e := range entries { + if !e.IsDir() { + continue + } + name := e.Name() + if !strings.HasPrefix(name, prefix) { + continue + } + tableName := strings.TrimPrefix(name, prefix) + if _, ok := tableSet[tableName]; ok { + matches = append(matches, filepath.Join(baseDir, name)) + } else { + unmatched = append(unmatched, filepath.Join(baseDir, name)) + } + } + return matches, unmatched, nil +} + +// hasTailpipeDb checks if a tailpipe.db file exists in the provided directory. +func hasTailpipeDb(dir string) bool { + if dir == "" { + return false + } + p := filepath.Join(dir, "tailpipe.db") + _, err := os.Stat(p) + return err == nil +} From 25294c9c74e28d23a3325494a66a4bb26dae81be Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Wed, 17 Sep 2025 17:28:29 +0530 Subject: [PATCH 42/61] Wrap migration errors and put details in a log file (#565) --- go.mod | 2 +- go.sum | 4 +-- internal/error_helpers/error_helpers.go | 11 ++++++++ internal/migration/error.go | 33 ++++++++++++++++++++++++ internal/migration/migration.go | 33 ++++++++++++++---------- internal/migration/status.go | 34 ++++++++++++++++++++++++- 6 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 internal/migration/error.go diff --git a/go.mod b/go.mod index 9654b36e..d3e9648b 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 - github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1 + github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 github.com/turbot/tailpipe-plugin-sdk v0.9.3 github.com/zclconf/go-cty v1.16.3 golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 diff --git a/go.sum b/go.sum index a2c06c77..7ba9ebf6 100644 --- a/go.sum +++ b/go.sum @@ -1308,8 +1308,8 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1 h1:4Y/51FNwJqavbz/O8T8NQkpp6+roiyoT7BrD/GLR2FU= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= +github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 h1:FfKLkfbNmwxyPQIqDCd7m6o9bmtPB7D8a5txbVzjZp4= +github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= diff --git a/internal/error_helpers/error_helpers.go b/internal/error_helpers/error_helpers.go index d8936ec5..611ba7d4 100644 --- a/internal/error_helpers/error_helpers.go +++ b/internal/error_helpers/error_helpers.go @@ -93,6 +93,17 @@ func ShowWarning(warning string) { fmt.Fprintf(opStream, "%s: %v\n", constants.ColoredWarn, warning) } +// ShowInfo prints a non-critical info message to the appropriate output stream. +// Behaves like ShowWarning but with a calmer label (Note) to avoid alarming users +// for successful outcomes or informational messages. +func ShowInfo(info string) { + if len(info) == 0 { + return + } + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %v\n", color.YellowString("Note"), info) +} + func PrefixError(err error, prefix string) error { return fmt.Errorf("%s: %s\n", prefix, TransformErrorToTailpipe(err).Error()) } diff --git a/internal/migration/error.go b/internal/migration/error.go new file mode 100644 index 00000000..eac57f06 --- /dev/null +++ b/internal/migration/error.go @@ -0,0 +1,33 @@ +package migration + +import ( + "fmt" +) + +// MigrationError is an aggregate error that wraps multiple child errors +// encountered during migration. +type MigrationError struct { + errors []error +} + +func NewMigrationError() *MigrationError { + return &MigrationError{errors: make([]error, 0)} +} + +func (m *MigrationError) Append(err error) { + if err == nil { + return + } + m.errors = append(m.errors, err) +} + +func (m *MigrationError) Len() int { return len(m.errors) } + +// Error provides a compact summary string +func (m *MigrationError) Error() string { + return fmt.Sprintf("%d error(s) occurred during migration", len(m.errors)) +} + +// Unwrap returns the list of child errors so that errors.Is/As can walk them +// (supported since Go 1.20 with Unwrap() []error) +func (m *MigrationError) Unwrap() []error { return m.errors } diff --git a/internal/migration/migration.go b/internal/migration/migration.go index bddd383a..49c84c21 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -3,7 +3,6 @@ package migration import ( "context" "database/sql" - "errors" "fmt" "io/fs" "log/slog" @@ -18,6 +17,7 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/filepaths" ) @@ -133,9 +133,12 @@ func MigrateDataToDucklake(ctx context.Context) error { err = doMigration(ctx, matchedTableDirs, schemas, status, updateStatus) sp.Stop() + logPath := filepath.Join(config.GlobalWorkspaceProfile.GetMigrationDir(), "migration.log") // If cancellation arrived after doMigration returned, prefer the CANCELLED outcome if perr.IsContextCancelledError(ctx.Err()) { - _ = onCancelled(status) + status.Finish("CANCELLED") + _ = status.WriteStatusToFile() + perr.ShowWarning(fmt.Sprintf("Migration cancelled. It will automatically resume next time you run Tailpipe.\nFor details, see %s\n", logPath)) cancelledHandled = true return ctx.Err() } @@ -145,10 +148,12 @@ func MigrateDataToDucklake(ctx context.Context) error { if err := onFailed(status); err != nil { return err } + perr.ShowWarning(fmt.Sprintf("Your data has been migrated to DuckLake with issues.\nFor details, see %s\n", logPath)) } else { if err := onSuccessful(status); err != nil { return err } + error_helpers.ShowInfo(fmt.Sprintf("Your data has been migrated to DuckLake.\nFor details, see %s\n", logPath)) } return err @@ -241,20 +246,19 @@ func migrateTableDirectory(ctx context.Context, db *database.DuckDb, tableName s } var parquetFiles []string - var errList []error + aggErr := NewMigrationError() for _, entry := range entries { // early exit on cancellation if ctx.Err() != nil { - errList = append(errList, ctx.Err()) - // TODO format better - return errors.Join(errList...) + aggErr.Append(ctx.Err()) + return aggErr } if entry.IsDir() { subDir := filepath.Join(dirPath, entry.Name()) if err := migrateTableDirectory(ctx, db, tableName, subDir, ts, status); err != nil { // just add to error list and continue with other entries - errList = append(errList, err) + aggErr.Append(err) } } @@ -267,12 +271,15 @@ func migrateTableDirectory(ctx context.Context, db *database.DuckDb, tableName s if len(parquetFiles) > 0 { err = migrateParquetFiles(ctx, db, tableName, dirPath, ts, status, parquetFiles) if err != nil { - errList = append(errList, err) + aggErr.Append(err) + status.AddError(fmt.Errorf("failed migrating parquet files for table '%s' at '%s': %w", tableName, dirPath, err)) } } - // TODO format better - return errors.Join(errList...) + if aggErr.Len() == 0 { + return nil + } + return aggErr } func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus, parquetFiles []string) error { @@ -479,7 +486,7 @@ func onSuccessful(status *MigrationStatus) error { return fmt.Errorf("failed to prune empty directories in migrating: %w", err) } status.Finish("SUCCESS") - perr.ShowWarning(status.StatusMessage()) + _ = status.WriteStatusToFile() return nil } @@ -488,7 +495,7 @@ func onCancelled(status *MigrationStatus) error { // Do not move db; just prune empties so tree is clean _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) status.Finish("CANCELLED") - perr.ShowWarning(status.StatusMessage()) + _ = status.WriteStatusToFile() return nil } @@ -506,6 +513,6 @@ func onFailed(status *MigrationStatus) error { } _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) status.Finish("INCOMPLETE") - perr.ShowWarning(status.StatusMessage()) + _ = status.WriteStatusToFile() return nil } diff --git a/internal/migration/status.go b/internal/migration/status.go index 2f223f40..6d3c479b 100644 --- a/internal/migration/status.go +++ b/internal/migration/status.go @@ -2,6 +2,8 @@ package migration import ( "fmt" + "os" + "path/filepath" "strings" "time" @@ -24,6 +26,8 @@ type MigrationStatus struct { FailedTables []string `json:"failed_tables,omitempty"` StartTime time.Time `json:"start_time"` Duration time.Duration `json:"duration"` + + Errors []string `json:"errors,omitempty"` } func NewMigrationStatus(total int) *MigrationStatus { @@ -57,6 +61,13 @@ func (s *MigrationStatus) OnFilesFailed(n int) { s.updateFiles() } +func (s *MigrationStatus) AddError(err error) { + if err == nil { + return + } + s.Errors = append(s.Errors, err.Error()) +} + func (s *MigrationStatus) update() { s.Remaining = s.Total - s.Migrated - s.Failed if s.Total > 0 { @@ -106,7 +117,7 @@ func (s *MigrationStatus) StatusMessage() string { if len(s.FailedTables) > 0 { failedList = strings.Join(s.FailedTables, ", ") } - return fmt.Sprintf( + base := fmt.Sprintf( "DuckLake migration completed with issues.\n"+ "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ @@ -119,7 +130,28 @@ func (s *MigrationStatus) StatusMessage() string { failedDir, migratedDir, ) + if len(s.Errors) > 0 { + base += fmt.Sprintf("\nErrors: %d error(s) occurred during migration\n", len(s.Errors)) + base += "Details:\n" + for _, e := range s.Errors { + base += "- " + e + "\n" + } + } + return base default: return "DuckLake migration status unknown" } } + +// WriteStatusToFile writes the status message to a migration stats file under the migration directory. +// The file is overwritten on each run (resume will update it). +func (s *MigrationStatus) WriteStatusToFile() error { + // Place the file under the migration root (e.g., ~/.tailpipe/migration/migration.log) + migrationRootDir := config.GlobalWorkspaceProfile.GetMigrationDir() + statsFile := filepath.Join(migrationRootDir, "migration.log") + msg := s.StatusMessage() + if err := os.MkdirAll(migrationRootDir, 0755); err != nil { + return err + } + return os.WriteFile(statsFile, []byte(msg), 0600) +} From 6d721f69bc5ed8f7c9a58e417fc0e63ef5c84cde Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Wed, 17 Sep 2025 18:03:59 +0530 Subject: [PATCH 43/61] Backup DuckLake metadata database (#564) --- cmd/collect.go | 7 ++ cmd/compact.go | 6 ++ cmd/partition.go | 7 ++ internal/database/backup.go | 138 ++++++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+) create mode 100644 internal/database/backup.go diff --git a/cmd/collect.go b/cmd/collect.go index 332847fc..80f228af 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -124,6 +124,13 @@ func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) er partitionNames = append(partitionNames, partition.FullName) } slog.Info("Starting collection", "partition(s)", partitionNames, "from", fromTime, "to", toTime) + + // Create backup of metadata database before starting collection + if err := database.BackupDucklakeMetadata(); err != nil { + slog.Warn("Failed to backup metadata database", "error", err) + // Continue with collection - backup failure shouldn't block the operation + } + // now we have the partitions, we can start collecting // start the plugin manager diff --git a/cmd/compact.go b/cmd/compact.go index a02605c0..d95d8e4e 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -77,6 +77,12 @@ func runCompactCmd(cmd *cobra.Command, args []string) { patterns, err := database.GetPartitionPatternsForArgs(maps.Keys(config.GlobalConfig.Partitions), args...) error_helpers.FailOnErrorWithMessage(err, "failed to get partition patterns") + // Create backup of metadata database before starting compaction + if err := database.BackupDucklakeMetadata(); err != nil { + slog.Warn("Failed to backup metadata database", "error", err) + // Continue with compaction - backup failure shouldn't block the operation + } + // do the compaction status, err := doCompaction(ctx, db, patterns) diff --git a/cmd/partition.go b/cmd/partition.go index 3a72c583..7faf8be2 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -301,6 +301,13 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { error_helpers.FailOnError(err) defer db.Close() + // Create backup before deletion + slog.Info("Creating backup before partition deletion", "partition", partitionName) + if err := database.BackupDucklakeMetadata(); err != nil { + slog.Warn("Failed to create backup before partition deletion", "error", err) + // Continue with deletion - backup failure should not prevent deletion + } + // show spinner while deleting the partition spinner := statushooks.NewStatusSpinnerHook() spinner.SetStatus(fmt.Sprintf("Deleting partition %s", partition.TableName)) diff --git a/internal/database/backup.go b/internal/database/backup.go new file mode 100644 index 00000000..05d106c4 --- /dev/null +++ b/internal/database/backup.go @@ -0,0 +1,138 @@ +package database + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe/internal/config" +) + +// BackupDucklakeMetadata creates a timestamped backup of the DuckLake metadata database. +// It creates backup files with format: metadata.sqlite.backup.YYYYMMDDHHMMSS +// and also backs up the WAL file if it exists: +// - metadata.sqlite-wal.backup.YYYYMMDDHHMMSS +// It removes any existing backup files to maintain only the most recent backup. +// +// The backup is created in the same directory as the original database file. +// If the database file doesn't exist, no backup is created and no error is returned. +// +// Returns an error if the backup operation fails. +func BackupDucklakeMetadata() error { + // Get the path to the DuckLake metadata database + dbPath := config.GlobalWorkspaceProfile.GetDucklakeDbPath() + + // Check if the database file exists + if _, err := os.Stat(dbPath); os.IsNotExist(err) { + slog.Debug("DuckLake metadata database does not exist, skipping backup", "path", dbPath) + return nil + } else if err != nil { + return fmt.Errorf("failed to check if database exists: %w", err) + } + + // Generate timestamp for backup filename + timestamp := time.Now().Format("20060102150405") // YYYYMMDDHHMMSS format + + // Create backup filenames + dbDir := filepath.Dir(dbPath) + mainBackupFilename := fmt.Sprintf("metadata.sqlite.backup.%s", timestamp) + mainBackupPath := filepath.Join(dbDir, mainBackupFilename) + + // Also prepare paths for WAL file + walPath := dbPath + "-wal" + walBackupFilename := fmt.Sprintf("metadata.sqlite-wal.backup.%s", timestamp) + walBackupPath := filepath.Join(dbDir, walBackupFilename) + + slog.Info("Creating backup of DuckLake metadata database", "source", dbPath, "backup", mainBackupPath) + + // Create the main database backup first + if err := utils.CopyFile(dbPath, mainBackupPath); err != nil { + return fmt.Errorf("failed to create main database backup: %w", err) + } + + // Backup WAL file if it exists + if _, err := os.Stat(walPath); err == nil { + if err := utils.CopyFile(walPath, walBackupPath); err != nil { + slog.Warn("Failed to backup WAL file", "source", walPath, "error", err) + // Continue - WAL backup failure is not critical + } else { + slog.Debug("Successfully backed up WAL file", "backup", walBackupPath) + } + } + + slog.Info("Successfully created backup of DuckLake metadata database", "backup", mainBackupPath) + + // Clean up old backup files after successfully creating the new one + if err := cleanupOldBackups(dbDir, timestamp); err != nil { + slog.Warn("Failed to clean up old backup files", "error", err) + // Don't return error - the backup was successful, cleanup is just housekeeping + } + return nil +} + +// isBackupFile checks if a filename matches any of the backup patterns +func isBackupFile(filename string) bool { + backupPrefixes := []string{ + "metadata.sqlite.backup.", + "metadata.sqlite-wal.backup.", + } + + for _, prefix := range backupPrefixes { + if strings.HasPrefix(filename, prefix) { + return true + } + } + return false +} + +// shouldRemoveBackup determines if a backup file should be removed +func shouldRemoveBackup(filename, excludeTimestamp string) bool { + if !isBackupFile(filename) { + return false + } + // Don't remove files with the current timestamp + return !strings.HasSuffix(filename, "."+excludeTimestamp) +} + +// cleanupOldBackups removes all existing backup files in the specified directory, +// except for the newly created backup files with the given timestamp. +// Backup files are identified by the patterns: +// - metadata.sqlite.backup.* +// - metadata.sqlite-wal.backup.* +func cleanupOldBackups(dir, excludeTimestamp string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("failed to read directory: %w", err) + } + + var deletedCount int + for _, entry := range entries { + if entry.IsDir() { + continue + } + + filename := entry.Name() + if !shouldRemoveBackup(filename, excludeTimestamp) { + continue + } + + backupPath := filepath.Join(dir, filename) + if err := os.Remove(backupPath); err != nil { + slog.Warn("Failed to remove old backup file", "file", backupPath, "error", err) + // Continue removing other files even if one fails + } else { + slog.Debug("Removed old backup file", "file", backupPath) + deletedCount++ + } + } + + if deletedCount > 0 { + slog.Debug("Cleaned up old backup files", "count", deletedCount) + } + + return nil +} From 2dfbef1d5a922f29b3165cff53bad3131f72b5c4 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Thu, 18 Sep 2025 17:56:37 +0100 Subject: [PATCH 44/61] Exclude rowid column from migration schema, as this is a system added column and including in migration schema breaks compaction. --- internal/database/compaction_types.go | 4 ++-- internal/database/tables.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/database/compaction_types.go b/internal/database/compaction_types.go index 4a2f98e1..bf052f7e 100644 --- a/internal/database/compaction_types.go +++ b/internal/database/compaction_types.go @@ -17,7 +17,7 @@ func getTimeRangesToReorder(ctx context.Context, db *DuckDb, pk *partitionKey, r if reindex { rm, err := newReorderMetadata(ctx, db, pk) if err != nil { - return nil, fmt.Errorf("failed to retiever stats for partition key: %w", err) + return nil, fmt.Errorf("failed to retrieve stats for partition key: %w", err) } // make a single time range @@ -52,7 +52,7 @@ func getTimeRangesToReorder(ctx context.Context, db *DuckDb, pk *partitionKey, r // get stats for the partition key rm, err := newReorderMetadata(ctx, db, pk) if err != nil { - return nil, fmt.Errorf("failed to retiever stats for partition key: %w", err) + return nil, fmt.Errorf("failed to retrieve stats for partition key: %w", err) } rm.unorderedRanges = unorderedRanges return rm, nil diff --git a/internal/database/tables.go b/internal/database/tables.go index 4d8f7acb..a38c1485 100644 --- a/internal/database/tables.go +++ b/internal/database/tables.go @@ -113,6 +113,11 @@ func GetLegacyTableViewSchema(ctx context.Context, viewName string, db *DuckDb) return nil, fmt.Errorf("failed to scan column schema: %w", err) } + // NOTE: legacy tailpipe views may include `rowid` which we must exclude from the schema as this is a DuckDb system column + // that is automatically added to every table + if columnName == "rowid" { + continue + } col := buildColumnSchema(columnName, columnType) ts.Columns = append(ts.Columns, col) } From bfcdc6fdd69b1fa66e08546b9a9523f08e01cedc Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Thu, 18 Sep 2025 18:02:59 +0100 Subject: [PATCH 45/61] Improve status display for migration. Improve status display for migration - show all setup steps as well as the actual migration Rename data folder into migrating folder, rather than copy and delete Prompt user before migrating Do not backup migrated folders into 'migrated' dir - just delete Do not migrate if the output format is machine readable or progress is false Set Cobra error output to stdout use local DisplayError and DisplayWarnin grather than pipe-fittings --- cmd/root.go | 6 + internal/cmdconfig/cmd_hooks.go | 29 ++- internal/error_helpers/error_helpers.go | 8 +- internal/migration/migration.go | 313 +++++++++++++++++------- internal/migration/status.go | 9 +- 5 files changed, 249 insertions(+), 116 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 69ef0a8a..6b5669c7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,8 @@ package cmd import ( + "os" + "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/turbot/pipe-fittings/v2/cmdconfig" @@ -62,6 +64,10 @@ func Execute() int { utils.LogTime("cmd.root.Execute start") defer utils.LogTime("cmd.root.Execute end") rootCmd := rootCommand() + + // set the error output to stdout (as it;s common usage to redirect stderr to a file to capture logs + rootCmd.SetErr(os.Stdout) + if err := rootCmd.Execute(); err != nil { exitCode = -1 } diff --git a/internal/cmdconfig/cmd_hooks.go b/internal/cmdconfig/cmd_hooks.go index c07cfac7..2272aa00 100644 --- a/internal/cmdconfig/cmd_hooks.go +++ b/internal/cmdconfig/cmd_hooks.go @@ -14,7 +14,8 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" + perror_helpers "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/filepaths" pparse "github.com/turbot/pipe-fittings/v2/parse" "github.com/turbot/pipe-fittings/v2/task" @@ -49,7 +50,7 @@ func preRunHook(cmd *cobra.Command, args []string) error { // display any warnings ew.ShowWarnings() // check for error - error_helpers.FailOnError(ew.Error) + perror_helpers.FailOnError(ew.Error) // pump in the initial set of logs (AFTER we have loaded the config, which may specify log level) displayStartupLog() @@ -76,15 +77,17 @@ func preRunHook(cmd *cobra.Command, args []string) error { // the new metadata model. // start migration err := migration.MigrateDataToDucklake(cmd.Context()) - if error_helpers.IsContextCancelledError(err) { - // suppress Cobra's usage/errors only for this cancelled invocation - // Cobra prints usage when a command returns an error. The cancellation returns an error (context cancelled) - // from preRun, so Cobra assumes "user error" and shows help. - // This conditional block sets cmd.SilenceUsage = true and cmd.SilenceErrors = true only for cancellation, - // telling Cobra "don't print usage or re-print the error". Without it, you get the usage dump. + if err != nil { + // we do not want Cobra usage errors for migration errors - suppress + + // suppress usage and error printing for migration errors cmd.SilenceUsage = true - cmd.SilenceErrors = true + // for cancelled errors, also silence the error message + if perror_helpers.IsCancelledError(err) { + cmd.SilenceErrors = true + } } + // return (possibly nil) error from migration return err } @@ -154,7 +157,7 @@ func runScheduledTasks(ctx context.Context, cmd *cobra.Command, args []string) c } // initConfig reads in config file and ENV variables if set. -func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { +func initGlobalConfig(ctx context.Context) perror_helpers.ErrorAndWarnings { utils.LogTime("cmdconfig.initGlobalConfig start") defer utils.LogTime("cmdconfig.initGlobalConfig end") @@ -171,14 +174,14 @@ func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { // load workspace profile from the configured install dir loader, err := cmdconfig.GetWorkspaceProfileLoader[*workspace_profile.TailpipeWorkspaceProfile](parseOpts...) if err != nil { - return error_helpers.NewErrorsAndWarning(err) + return perror_helpers.NewErrorsAndWarning(err) } config.GlobalWorkspaceProfile = loader.GetActiveWorkspaceProfile() // create the required data and internal folder for this workspace if needed err = config.GlobalWorkspaceProfile.EnsureWorkspaceDirs() if err != nil { - return error_helpers.NewErrorsAndWarning(err) + return perror_helpers.NewErrorsAndWarning(err) } var cmd = viper.Get(pconstants.ConfigKeyActiveCommand).(*cobra.Command) @@ -199,7 +202,7 @@ func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { // NOTE: if this installed the core plugin, the plugin version file will be updated and the updated file returned pluginVersionFile, err := plugin.EnsureCorePlugin(ctx) if err != nil { - return error_helpers.NewErrorsAndWarning(err) + return perror_helpers.NewErrorsAndWarning(err) } // load the connection config and HCL options (passing plugin versions diff --git a/internal/error_helpers/error_helpers.go b/internal/error_helpers/error_helpers.go index 611ba7d4..8c55d866 100644 --- a/internal/error_helpers/error_helpers.go +++ b/internal/error_helpers/error_helpers.go @@ -108,14 +108,14 @@ func PrefixError(err error, prefix string) error { return fmt.Errorf("%s: %s\n", prefix, TransformErrorToTailpipe(err).Error()) } -// isMachineReadableOutput checks if the current output format is machine readable (CSV or JSON) -func isMachineReadableOutput() bool { +// IsMachineReadableOutput checks if the current output format is machine readable (CSV or JSON) +func IsMachineReadableOutput() bool { outputFormat := viper.GetString(constants.ArgOutput) - return outputFormat == constants.OutputFormatCSV || outputFormat == constants.OutputFormatJSON + return outputFormat == constants.OutputFormatCSV || outputFormat == constants.OutputFormatJSON || outputFormat == constants.OutputFormatLine } func GetWarningOutputStream() io.Writer { - if isMachineReadableOutput() { + if IsMachineReadableOutput() { // For machine-readable formats, output warnings and errors to stderr return os.Stderr } diff --git a/internal/migration/migration.go b/internal/migration/migration.go index 49c84c21..7fa3459b 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -1,6 +1,7 @@ package migration import ( + "bufio" "context" "database/sql" "fmt" @@ -12,6 +13,8 @@ import ( "time" "github.com/briandowns/spinner" + "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/v2/constants" perr "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe-plugin-sdk/schema" @@ -21,22 +24,28 @@ import ( "github.com/turbot/tailpipe/internal/filepaths" ) +// StatusType represents different types of migration status messages +type StatusType int + +const ( + InitialisationFailed StatusType = iota + MigrationFailed + CleanupAfterSuccess + PartialSuccess + Success +) + // MigrateDataToDucklake performs migration of views from tailpipe.db and associated parquet files // into the new DuckLake metadata catalog -func MigrateDataToDucklake(ctx context.Context) error { +func MigrateDataToDucklake(ctx context.Context) (err error) { + var statusMsg string + var partialMigrated bool + // Determine source and migration directories dataDefaultDir := config.GlobalWorkspaceProfile.GetDataDir() migratingDefaultDir := config.GlobalWorkspaceProfile.GetMigratingDir() - // failed dir is derived via GetMigrationFailedDir() where needed var matchedTableDirs, unmatchedTableDirs []string - status := NewMigrationStatus(0) - cancelledHandled := false - defer func() { - if ctx.Err() != nil && !cancelledHandled { - _ = onCancelled(status) - } - }() // if the ~/.tailpipe/data directory has a .db file, it means that this is the first time we are migrating // if the ~/.tailpipe/migration/migrating directory has a .db file, it means that this is a resume migration @@ -45,7 +54,7 @@ func MigrateDataToDucklake(ctx context.Context) error { // validate: both should not be true - return that last migration left things in a bad state if initialMigration && continueMigration { - return fmt.Errorf("Invalid migration state: found tailpipe.db in both data and migrating directories. This should not happen. Please contact Turbot support for assistance.") + return fmt.Errorf("invalid migration state: found tailpipe.db in both data and migrating directories. This should not happen. Please contact Turbot support for assistance") } // STEP 1: Check if migration is needed @@ -55,6 +64,67 @@ func MigrateDataToDucklake(ctx context.Context) error { return nil } + // if the output for this command is a machine readable format (csv/json) or progress is false, + // it is possible/likely that tailpipe is being used in a non interactive way - in this case, + // we should not prompt the user, instead return an error + msgFormat := "data must be migrated to Ducklake format - migration is not supported with '%s'.\n\nRun 'tailpipe query' to migrate your data to DuckLake format" + if error_helpers.IsMachineReadableOutput() { + return fmt.Errorf(msgFormat, "--output "+viper.GetString(constants.ArgOutput)) + } else if viper.IsSet(constants.ArgProgress) && !viper.GetBool(constants.ArgProgress) { + return fmt.Errorf(msgFormat, "--progress=false") + } + + // Prompt the user to confirm migration + shouldContinue, err := promptUserForMigration(ctx, dataDefaultDir) + if err != nil { + return fmt.Errorf("failed to get user confirmation: %w", err) + } + if !shouldContinue { + return context.Canceled + } + + // Initialize migration status + status := NewMigrationStatus(0) + + // add any error to status and write to file before returning + defer func() { + if err != nil { + status.AddError(err) + if perr.IsContextCancelledError(ctx.Err()) { + // set cancel status and prune the tree + _ = onCancelled(status) + } else { + status.Finish("FAILED") + } + } + + if statusMsg != "" { + if err != nil || partialMigrated { + // if there is an error or a partial migration, show as warning + error_helpers.ShowWarning(statusMsg) + } else { + // show as info if there is no error, or if it is not a partial migration + error_helpers.ShowInfo(statusMsg) + } + } + // write the status back + _ = status.WriteStatusToFile() + }() + + logPath := filepath.Join(config.GlobalWorkspaceProfile.GetMigrationDir(), "migration.log") + + // Spinner for migration progress + sp := spinner.New( + spinner.CharSets[14], + 100*time.Millisecond, + spinner.WithHiddenCursor(true), + spinner.WithWriter(os.Stdout), + ) + + sp.Suffix = " Migrating data to DuckLake format" + sp.Start() + defer sp.Stop() + // Choose DB path for discovery // If this is the first time we are migrating, we need to use .db file from the ~/.tailpipe/data directory // If this is a resume migration, we need to use .db file from the ~/.tailpipe/migration/migrating directory @@ -65,10 +135,12 @@ func MigrateDataToDucklake(ctx context.Context) error { discoveryDbPath = filepath.Join(migratingDefaultDir, "tailpipe.db") } + sp.Suffix = " Migrating data to DuckLake format: discover tables to migrate" // STEP 2: Discover legacy tables and their schemas (from chosen DB path) // This returns the list of views and a map of view name to its schema views, schemas, err := discoverLegacyTablesAndSchemas(ctx, discoveryDbPath) if err != nil { + statusMsg = getStatus(ctx, InitialisationFailed, "") return fmt.Errorf("failed to discover legacy tables: %w", err) } @@ -77,10 +149,13 @@ func MigrateDataToDucklake(ctx context.Context) error { // STEP 3: If this is the first time we are migrating(tables in ~/.tailpipe/data) then move the whole contents of data dir // into ~/.tailpipe/migration/migrating respecting the same folder structure. - // First-run: transactionally move contents via moveDirContents: copy data to migrating, move tailpipe.db to migrated, - // then empty the original data directory. On any failure, the migrating directory is removed. + // We do this by simply renaming the directory. if initialMigration { - if err := moveDirContents(ctx, dataDefaultDir, migratingDefaultDir); err != nil { + sp.Suffix = " Migrating data to DuckLake format: moving legacy data to migration area" + + if err := moveDataToMigrating(ctx, dataDefaultDir, migratingDefaultDir); err != nil { + slog.Error("Failed to move data to migrating directory", "error", err) + statusMsg = getStatus(ctx, InitialisationFailed, logPath) return err } } @@ -96,17 +171,23 @@ func MigrateDataToDucklake(ctx context.Context) error { baseDir := migratingDefaultDir matchedTableDirs, unmatchedTableDirs, err = findMatchingTableDirs(baseDir, views) if err != nil { - return err - } - // move the unmatched table directories to 'unmigrated' - if err = archiveUnmatchedDirs(ctx, unmatchedTableDirs); err != nil { - return err + statusMsg = getStatus(ctx, MigrationFailed, logPath) + return fmt.Errorf("failed to find matching table directories: %w", err) } + if len(unmatchedTableDirs) > 0 { + sp.Suffix = " Migrating data to DuckLake format: archiving tables without views" + // move the unmatched table directories to 'unmigrated' + if err = archiveUnmatchedDirs(ctx, unmatchedTableDirs); err != nil { + statusMsg = getStatus(ctx, MigrationFailed, logPath) + return fmt.Errorf("failed to archive unmatched table directories: %w", err) + } + } // Initialize status with total tables to migrate status.Total = len(matchedTableDirs) status.update() + sp.Suffix = " Migrating data to DuckLake format: counting parquet files" // Pre-compute total parquet files across matched directories totalFiles, err := countParquetFiles(ctx, matchedTableDirs) if err == nil { @@ -114,83 +195,144 @@ func MigrateDataToDucklake(ctx context.Context) error { status.updateFiles() } - // Spinner for migration progress - sp := spinner.New( - spinner.CharSets[14], - 100*time.Millisecond, - spinner.WithHiddenCursor(true), - spinner.WithWriter(os.Stdout), - ) - sp.Suffix = fmt.Sprintf(" Migrating tables to DuckLake (%d/%d, %0.1f%%) | parquet files (%d/%d)", status.Migrated, status.Total, status.ProgressPercent, status.MigratedFiles, status.TotalFiles) - sp.Start() + sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format (%d/%d, %0.1f%%) | parquet files (%d/%d)", status.Migrated, status.Total, status.ProgressPercent, status.MigratedFiles, status.TotalFiles) updateStatus := func(st *MigrationStatus) { - sp.Suffix = fmt.Sprintf(" Migrating tables to DuckLake (%d/%d, %0.1f%%) | parquet files (%d/%d)", st.Migrated, st.Total, st.ProgressPercent, st.MigratedFiles, st.TotalFiles) + sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format (%d/%d, %0.1f%%) | parquet files (%d/%d)", st.Migrated, st.Total, st.ProgressPercent, st.MigratedFiles, st.TotalFiles) } // STEP 5: Do Migration: Traverse matched table directories, find leaf nodes with parquet files, // and perform INSERT within a transaction. On success, move leaf dir to migrated. err = doMigration(ctx, matchedTableDirs, schemas, status, updateStatus) - sp.Stop() - - logPath := filepath.Join(config.GlobalWorkspaceProfile.GetMigrationDir(), "migration.log") - // If cancellation arrived after doMigration returned, prefer the CANCELLED outcome + // If cancellation arrived during migration, prefer the CANCELLED outcome and do not + // treat it as a failure (which would incorrectly move tailpipe.db to failed) if perr.IsContextCancelledError(ctx.Err()) { - status.Finish("CANCELLED") - _ = status.WriteStatusToFile() - perr.ShowWarning(fmt.Sprintf("Migration cancelled. It will automatically resume next time you run Tailpipe.\nFor details, see %s\n", logPath)) - cancelledHandled = true + statusMsg = getStatus(ctx, MigrationFailed, logPath) return ctx.Err() } + if err != nil { + statusMsg = getStatus(ctx, MigrationFailed, logPath) + return fmt.Errorf("migration failed: %w", err) + } + // Post-migration outcomes + if status.Failed > 0 { if err := onFailed(status); err != nil { - return err - } - perr.ShowWarning(fmt.Sprintf("Your data has been migrated to DuckLake with issues.\nFor details, see %s\n", logPath)) - } else { - if err := onSuccessful(status); err != nil { - return err + statusMsg = getStatus(ctx, MigrationFailed, logPath) + return fmt.Errorf("failed to cleanup after failed migration: %w", err) } - error_helpers.ShowInfo(fmt.Sprintf("Your data has been migrated to DuckLake.\nFor details, see %s\n", logPath)) + partialMigrated = true + statusMsg = getStatus(ctx, PartialSuccess, logPath) + return err + } + // so we are successful - cleanup + if err := onSuccessful(status); err != nil { + statusMsg = getStatus(ctx, CleanupAfterSuccess, logPath) + return fmt.Errorf("failed to cleanup after successful migration: %w", err) + } + + // all good! + statusMsg = getStatus(ctx, Success, logPath) + return err } -// moveDirContents handles the initial migration move: copy data dir into migrating and move the legacy DB -// into migrated. If any step fails, it removes the migrating directory and shows a support warning. -func moveDirContents(ctx context.Context, dataDefaultDir, migratingDefaultDir string) (err error) { - migratedDir := config.GlobalWorkspaceProfile.GetMigratedDir() - defer func() { - if err != nil { - _ = os.RemoveAll(migratingDefaultDir) - perr.ShowWarning(fmt.Sprintf("Migration initialisation failed. Cleaned up '%s'. Please contact Turbot support.", migratingDefaultDir)) +// moveDataToMigrating ensures the migration folder exists and handles any existing migrating folder +func moveDataToMigrating(ctx context.Context, dataDefaultDir, migratingDefaultDir string) error { + // Ensure the 'migrating' folder exists + migrationDir := config.GlobalWorkspaceProfile.GetMigratingDir() + if err := os.MkdirAll(migrationDir, 0755); err != nil { + return fmt.Errorf("failed to create migration directory: %w", err) + } + + // If the migrating folder exists, it can't have a db as we already checked - delete it + if _, err := os.Stat(migratingDefaultDir); err == nil { + // Directory exists, remove it since we already verified it doesn't contain a db + if err := os.RemoveAll(migratingDefaultDir); err != nil { + return fmt.Errorf("failed to remove existing migrating directory: %w", err) } - }() + } - // 1) Ensure the destination for the DB exists first - // Reason: we will move tailpipe.db after copying data; guaranteeing the target avoids partial moves later. - if err = os.MkdirAll(migratedDir, 0755); err != nil { - return err + // Now move the data directory to the migrating directory + if err := os.Rename(dataDefaultDir, migratingDefaultDir); err != nil { + return fmt.Errorf("failed to move data to migration area: %w", err) } - // 2) Copy ALL data from data/default -> migration/migrating/default (do not delete source yet) - // Reason: copying first keeps the legacy data readable if the process crashes midway. - if err = utils.CopyDir(ctx, dataDefaultDir, migratingDefaultDir); err != nil { - return err + + // now recreate the moved folder + if err := os.MkdirAll(dataDefaultDir, 0755); err != nil { + return fmt.Errorf("failed to recreate data directory after moving: %w", err) } - // 3) Move the DB file from data/default -> migration/migrated/default - // Reason: once data copy succeeded, moving tailpipe.db signals the backup exists and clarifies resume semantics. - if err = utils.MoveFile(filepath.Join(dataDefaultDir, "tailpipe.db"), filepath.Join(migratedDir, "tailpipe.db")); err != nil { - return err + return nil +} + +// promptUserForMigration prompts the user to confirm migration and returns true if they want to continue +func promptUserForMigration(ctx context.Context, dataDir string) (bool, error) { + // Check if context is already cancelled + if ctx.Err() != nil { + return false, ctx.Err() } - // 4) Empty the original data directory last to emulate an atomic move - // Reason: only after successful copy+db move do we clear the source so we never strand users without their legacy data. - if err = utils.EmptyDir(dataDefaultDir); err != nil { - return err + + //nolint: forbidigo // UI output + fmt.Printf("We're about to migrate your data to the Ducklake format.\nIf you'd like a backup, your data folder is located at: %s\n\nContinue? [y/N]: ", dataDir) + + // Use goroutine to read input while allowing context cancellation + type result struct { + response string + err error + } + + resultChan := make(chan result, 1) + go func() { + reader := bufio.NewReader(os.Stdin) + response, err := reader.ReadString('\n') + resultChan <- result{response, err} + }() + + select { + case <-ctx.Done(): + return false, ctx.Err() + case res := <-resultChan: + if res.err != nil { + return false, fmt.Errorf("failed to read user input: %w", res.err) + } + + response := strings.TrimSpace(strings.ToLower(res.response)) + return response == "y" || response == "yes", nil + } +} + +// getStatus returns the appropriate status message based on error type and context +// It handles cancellation checking internally and returns the appropriate message +func getStatus(ctx context.Context, msgType StatusType, logPath string) string { + // Handle cancellation first + if perr.IsContextCancelledError(ctx.Err()) { + switch msgType { + case InitialisationFailed: + return "Migration cancelled. Migration data cleaned up and all original data files remain unchanged. Migration will automatically resume next time you run Tailpipe.\n" + default: + return "Migration cancelled. Migration will automatically resume next time you run Tailpipe.\n" + } + } + + // Handle non-cancellation cases + switch msgType { + case InitialisationFailed: + return "Migration initialisation failed\nMigration data cleaned up and all original data files remain unchanged. Migration will automatically resume next time you run Tailpipe.\n" + case MigrationFailed: + return fmt.Sprintf("Migration failed\nPlease contact tailpipe support. For details, see %s\n", logPath) + case CleanupAfterSuccess: + return fmt.Sprintf("Migration succeeded but cleanup failed\nFor details, see %s\n", logPath) + case PartialSuccess: + // TODO puskar improve this message + return fmt.Sprintf("Your data has been migrated to DuckLake with issues.\nFor details, see %s\n", logPath) + // success + default: + return fmt.Sprintf("Your data has been migrated to DuckLake.\nFor details, see %s\n", logPath) } - return nil } // discoverLegacyTablesAndSchemas enumerates legacy DuckDB views and, for each view, its schema. @@ -317,29 +459,15 @@ func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName str slog.Info("Successfully committed transaction", "table", tableName, "dir", dirPath, "files", filesInLeaf) - // On success, move the entire leaf directory from migrating to migrated - migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() - migratedRoot := config.GlobalWorkspaceProfile.GetMigratedDir() - rel, err := filepath.Rel(migratingRoot, dirPath) - if err != nil { + if err := os.RemoveAll(dirPath); err != nil { + slog.Error("Failed to remove leaf directory after successful migration", "table", tableName) moveTableDirToFailed(ctx, dirPath) status.OnFilesFailed(filesInLeaf) - return err - } - destDir := filepath.Join(migratedRoot, rel) - if err := os.MkdirAll(filepath.Dir(destDir), 0755); err != nil { - moveTableDirToFailed(ctx, dirPath) - status.OnFilesFailed(filesInLeaf) - return err - } - if err := utils.MoveDirContents(ctx, dirPath, destDir); err != nil { - moveTableDirToFailed(ctx, dirPath) - status.OnFilesFailed(filesInLeaf) - return err + return fmt.Errorf("failed to remove leaf directory after migration: %w", err) } _ = os.Remove(dirPath) status.OnFilesMigrated(filesInLeaf) - slog.Info("Migrated leaf node", "table", tableName, "source", dirPath, "destination", destDir) + slog.Info("Migrated leaf node", "table", tableName, "source", dirPath) return nil } @@ -480,13 +608,15 @@ func insertFromParquetFiles(ctx context.Context, tx *sql.Tx, tableName string, c // onSuccessful handles success outcome: cleans migrating db, prunes empty dirs, prints summary func onSuccessful(status *MigrationStatus) error { // Remove any leftover db in migrating - _ = os.Remove(filepath.Join(config.GlobalWorkspaceProfile.GetMigratingDir(), "tailpipe.db")) + if err := os.Remove(filepath.Join(config.GlobalWorkspaceProfile.GetMigratingDir(), "tailpipe.db")); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove leftover migrating db: %w", err) + } + // Prune empty dirs in migrating if err := filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()); err != nil { return fmt.Errorf("failed to prune empty directories in migrating: %w", err) } status.Finish("SUCCESS") - _ = status.WriteStatusToFile() return nil } @@ -495,12 +625,13 @@ func onCancelled(status *MigrationStatus) error { // Do not move db; just prune empties so tree is clean _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) status.Finish("CANCELLED") - _ = status.WriteStatusToFile() return nil } // onFailed handles failure outcome: move db to failed, prune empties, print summary func onFailed(status *MigrationStatus) error { + status.Finish("INCOMPLETE") + failedDefaultDir := config.GlobalWorkspaceProfile.GetMigrationFailedDir() if err := os.MkdirAll(failedDefaultDir, 0755); err != nil { return err @@ -512,7 +643,5 @@ func onFailed(status *MigrationStatus) error { } } _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) - status.Finish("INCOMPLETE") - _ = status.WriteStatusToFile() return nil } diff --git a/internal/migration/status.go b/internal/migration/status.go index 6d3c479b..357773f5 100644 --- a/internal/migration/status.go +++ b/internal/migration/status.go @@ -86,7 +86,6 @@ func (s *MigrationStatus) Finish(outcome string) { // StatusMessage returns a user-facing status message (with stats) based on current migration status func (s *MigrationStatus) StatusMessage() string { - migratedDir := config.GlobalWorkspaceProfile.GetMigratedDir() failedDir := config.GlobalWorkspaceProfile.GetMigrationFailedDir() migratingDir := config.GlobalWorkspaceProfile.GetMigratingDir() @@ -95,11 +94,9 @@ func (s *MigrationStatus) StatusMessage() string { return fmt.Sprintf( "DuckLake migration complete.\n"+ "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ - "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ - "- Backup of migrated legacy data: '%s'\n", + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n", s.Migrated, s.Total, s.Failed, s.Remaining, s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, - migratedDir, ) case "CANCELLED": return fmt.Sprintf( @@ -122,13 +119,11 @@ func (s *MigrationStatus) StatusMessage() string { "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ "- Failed tables (%d): %s\n"+ - "- Failed data and legacy DB: '%s'\n"+ - "- Backup of migrated legacy data: '%s'\n", + "- Failed data and legacy DB: '%s'\n", s.Migrated, s.Total, s.Failed, s.Remaining, s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, len(s.FailedTables), failedList, failedDir, - migratedDir, ) if len(s.Errors) > 0 { base += fmt.Sprintf("\nErrors: %d error(s) occurred during migration\n", len(s.Errors)) From 8909e5857108db53a4cdfacfe0a34146310084e6 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 18 Sep 2025 18:40:11 +0100 Subject: [PATCH 46/61] update go-duckdb reenable merge_adjacent_files --- go.mod | 72 +++++++++--------- go.sum | 144 +++++++++++++++++------------------ internal/database/compact.go | 32 ++++---- internal/database/duck_db.go | 1 - 4 files changed, 124 insertions(+), 125 deletions(-) diff --git a/go.mod b/go.mod index d3e9648b..f3433e3f 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.0 github.com/turbot/go-kit v1.3.0 github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 github.com/turbot/tailpipe-plugin-sdk v0.9.3 @@ -39,39 +39,39 @@ require ( github.com/hashicorp/go-plugin v1.6.1 github.com/hashicorp/go-version v1.7.0 github.com/jedib0t/go-pretty/v6 v6.5.9 - github.com/marcboeker/go-duckdb/v2 v2.3.5 + github.com/marcboeker/go-duckdb/v2 v2.4.0 github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 github.com/thediveo/enumflag/v2 v2.0.5 github.com/turbot/tailpipe-plugin-core v0.2.10 - golang.org/x/text v0.27.0 - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 + golang.org/x/text v0.28.0 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.8 ) require ( github.com/goccy/go-json v0.10.5 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect ) require ( - cel.dev/expr v0.23.0 // indirect + cel.dev/expr v0.24.0 // indirect cloud.google.com/go v0.121.0 // indirect cloud.google.com/go/auth v0.16.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect cloud.google.com/go/iam v1.5.0 // indirect cloud.google.com/go/monitoring v1.24.0 // indirect cloud.google.com/go/storage v1.52.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/apache/arrow-go/v18 v18.4.0 // indirect + github.com/apache/arrow-go/v18 v18.4.1 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go v1.44.183 // indirect @@ -97,7 +97,7 @@ require ( github.com/charmbracelet/lipgloss v1.0.0 // indirect github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/containerd/containerd v1.7.27 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -106,12 +106,12 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/duckdb/duckdb-go-bindings v0.1.17 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 // indirect github.com/elastic/go-grok v0.3.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect @@ -122,8 +122,8 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.0 // indirect github.com/go-git/go-git/v5 v5.13.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -169,8 +169,8 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 // indirect - github.com/marcboeker/go-duckdb/mapping v0.0.11 // indirect + github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 // indirect + github.com/marcboeker/go-duckdb/mapping v0.0.19 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -220,30 +220,30 @@ require ( github.com/zclconf/go-cty-yaml v1.0.3 // indirect github.com/zeebo/errs v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.35.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/oauth2 v0.29.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/term v0.33.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.35.0 // indirect + golang.org/x/tools v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.230.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7ba9ebf6..92a61b42 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss= -cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -184,8 +184,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -621,8 +621,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= @@ -651,8 +651,8 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow-go/v18 v18.4.0 h1:/RvkGqH517iY8bZKc4FD5/kkdwXJGjxf28JIXbJ/oB0= -github.com/apache/arrow-go/v18 v18.4.0/go.mod h1:Aawvwhj8x2jURIzD9Moy72cF0FyJXOpkYpdmGRHcw14= +github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4= +github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= @@ -743,8 +743,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= -github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= @@ -774,18 +774,18 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUn github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/duckdb/duckdb-go-bindings v0.1.17 h1:SjpRwrJ7v0vqnIvLeVFHlhuS72+Lp8xxQ5jIER2LZP4= -github.com/duckdb/duckdb-go-bindings v0.1.17/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 h1:8CLBnsq9YDhi2Gmt3sjSUeXxMzyMQAKefjqUy9zVPFk= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 h1:wjO4I0GhMh2xIpiUgRpzuyOT4KxXLoUS/rjU7UUVvCE= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 h1:HzKQi2C+1jzmwANsPuYH6x9Sfw62SQTjNAEq3OySKFI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 h1:YGSR7AFLw2gJ7IbgLE6DkKYmgKv1LaRSd/ZKF1yh2oE= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 h1:2aduW6fnFnT2Q45PlIgHbatsPOxV9WSZ5B2HzFfxaxA= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/duckdb/duckdb-go-bindings v0.1.19 h1:t8fwgKlr/5BEa5TJzvo3Vdr3yAgoYiR7L/TqyMuUQ2k= +github.com/duckdb/duckdb-go-bindings v0.1.19/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 h1:CdNZfRcFUFxI4Q+1Tu4TBFln9tkIn6bDwVwh9LeEsoo= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 h1:mVijr3WFz3TXZLtAm5Hb6qEnstacZdFI5QQNuE9R2QQ= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 h1:jhchUY24T5bQLOwGyK0BzB6+HQmsRjAbgUZDKWo4ajs= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 h1:CFcH+Bze2OgTaTLM94P3gJ554alnCCDnt1BH/nO8RJ8= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 h1:x/8t04sgCVU8JL0XLUZWmC1FAX13ZjM58EmsyPjvrvY= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -851,13 +851,13 @@ github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkv github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -1100,8 +1100,8 @@ github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -1125,12 +1125,12 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 h1:G1W+GVnUefR8uy7jHdNO+CRMsmFG5mFPIHVAespfFCA= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.10/go.mod h1:jccUb8TYD0p5TsEEeN4SXuslNJHo23QaKOqKD+U6uFU= -github.com/marcboeker/go-duckdb/mapping v0.0.11 h1:fusN1b1l7Myxafifp596I6dNLNhN5Uv/rw31qAqBwqw= -github.com/marcboeker/go-duckdb/mapping v0.0.11/go.mod h1:aYBjFLgfKO0aJIbDtXPiaL5/avRQISveX/j9tMf9JhU= -github.com/marcboeker/go-duckdb/v2 v2.3.5 h1:dpLZdPppUPdwd37/kDEE025iVgQoRw2Q4qXFtXroNIo= -github.com/marcboeker/go-duckdb/v2 v2.3.5/go.mod h1:8adNrftF4Ye29XMrpIl5NYNosTVsZu1mz3C82WdVvrk= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 h1:kMxJBauR2+jwRoSFjiL/DysQtKRBCkNSLZz7GUvEG8A= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.19/go.mod h1:19JWoch6I++gIrWUz1MLImIoFGri9yL54JaWn/Ujvbo= +github.com/marcboeker/go-duckdb/mapping v0.0.19 h1:xZ7LCyFZZm/4X631lOZY74p3QHINMnWJ+OakKw5d3Ao= +github.com/marcboeker/go-duckdb/mapping v0.0.19/go.mod h1:Kz9xYOkhhkgCaGgAg34ciKaks9ED2V7BzHzG6dnVo/o= +github.com/marcboeker/go-duckdb/v2 v2.4.0 h1:XztCDzB0fYvokiVer1myuFX4QvOdnicdTPRp4D+x2Ok= +github.com/marcboeker/go-duckdb/v2 v2.4.0/go.mod h1:qpTBjqtTS5+cfD3o2Sl/W70cmxKj6zhjtvVxs1Wuy7k= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1290,8 +1290,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thediveo/enumflag/v2 v2.0.5 h1:VJjvlAqUb6m6mxOrB/0tfBJI0Kvi9wJ8ulh38xK87i8= @@ -1355,24 +1355,24 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= -go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1393,8 +1393,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1457,8 +1457,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1522,8 +1522,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1553,8 +1553,8 @@ golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= -golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= -golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1671,8 +1671,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1688,8 +1688,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1710,8 +1710,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1783,8 +1783,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2005,10 +2005,10 @@ google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= -google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e h1:UdXH7Kzbj+Vzastr5nVfccbmFsmYNygVLSPk1pEfDoY= -google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e/go.mod h1:085qFyf2+XaZlRdCgKNCIZ3afY2p4HHZdoIRpId8F4A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1:29cjnHVylHwTzH66WfFZqgSQgnxzvWE+jvBwpZCLRxY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2050,8 +2050,8 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2071,8 +2071,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/database/compact.go b/internal/database/compact.go index a9e9c7b4..dd89ffb6 100644 --- a/internal/database/compact.go +++ b/internal/database/compact.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "log/slog" "strings" "time" @@ -46,14 +47,14 @@ func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(Compactio } slog.Info("[SKIPPING] Merging adjacent DuckLake parquet files") - // TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 + // so we should now have multiple, time ordered parquet files // now merge the the parquet files in the duckdb database // the will minimise the parquet file count to the optimum - // if err := mergeParquetFiles(ctx, db); err != nil { - // slog.Error("Failed to merge DuckLake parquet files", "error", err) - // return nil, err - // } + if err := mergeParquetFiles(ctx, db); err != nil { + slog.Error("Failed to merge DuckLake parquet files", "error", err) + return err + } // delete unused files if err := cleanupExpiredFiles(ctx, db); err != nil { @@ -77,17 +78,16 @@ func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(Compactio return nil } -// TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 -//// mergeParquetFiles combines adjacent parquet files in the DuckDB database. -//func mergeParquetFiles(ctx context.Context, db *database.DuckDb) error { -// if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { -// if ctx.Err() != nil { -// return err -// } -// return fmt.Errorf("failed to merge parquet files: %w", err) -// } -// return nil -//} +// mergeParquetFiles combines adjacent parquet files in the DuckDB database. +func mergeParquetFiles(ctx context.Context, db *DuckDb) error { + if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { + if ctx.Err() != nil { + return err + } + return fmt.Errorf("failed to merge parquet files: %w", err) + } + return nil +} // we order data files as follows: // - get list of partition keys matching patterns. For each key: diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index a6f5fcb2..d3033dbb 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -271,7 +271,6 @@ func GetDucklakeInitCommands(readonly bool) []SqlCommand { attachOptions := []string{ fmt.Sprintf("data_path '%s'", config.GlobalWorkspaceProfile.GetDataDir()), "meta_journal_mode 'WAL'", - "meta_synchronous 'NORMAL'", } // if readonly mode is requested, add the option if readonly { From 474a7d09a2dcd7fda3ac1c0684c5693fefe8e9b8 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 18 Sep 2025 18:40:17 +0100 Subject: [PATCH 47/61] update go-duckdb reenable merge_adjacent_files --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index dbd800f9..176917a0 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ import ( "github.com/turbot/tailpipe/cmd" "github.com/turbot/tailpipe/internal/cmdconfig" localconstants "github.com/turbot/tailpipe/internal/constants" - error_helpers "github.com/turbot/tailpipe/internal/error_helpers" + "github.com/turbot/tailpipe/internal/error_helpers" ) var exitCode int From 468727d3cba1d04f7623e37b327515daf5e1551d Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 18 Sep 2025 18:45:19 +0100 Subject: [PATCH 48/61] Revert "update go-duckdb" This reverts commit 474a7d09a2dcd7fda3ac1c0684c5693fefe8e9b8. --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 176917a0..dbd800f9 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ import ( "github.com/turbot/tailpipe/cmd" "github.com/turbot/tailpipe/internal/cmdconfig" localconstants "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/error_helpers" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) var exitCode int From 22bed561627d818c0acf1baabb5d23025c5cd148 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 18 Sep 2025 18:45:19 +0100 Subject: [PATCH 49/61] Revert "update go-duckdb" This reverts commit 8909e5857108db53a4cdfacfe0a34146310084e6. --- go.mod | 72 +++++++++--------- go.sum | 144 +++++++++++++++++------------------ internal/database/compact.go | 32 ++++---- internal/database/duck_db.go | 1 + 4 files changed, 125 insertions(+), 124 deletions(-) diff --git a/go.mod b/go.mod index f3433e3f..d3e9648b 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.11.0 + github.com/stretchr/testify v1.10.0 github.com/turbot/go-kit v1.3.0 github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 github.com/turbot/tailpipe-plugin-sdk v0.9.3 @@ -39,39 +39,39 @@ require ( github.com/hashicorp/go-plugin v1.6.1 github.com/hashicorp/go-version v1.7.0 github.com/jedib0t/go-pretty/v6 v6.5.9 - github.com/marcboeker/go-duckdb/v2 v2.4.0 + github.com/marcboeker/go-duckdb/v2 v2.3.5 github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 github.com/thediveo/enumflag/v2 v2.0.5 github.com/turbot/tailpipe-plugin-core v0.2.10 - golang.org/x/text v0.28.0 - google.golang.org/grpc v1.75.0 - google.golang.org/protobuf v1.36.8 + golang.org/x/text v0.27.0 + google.golang.org/grpc v1.73.0 + google.golang.org/protobuf v1.36.6 ) require ( github.com/goccy/go-json v0.10.5 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect ) require ( - cel.dev/expr v0.24.0 // indirect + cel.dev/expr v0.23.0 // indirect cloud.google.com/go v0.121.0 // indirect cloud.google.com/go/auth v0.16.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/iam v1.5.0 // indirect cloud.google.com/go/monitoring v1.24.0 // indirect cloud.google.com/go/storage v1.52.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/apache/arrow-go/v18 v18.4.1 // indirect + github.com/apache/arrow-go/v18 v18.4.0 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go v1.44.183 // indirect @@ -97,7 +97,7 @@ require ( github.com/charmbracelet/lipgloss v1.0.0 // indirect github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect github.com/containerd/containerd v1.7.27 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -106,12 +106,12 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/duckdb/duckdb-go-bindings v0.1.19 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 // indirect - github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 // indirect - github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 // indirect - github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings v0.1.17 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 // indirect github.com/elastic/go-grok v0.3.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect @@ -122,8 +122,8 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.0 // indirect github.com/go-git/go-git/v5 v5.13.0 // indirect - github.com/go-jose/go-jose/v4 v4.1.1 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -169,8 +169,8 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 // indirect - github.com/marcboeker/go-duckdb/mapping v0.0.19 // indirect + github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 // indirect + github.com/marcboeker/go-duckdb/mapping v0.0.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -220,30 +220,30 @@ require ( github.com/zclconf/go-cty-yaml v1.0.3 // indirect github.com/zeebo/errs v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/sdk v1.37.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/mod v0.27.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/oauth2 v0.29.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/term v0.34.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.36.0 // indirect + golang.org/x/tools v0.35.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.230.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 92a61b42..7ba9ebf6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= -cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss= +cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -184,8 +184,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -621,8 +621,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= @@ -651,8 +651,8 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4= -github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E= +github.com/apache/arrow-go/v18 v18.4.0 h1:/RvkGqH517iY8bZKc4FD5/kkdwXJGjxf28JIXbJ/oB0= +github.com/apache/arrow-go/v18 v18.4.0/go.mod h1:Aawvwhj8x2jURIzD9Moy72cF0FyJXOpkYpdmGRHcw14= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= @@ -743,8 +743,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= -github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= @@ -774,18 +774,18 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUn github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/duckdb/duckdb-go-bindings v0.1.19 h1:t8fwgKlr/5BEa5TJzvo3Vdr3yAgoYiR7L/TqyMuUQ2k= -github.com/duckdb/duckdb-go-bindings v0.1.19/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 h1:CdNZfRcFUFxI4Q+1Tu4TBFln9tkIn6bDwVwh9LeEsoo= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 h1:mVijr3WFz3TXZLtAm5Hb6qEnstacZdFI5QQNuE9R2QQ= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 h1:jhchUY24T5bQLOwGyK0BzB6+HQmsRjAbgUZDKWo4ajs= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 h1:CFcH+Bze2OgTaTLM94P3gJ554alnCCDnt1BH/nO8RJ8= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 h1:x/8t04sgCVU8JL0XLUZWmC1FAX13ZjM58EmsyPjvrvY= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/duckdb/duckdb-go-bindings v0.1.17 h1:SjpRwrJ7v0vqnIvLeVFHlhuS72+Lp8xxQ5jIER2LZP4= +github.com/duckdb/duckdb-go-bindings v0.1.17/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 h1:8CLBnsq9YDhi2Gmt3sjSUeXxMzyMQAKefjqUy9zVPFk= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 h1:wjO4I0GhMh2xIpiUgRpzuyOT4KxXLoUS/rjU7UUVvCE= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 h1:HzKQi2C+1jzmwANsPuYH6x9Sfw62SQTjNAEq3OySKFI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 h1:YGSR7AFLw2gJ7IbgLE6DkKYmgKv1LaRSd/ZKF1yh2oE= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 h1:2aduW6fnFnT2Q45PlIgHbatsPOxV9WSZ5B2HzFfxaxA= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -851,13 +851,13 @@ github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkv github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= -github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -1100,8 +1100,8 @@ github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -1125,12 +1125,12 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 h1:kMxJBauR2+jwRoSFjiL/DysQtKRBCkNSLZz7GUvEG8A= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.19/go.mod h1:19JWoch6I++gIrWUz1MLImIoFGri9yL54JaWn/Ujvbo= -github.com/marcboeker/go-duckdb/mapping v0.0.19 h1:xZ7LCyFZZm/4X631lOZY74p3QHINMnWJ+OakKw5d3Ao= -github.com/marcboeker/go-duckdb/mapping v0.0.19/go.mod h1:Kz9xYOkhhkgCaGgAg34ciKaks9ED2V7BzHzG6dnVo/o= -github.com/marcboeker/go-duckdb/v2 v2.4.0 h1:XztCDzB0fYvokiVer1myuFX4QvOdnicdTPRp4D+x2Ok= -github.com/marcboeker/go-duckdb/v2 v2.4.0/go.mod h1:qpTBjqtTS5+cfD3o2Sl/W70cmxKj6zhjtvVxs1Wuy7k= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 h1:G1W+GVnUefR8uy7jHdNO+CRMsmFG5mFPIHVAespfFCA= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.10/go.mod h1:jccUb8TYD0p5TsEEeN4SXuslNJHo23QaKOqKD+U6uFU= +github.com/marcboeker/go-duckdb/mapping v0.0.11 h1:fusN1b1l7Myxafifp596I6dNLNhN5Uv/rw31qAqBwqw= +github.com/marcboeker/go-duckdb/mapping v0.0.11/go.mod h1:aYBjFLgfKO0aJIbDtXPiaL5/avRQISveX/j9tMf9JhU= +github.com/marcboeker/go-duckdb/v2 v2.3.5 h1:dpLZdPppUPdwd37/kDEE025iVgQoRw2Q4qXFtXroNIo= +github.com/marcboeker/go-duckdb/v2 v2.3.5/go.mod h1:8adNrftF4Ye29XMrpIl5NYNosTVsZu1mz3C82WdVvrk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1290,8 +1290,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= -github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thediveo/enumflag/v2 v2.0.5 h1:VJjvlAqUb6m6mxOrB/0tfBJI0Kvi9wJ8ulh38xK87i8= @@ -1355,24 +1355,24 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= -go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1393,8 +1393,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1457,8 +1457,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1522,8 +1522,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1553,8 +1553,8 @@ golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1671,8 +1671,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1688,8 +1688,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1710,8 +1710,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1783,8 +1783,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2005,10 +2005,10 @@ google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= -google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e h1:UdXH7Kzbj+Vzastr5nVfccbmFsmYNygVLSPk1pEfDoY= +google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e/go.mod h1:085qFyf2+XaZlRdCgKNCIZ3afY2p4HHZdoIRpId8F4A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1:29cjnHVylHwTzH66WfFZqgSQgnxzvWE+jvBwpZCLRxY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2050,8 +2050,8 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2071,8 +2071,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/database/compact.go b/internal/database/compact.go index dd89ffb6..a9e9c7b4 100644 --- a/internal/database/compact.go +++ b/internal/database/compact.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "log/slog" "strings" "time" @@ -47,14 +46,14 @@ func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(Compactio } slog.Info("[SKIPPING] Merging adjacent DuckLake parquet files") - + // TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 // so we should now have multiple, time ordered parquet files // now merge the the parquet files in the duckdb database // the will minimise the parquet file count to the optimum - if err := mergeParquetFiles(ctx, db); err != nil { - slog.Error("Failed to merge DuckLake parquet files", "error", err) - return err - } + // if err := mergeParquetFiles(ctx, db); err != nil { + // slog.Error("Failed to merge DuckLake parquet files", "error", err) + // return nil, err + // } // delete unused files if err := cleanupExpiredFiles(ctx, db); err != nil { @@ -78,16 +77,17 @@ func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(Compactio return nil } -// mergeParquetFiles combines adjacent parquet files in the DuckDB database. -func mergeParquetFiles(ctx context.Context, db *DuckDb) error { - if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { - if ctx.Err() != nil { - return err - } - return fmt.Errorf("failed to merge parquet files: %w", err) - } - return nil -} +// TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 +//// mergeParquetFiles combines adjacent parquet files in the DuckDB database. +//func mergeParquetFiles(ctx context.Context, db *database.DuckDb) error { +// if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { +// if ctx.Err() != nil { +// return err +// } +// return fmt.Errorf("failed to merge parquet files: %w", err) +// } +// return nil +//} // we order data files as follows: // - get list of partition keys matching patterns. For each key: diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index d3033dbb..a6f5fcb2 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -271,6 +271,7 @@ func GetDucklakeInitCommands(readonly bool) []SqlCommand { attachOptions := []string{ fmt.Sprintf("data_path '%s'", config.GlobalWorkspaceProfile.GetDataDir()), "meta_journal_mode 'WAL'", + "meta_synchronous 'NORMAL'", } // if readonly mode is requested, add the option if readonly { From fad790a9c5124535debc6f9fa6f753930ab7f300 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Fri, 19 Sep 2025 11:09:26 +0100 Subject: [PATCH 50/61] update go-duckdb to v2.4.0 --- go.mod | 72 ++++++------- go.sum | 144 +++++++++++++------------- internal/database/compact.go | 32 +++--- internal/database/compaction_types.go | 2 +- internal/database/duck_db.go | 1 - main.go | 2 +- 6 files changed, 125 insertions(+), 128 deletions(-) diff --git a/go.mod b/go.mod index d3e9648b..f3433e3f 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.0 github.com/turbot/go-kit v1.3.0 github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 github.com/turbot/tailpipe-plugin-sdk v0.9.3 @@ -39,39 +39,39 @@ require ( github.com/hashicorp/go-plugin v1.6.1 github.com/hashicorp/go-version v1.7.0 github.com/jedib0t/go-pretty/v6 v6.5.9 - github.com/marcboeker/go-duckdb/v2 v2.3.5 + github.com/marcboeker/go-duckdb/v2 v2.4.0 github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 github.com/thediveo/enumflag/v2 v2.0.5 github.com/turbot/tailpipe-plugin-core v0.2.10 - golang.org/x/text v0.27.0 - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 + golang.org/x/text v0.28.0 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.8 ) require ( github.com/goccy/go-json v0.10.5 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect ) require ( - cel.dev/expr v0.23.0 // indirect + cel.dev/expr v0.24.0 // indirect cloud.google.com/go v0.121.0 // indirect cloud.google.com/go/auth v0.16.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect cloud.google.com/go/iam v1.5.0 // indirect cloud.google.com/go/monitoring v1.24.0 // indirect cloud.google.com/go/storage v1.52.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/apache/arrow-go/v18 v18.4.0 // indirect + github.com/apache/arrow-go/v18 v18.4.1 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go v1.44.183 // indirect @@ -97,7 +97,7 @@ require ( github.com/charmbracelet/lipgloss v1.0.0 // indirect github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/containerd/containerd v1.7.27 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -106,12 +106,12 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/duckdb/duckdb-go-bindings v0.1.17 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 // indirect - github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 // indirect + github.com/duckdb/duckdb-go-bindings v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 // indirect github.com/elastic/go-grok v0.3.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect @@ -122,8 +122,8 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.0 // indirect github.com/go-git/go-git/v5 v5.13.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -169,8 +169,8 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 // indirect - github.com/marcboeker/go-duckdb/mapping v0.0.11 // indirect + github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 // indirect + github.com/marcboeker/go-duckdb/mapping v0.0.19 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -220,30 +220,30 @@ require ( github.com/zclconf/go-cty-yaml v1.0.3 // indirect github.com/zeebo/errs v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.35.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/oauth2 v0.29.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/term v0.33.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.35.0 // indirect + golang.org/x/tools v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.230.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7ba9ebf6..92a61b42 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss= -cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -184,8 +184,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -621,8 +621,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= @@ -651,8 +651,8 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow-go/v18 v18.4.0 h1:/RvkGqH517iY8bZKc4FD5/kkdwXJGjxf28JIXbJ/oB0= -github.com/apache/arrow-go/v18 v18.4.0/go.mod h1:Aawvwhj8x2jURIzD9Moy72cF0FyJXOpkYpdmGRHcw14= +github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4= +github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= @@ -743,8 +743,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= -github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= @@ -774,18 +774,18 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUn github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/duckdb/duckdb-go-bindings v0.1.17 h1:SjpRwrJ7v0vqnIvLeVFHlhuS72+Lp8xxQ5jIER2LZP4= -github.com/duckdb/duckdb-go-bindings v0.1.17/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 h1:8CLBnsq9YDhi2Gmt3sjSUeXxMzyMQAKefjqUy9zVPFk= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 h1:wjO4I0GhMh2xIpiUgRpzuyOT4KxXLoUS/rjU7UUVvCE= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 h1:HzKQi2C+1jzmwANsPuYH6x9Sfw62SQTjNAEq3OySKFI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 h1:YGSR7AFLw2gJ7IbgLE6DkKYmgKv1LaRSd/ZKF1yh2oE= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 h1:2aduW6fnFnT2Q45PlIgHbatsPOxV9WSZ5B2HzFfxaxA= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/duckdb/duckdb-go-bindings v0.1.19 h1:t8fwgKlr/5BEa5TJzvo3Vdr3yAgoYiR7L/TqyMuUQ2k= +github.com/duckdb/duckdb-go-bindings v0.1.19/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 h1:CdNZfRcFUFxI4Q+1Tu4TBFln9tkIn6bDwVwh9LeEsoo= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 h1:mVijr3WFz3TXZLtAm5Hb6qEnstacZdFI5QQNuE9R2QQ= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 h1:jhchUY24T5bQLOwGyK0BzB6+HQmsRjAbgUZDKWo4ajs= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 h1:CFcH+Bze2OgTaTLM94P3gJ554alnCCDnt1BH/nO8RJ8= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 h1:x/8t04sgCVU8JL0XLUZWmC1FAX13ZjM58EmsyPjvrvY= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -851,13 +851,13 @@ github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkv github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -1100,8 +1100,8 @@ github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -1125,12 +1125,12 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 h1:G1W+GVnUefR8uy7jHdNO+CRMsmFG5mFPIHVAespfFCA= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.10/go.mod h1:jccUb8TYD0p5TsEEeN4SXuslNJHo23QaKOqKD+U6uFU= -github.com/marcboeker/go-duckdb/mapping v0.0.11 h1:fusN1b1l7Myxafifp596I6dNLNhN5Uv/rw31qAqBwqw= -github.com/marcboeker/go-duckdb/mapping v0.0.11/go.mod h1:aYBjFLgfKO0aJIbDtXPiaL5/avRQISveX/j9tMf9JhU= -github.com/marcboeker/go-duckdb/v2 v2.3.5 h1:dpLZdPppUPdwd37/kDEE025iVgQoRw2Q4qXFtXroNIo= -github.com/marcboeker/go-duckdb/v2 v2.3.5/go.mod h1:8adNrftF4Ye29XMrpIl5NYNosTVsZu1mz3C82WdVvrk= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 h1:kMxJBauR2+jwRoSFjiL/DysQtKRBCkNSLZz7GUvEG8A= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.19/go.mod h1:19JWoch6I++gIrWUz1MLImIoFGri9yL54JaWn/Ujvbo= +github.com/marcboeker/go-duckdb/mapping v0.0.19 h1:xZ7LCyFZZm/4X631lOZY74p3QHINMnWJ+OakKw5d3Ao= +github.com/marcboeker/go-duckdb/mapping v0.0.19/go.mod h1:Kz9xYOkhhkgCaGgAg34ciKaks9ED2V7BzHzG6dnVo/o= +github.com/marcboeker/go-duckdb/v2 v2.4.0 h1:XztCDzB0fYvokiVer1myuFX4QvOdnicdTPRp4D+x2Ok= +github.com/marcboeker/go-duckdb/v2 v2.4.0/go.mod h1:qpTBjqtTS5+cfD3o2Sl/W70cmxKj6zhjtvVxs1Wuy7k= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1290,8 +1290,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thediveo/enumflag/v2 v2.0.5 h1:VJjvlAqUb6m6mxOrB/0tfBJI0Kvi9wJ8ulh38xK87i8= @@ -1355,24 +1355,24 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= -go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1393,8 +1393,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1457,8 +1457,8 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1522,8 +1522,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1553,8 +1553,8 @@ golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= -golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= -golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1671,8 +1671,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1688,8 +1688,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1710,8 +1710,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1783,8 +1783,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2005,10 +2005,10 @@ google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= -google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e h1:UdXH7Kzbj+Vzastr5nVfccbmFsmYNygVLSPk1pEfDoY= -google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e/go.mod h1:085qFyf2+XaZlRdCgKNCIZ3afY2p4HHZdoIRpId8F4A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1:29cjnHVylHwTzH66WfFZqgSQgnxzvWE+jvBwpZCLRxY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2050,8 +2050,8 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2071,8 +2071,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/database/compact.go b/internal/database/compact.go index a9e9c7b4..c81b5a0a 100644 --- a/internal/database/compact.go +++ b/internal/database/compact.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "log/slog" "strings" "time" @@ -45,15 +46,13 @@ func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(Compactio return err } - slog.Info("[SKIPPING] Merging adjacent DuckLake parquet files") - // TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 // so we should now have multiple, time ordered parquet files // now merge the the parquet files in the duckdb database // the will minimise the parquet file count to the optimum - // if err := mergeParquetFiles(ctx, db); err != nil { - // slog.Error("Failed to merge DuckLake parquet files", "error", err) - // return nil, err - // } + if err := mergeParquetFiles(ctx, db); err != nil { + slog.Error("Failed to merge DuckLake parquet files", "error", err) + return err + } // delete unused files if err := cleanupExpiredFiles(ctx, db); err != nil { @@ -77,17 +76,16 @@ func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(Compactio return nil } -// TODO merge_adjacent_files sometimes crashes, awaiting fix from DuckDb https://github.com/turbot/tailpipe/issues/530 -//// mergeParquetFiles combines adjacent parquet files in the DuckDB database. -//func mergeParquetFiles(ctx context.Context, db *database.DuckDb) error { -// if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { -// if ctx.Err() != nil { -// return err -// } -// return fmt.Errorf("failed to merge parquet files: %w", err) -// } -// return nil -//} +// mergeParquetFiles combines adjacent parquet files in the DuckDB database. +func mergeParquetFiles(ctx context.Context, db *DuckDb) error { + if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { + if ctx.Err() != nil { + return err + } + return fmt.Errorf("failed to merge parquet files: %w", err) + } + return nil +} // we order data files as follows: // - get list of partition keys matching patterns. For each key: diff --git a/internal/database/compaction_types.go b/internal/database/compaction_types.go index bf052f7e..2fc88d87 100644 --- a/internal/database/compaction_types.go +++ b/internal/database/compaction_types.go @@ -77,7 +77,7 @@ func getFileRangesForPartitionKey(ctx context.Context, db *DuckDb, pk *partition on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 join __ducklake_metadata_tailpipe_ducklake.ducklake_table t on df.table_id = t.table_id - join __ducklake_metadata_tailpipe_ducklake.ducklake_file_column_statistics fcs + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_column_stats fcs on df.data_file_id = fcs.data_file_id and df.table_id = fcs.table_id join __ducklake_metadata_tailpipe_ducklake.ducklake_column c diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index a6f5fcb2..d3033dbb 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -271,7 +271,6 @@ func GetDucklakeInitCommands(readonly bool) []SqlCommand { attachOptions := []string{ fmt.Sprintf("data_path '%s'", config.GlobalWorkspaceProfile.GetDataDir()), "meta_journal_mode 'WAL'", - "meta_synchronous 'NORMAL'", } // if readonly mode is requested, add the option if readonly { diff --git a/main.go b/main.go index dbd800f9..176917a0 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ import ( "github.com/turbot/tailpipe/cmd" "github.com/turbot/tailpipe/internal/cmdconfig" localconstants "github.com/turbot/tailpipe/internal/constants" - error_helpers "github.com/turbot/tailpipe/internal/error_helpers" + "github.com/turbot/tailpipe/internal/error_helpers" ) var exitCode int From 6754bdad8bdb765bbfc6feff81507f1685f1e4c6 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Fri, 19 Sep 2025 11:09:56 +0100 Subject: [PATCH 51/61] Update migration status reporting. --- internal/migration/migration.go | 120 ++++++++++++++++---------------- internal/migration/status.go | 74 +++++++++++--------- 2 files changed, 103 insertions(+), 91 deletions(-) diff --git a/internal/migration/migration.go b/internal/migration/migration.go index 7fa3459b..e593e736 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -38,9 +38,24 @@ const ( // MigrateDataToDucklake performs migration of views from tailpipe.db and associated parquet files // into the new DuckLake metadata catalog func MigrateDataToDucklake(ctx context.Context) (err error) { + // define a status message var - this will be set when we encounter any issues - or when we are successful + // this will be printed at the end of the function var statusMsg string var partialMigrated bool + // if there is a status message, print it out at the end + defer func() { + if statusMsg != "" { + if err != nil || partialMigrated { + // if there is an error or a partial migration, show as warning + error_helpers.ShowWarning(statusMsg) + } else { + // show as info if there is no error, or if it is not a partial migration + error_helpers.ShowInfo(statusMsg) + } + } + }() + // Determine source and migration directories dataDefaultDir := config.GlobalWorkspaceProfile.GetDataDir() migratingDefaultDir := config.GlobalWorkspaceProfile.GetMigratingDir() @@ -83,34 +98,6 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { return context.Canceled } - // Initialize migration status - status := NewMigrationStatus(0) - - // add any error to status and write to file before returning - defer func() { - if err != nil { - status.AddError(err) - if perr.IsContextCancelledError(ctx.Err()) { - // set cancel status and prune the tree - _ = onCancelled(status) - } else { - status.Finish("FAILED") - } - } - - if statusMsg != "" { - if err != nil || partialMigrated { - // if there is an error or a partial migration, show as warning - error_helpers.ShowWarning(statusMsg) - } else { - // show as info if there is no error, or if it is not a partial migration - error_helpers.ShowInfo(statusMsg) - } - } - // write the status back - _ = status.WriteStatusToFile() - }() - logPath := filepath.Join(config.GlobalWorkspaceProfile.GetMigrationDir(), "migration.log") // Spinner for migration progress @@ -120,7 +107,7 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { spinner.WithHiddenCursor(true), spinner.WithWriter(os.Stdout), ) - + // set suffix and start sp.Suffix = " Migrating data to DuckLake format" sp.Start() defer sp.Stop() @@ -140,7 +127,7 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { // This returns the list of views and a map of view name to its schema views, schemas, err := discoverLegacyTablesAndSchemas(ctx, discoveryDbPath) if err != nil { - statusMsg = getStatus(ctx, InitialisationFailed, "") + statusMsg = getStatusMessage(ctx, InitialisationFailed, "") return fmt.Errorf("failed to discover legacy tables: %w", err) } @@ -155,7 +142,7 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { if err := moveDataToMigrating(ctx, dataDefaultDir, migratingDefaultDir); err != nil { slog.Error("Failed to move data to migrating directory", "error", err) - statusMsg = getStatus(ctx, InitialisationFailed, logPath) + statusMsg = getStatusMessage(ctx, InitialisationFailed, logPath) return err } } @@ -171,7 +158,7 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { baseDir := migratingDefaultDir matchedTableDirs, unmatchedTableDirs, err = findMatchingTableDirs(baseDir, views) if err != nil { - statusMsg = getStatus(ctx, MigrationFailed, logPath) + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) return fmt.Errorf("failed to find matching table directories: %w", err) } @@ -179,64 +166,82 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { sp.Suffix = " Migrating data to DuckLake format: archiving tables without views" // move the unmatched table directories to 'unmigrated' if err = archiveUnmatchedDirs(ctx, unmatchedTableDirs); err != nil { - statusMsg = getStatus(ctx, MigrationFailed, logPath) + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) return fmt.Errorf("failed to archive unmatched table directories: %w", err) } } - // Initialize status with total tables to migrate - status.Total = len(matchedTableDirs) - status.update() - sp.Suffix = " Migrating data to DuckLake format: counting parquet files" // Pre-compute total parquet files across matched directories + sp.Suffix = " Migrating data to DuckLake format: counting parquet files" totalFiles, err := countParquetFiles(ctx, matchedTableDirs) - if err == nil { - status.TotalFiles = totalFiles - status.updateFiles() + if err != nil { + return fmt.Errorf("failed to count parquet files: %w", err) } - sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format (%d/%d, %0.1f%%) | parquet files (%d/%d)", status.Migrated, status.Total, status.ProgressPercent, status.MigratedFiles, status.TotalFiles) - - updateStatus := func(st *MigrationStatus) { - sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format (%d/%d, %0.1f%%) | parquet files (%d/%d)", st.Migrated, st.Total, st.ProgressPercent, st.MigratedFiles, st.TotalFiles) + // create an update func to update th espinner + updateFunc := func(st *MigrationStatus) { + sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format | tables (%d/%d) | parquet files (%d/%d, %0.1f%%))", st.MigratedTables, st.TotalTables, st.MigratedFiles, st.TotalFiles, st.ProgressPercent) } + // Initialize migration status, paaing in the file and table count and status update func + totalTables := len(matchedTableDirs) + status := NewMigrationStatus(totalFiles, totalTables, updateFunc) + // ensure we save the status to file at the end + defer func() { + // add any error to status and write to file before returning + if err != nil { + status.AddError(err) + if perr.IsContextCancelledError(ctx.Err()) { + // set cancel status and prune the tree + _ = onCancelled(status) + } else { + status.Finish("FAILED") + } + } + + // write the status back + _ = status.WriteStatusToFile() + }() + + // call initial update on status - this will set the spinner message correctly + status.update() + // STEP 5: Do Migration: Traverse matched table directories, find leaf nodes with parquet files, // and perform INSERT within a transaction. On success, move leaf dir to migrated. - err = doMigration(ctx, matchedTableDirs, schemas, status, updateStatus) + err = doMigration(ctx, matchedTableDirs, schemas, status) // If cancellation arrived during migration, prefer the CANCELLED outcome and do not // treat it as a failure (which would incorrectly move tailpipe.db to failed) if perr.IsContextCancelledError(ctx.Err()) { - statusMsg = getStatus(ctx, MigrationFailed, logPath) + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) return ctx.Err() } if err != nil { - statusMsg = getStatus(ctx, MigrationFailed, logPath) + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) return fmt.Errorf("migration failed: %w", err) } // Post-migration outcomes - if status.Failed > 0 { + if status.FailedTables > 0 { if err := onFailed(status); err != nil { - statusMsg = getStatus(ctx, MigrationFailed, logPath) + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) return fmt.Errorf("failed to cleanup after failed migration: %w", err) } partialMigrated = true - statusMsg = getStatus(ctx, PartialSuccess, logPath) + statusMsg = getStatusMessage(ctx, PartialSuccess, logPath) return err } // so we are successful - cleanup if err := onSuccessful(status); err != nil { - statusMsg = getStatus(ctx, CleanupAfterSuccess, logPath) + statusMsg = getStatusMessage(ctx, CleanupAfterSuccess, logPath) return fmt.Errorf("failed to cleanup after successful migration: %w", err) } // all good! - statusMsg = getStatus(ctx, Success, logPath) + statusMsg = getStatusMessage(ctx, Success, logPath) return err } @@ -305,9 +310,9 @@ func promptUserForMigration(ctx context.Context, dataDir string) (bool, error) { } } -// getStatus returns the appropriate status message based on error type and context +// getStatusMessage returns the appropriate status message based on error type and context // It handles cancellation checking internally and returns the appropriate message -func getStatus(ctx context.Context, msgType StatusType, logPath string) string { +func getStatusMessage(ctx context.Context, msgType StatusType, logPath string) string { // Handle cancellation first if perr.IsContextCancelledError(ctx.Err()) { switch msgType { @@ -502,10 +507,7 @@ func archiveUnmatchedDirs(ctx context.Context, unmatchedTableDirs []string) erro } // doMigration performs the migration of the matched table directories and updates status -func doMigration(ctx context.Context, matchedTableDirs []string, schemas map[string]*schema.TableSchema, status *MigrationStatus, onUpdate func(*MigrationStatus)) error { - if onUpdate == nil { - return fmt.Errorf("onUpdate callback is required") - } +func doMigration(ctx context.Context, matchedTableDirs []string, schemas map[string]*schema.TableSchema, status *MigrationStatus) error { ducklakeDb, err := database.NewDuckDb(database.WithDuckLake()) if err != nil { return err @@ -524,8 +526,6 @@ func doMigration(ctx context.Context, matchedTableDirs []string, schemas map[str } else { status.OnTableMigrated() } - // update our status - onUpdate(status) } return nil } diff --git a/internal/migration/status.go b/internal/migration/status.go index 357773f5..1d18b96a 100644 --- a/internal/migration/status.go +++ b/internal/migration/status.go @@ -12,10 +12,10 @@ import ( type MigrationStatus struct { Status string `json:"status"` - Total int `json:"total"` - Migrated int `json:"migrated"` - Failed int `json:"failed"` - Remaining int `json:"remaining"` + TotalTables int `json:"totaltables"` + MigratedTables int `json:"migratedtables"` + FailedTables int `json:"failedtables"` + RemainingTables int `json:"remainingtables"` ProgressPercent float64 `json:"progress_percent"` TotalFiles int `json:"total_files"` @@ -23,25 +23,35 @@ type MigrationStatus struct { FailedFiles int `json:"failed_files"` RemainingFiles int `json:"remaining_files"` - FailedTables []string `json:"failed_tables,omitempty"` - StartTime time.Time `json:"start_time"` - Duration time.Duration `json:"duration"` + FailedTableNames []string `json:"failed_table_names,omitempty"` + StartTime time.Time `json:"start_time"` + Duration time.Duration `json:"duration"` Errors []string `json:"errors,omitempty"` + + // update func + updateFunc func(st *MigrationStatus) } -func NewMigrationStatus(total int) *MigrationStatus { - return &MigrationStatus{Total: total, Remaining: total, StartTime: time.Now()} +func NewMigrationStatus(totalFiles, totalTables int, updateFunc func(st *MigrationStatus)) *MigrationStatus { + return &MigrationStatus{ + TotalTables: totalTables, + RemainingTables: totalTables, + TotalFiles: totalFiles, + RemainingFiles: totalFiles, + StartTime: time.Now(), + updateFunc: updateFunc, + } } func (s *MigrationStatus) OnTableMigrated() { - s.Migrated++ + s.MigratedTables++ s.update() } func (s *MigrationStatus) OnTableFailed(tableName string) { - s.Failed++ - s.FailedTables = append(s.FailedTables, tableName) + s.FailedTables++ + s.FailedTableNames = append(s.FailedTableNames, tableName) s.update() } @@ -50,7 +60,7 @@ func (s *MigrationStatus) OnFilesMigrated(n int) { return } s.MigratedFiles += n - s.updateFiles() + s.update() } func (s *MigrationStatus) OnFilesFailed(n int) { @@ -58,7 +68,7 @@ func (s *MigrationStatus) OnFilesFailed(n int) { return } s.FailedFiles += n - s.updateFiles() + s.update() } func (s *MigrationStatus) AddError(err error) { @@ -68,17 +78,6 @@ func (s *MigrationStatus) AddError(err error) { s.Errors = append(s.Errors, err.Error()) } -func (s *MigrationStatus) update() { - s.Remaining = s.Total - s.Migrated - s.Failed - if s.Total > 0 { - s.ProgressPercent = float64(s.Migrated) * 100.0 / float64(s.Total) - } -} - -func (s *MigrationStatus) updateFiles() { - s.RemainingFiles = s.TotalFiles - s.MigratedFiles - s.FailedFiles -} - func (s *MigrationStatus) Finish(outcome string) { s.Status = outcome s.Duration = time.Since(s.StartTime) @@ -95,7 +94,7 @@ func (s *MigrationStatus) StatusMessage() string { "DuckLake migration complete.\n"+ "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n", - s.Migrated, s.Total, s.Failed, s.Remaining, + s.MigratedTables, s.TotalTables, s.FailedTables, s.RemainingTables, s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, ) case "CANCELLED": @@ -105,14 +104,14 @@ func (s *MigrationStatus) StatusMessage() string { "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ "- Legacy DB preserved: '%s/tailpipe.db'\n\n"+ "Re-run Tailpipe to resume migrating your data.\n", - s.Migrated, s.Total, s.Failed, s.Remaining, + s.MigratedTables, s.TotalTables, s.FailedTables, s.RemainingTables, s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, migratingDir, ) case "INCOMPLETE": failedList := "(none)" - if len(s.FailedTables) > 0 { - failedList = strings.Join(s.FailedTables, ", ") + if len(s.FailedTableNames) > 0 { + failedList = strings.Join(s.FailedTableNames, ", ") } base := fmt.Sprintf( "DuckLake migration completed with issues.\n"+ @@ -120,9 +119,9 @@ func (s *MigrationStatus) StatusMessage() string { "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ "- Failed tables (%d): %s\n"+ "- Failed data and legacy DB: '%s'\n", - s.Migrated, s.Total, s.Failed, s.Remaining, + s.MigratedTables, s.TotalTables, s.FailedTables, s.RemainingTables, s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, - len(s.FailedTables), failedList, + len(s.FailedTableNames), failedList, failedDir, ) if len(s.Errors) > 0 { @@ -150,3 +149,16 @@ func (s *MigrationStatus) WriteStatusToFile() error { } return os.WriteFile(statsFile, []byte(msg), 0600) } + +// update recalculates remaining counts and progress percent, and calls the update func if set +func (s *MigrationStatus) update() { + s.RemainingTables = s.TotalTables - s.MigratedTables - s.FailedTables + s.RemainingFiles = s.TotalFiles - s.MigratedFiles - s.FailedFiles + if s.TotalFiles > 0 { + s.ProgressPercent = float64(s.MigratedFiles+s.FailedFiles) * 100.0 / float64(s.TotalFiles) + } + // call our update func + if s.updateFunc != nil { + s.updateFunc(s) + } +} From 2f7cde065ed95e818ba33ad027c92fb9559f4669 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Sep 2025 11:11:40 +0100 Subject: [PATCH 52/61] fix status message --- internal/migration/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/migration/migration.go b/internal/migration/migration.go index e593e736..3948d4ef 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -180,7 +180,7 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { // create an update func to update th espinner updateFunc := func(st *MigrationStatus) { - sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format | tables (%d/%d) | parquet files (%d/%d, %0.1f%%))", st.MigratedTables, st.TotalTables, st.MigratedFiles, st.TotalFiles, st.ProgressPercent) + sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format | tables (%d/%d) | parquet files (%d/%d, %0.1f%%)", st.MigratedTables, st.TotalTables, st.MigratedFiles, st.TotalFiles, st.ProgressPercent) } // Initialize migration status, paaing in the file and table count and status update func From 58e67e56eed1ef7aa0f3d81b15dcca0c8d067f01 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Fri, 19 Sep 2025 17:18:56 +0530 Subject: [PATCH 53/61] minor migration message tweaks --- internal/migration/migration.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/internal/migration/migration.go b/internal/migration/migration.go index 3948d4ef..f3879171 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -282,7 +282,7 @@ func promptUserForMigration(ctx context.Context, dataDir string) (bool, error) { } //nolint: forbidigo // UI output - fmt.Printf("We're about to migrate your data to the Ducklake format.\nIf you'd like a backup, your data folder is located at: %s\n\nContinue? [y/N]: ", dataDir) + fmt.Printf("This version of Tailpipe requires your data to be migrated to the new Ducklake format.\n\nThis operation is irreversible. If desired, back up your data folder (%s) before proceeding.\n\nContinue? [y/N]: ", dataDir) // Use goroutine to read input while allowing context cancellation type result struct { @@ -326,17 +326,16 @@ func getStatusMessage(ctx context.Context, msgType StatusType, logPath string) s // Handle non-cancellation cases switch msgType { case InitialisationFailed: - return "Migration initialisation failed\nMigration data cleaned up and all original data files remain unchanged. Migration will automatically resume next time you run Tailpipe.\n" + return "Migration initialisation failed.\nMigration data cleaned up and all original data files remain unchanged. Migration will automatically resume next time you run Tailpipe.\n" case MigrationFailed: - return fmt.Sprintf("Migration failed\nPlease contact tailpipe support. For details, see %s\n", logPath) + return fmt.Sprintf("Migration failed.\nFor details, see %s\nPlease contact Turbot support on Slack (#tailpipe).", logPath) case CleanupAfterSuccess: return fmt.Sprintf("Migration succeeded but cleanup failed\nFor details, see %s\n", logPath) case PartialSuccess: - // TODO puskar improve this message - return fmt.Sprintf("Your data has been migrated to DuckLake with issues.\nFor details, see %s\n", logPath) + return fmt.Sprintf("Your data has been migrated to DuckLake, but some files could not be migrated.\nFor details, see %s\nIf you need help, please contact Turbot support on Slack (#tailpipe).", logPath) // success default: - return fmt.Sprintf("Your data has been migrated to DuckLake.\nFor details, see %s\n", logPath) + return fmt.Sprintf("Your data has been migrated to DuckLake format.\nFor details, see %s\n", logPath) } } @@ -464,13 +463,11 @@ func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName str slog.Info("Successfully committed transaction", "table", tableName, "dir", dirPath, "files", filesInLeaf) + // Clean up the now-empty source dir. If this fails (e.g., hidden files), log and continue; + // do NOT classify as a failed migration, since data has been committed successfully. if err := os.RemoveAll(dirPath); err != nil { - slog.Error("Failed to remove leaf directory after successful migration", "table", tableName) - moveTableDirToFailed(ctx, dirPath) - status.OnFilesFailed(filesInLeaf) - return fmt.Errorf("failed to remove leaf directory after migration: %w", err) + slog.Warn("Cleanup: could not remove migrated leaf directory", "table", tableName, "dir", dirPath, "error", err) } - _ = os.Remove(dirPath) status.OnFilesMigrated(filesInLeaf) slog.Info("Migrated leaf node", "table", tableName, "source", dirPath) return nil @@ -532,6 +529,10 @@ func doMigration(ctx context.Context, matchedTableDirs []string, schemas map[str // moveTableDirToFailed moves a table directory from migrating to failed, preserving relative path. func moveTableDirToFailed(ctx context.Context, dirPath string) { + // If the migration was cancelled, do not classify this table as failed + if perr.IsContextCancelledError(ctx.Err()) { + return + } migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() failedRoot := config.GlobalWorkspaceProfile.GetMigrationFailedDir() rel, err := filepath.Rel(migratingRoot, dirPath) From b417ca3a1a9a61b641c807d9393a5262020aa038 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Fri, 19 Sep 2025 13:19:37 +0100 Subject: [PATCH 54/61] Set ExitCodeMigrationUnsupported exit code for migration unsupported (#573) --- cmd/root.go | 10 +++++++++- go.mod | 4 ++-- go.sum | 4 ++-- internal/migration/errors.go | 27 +++++++++++++++++++++++++++ internal/migration/migration.go | 33 ++++++++++++++++++++++----------- 5 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 internal/migration/errors.go diff --git a/cmd/root.go b/cmd/root.go index 6b5669c7..402c3c0e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "os" "github.com/spf13/cobra" @@ -11,6 +12,7 @@ import ( "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/migration" ) var exitCode int @@ -68,8 +70,14 @@ func Execute() int { // set the error output to stdout (as it;s common usage to redirect stderr to a file to capture logs rootCmd.SetErr(os.Stdout) + // if the error is dues to unsupported migration, set a specific exit code - this will bve picked up by powerpipe if err := rootCmd.Execute(); err != nil { - exitCode = -1 + var unsupportedErr *migration.UnsupportedError + if errors.As(err, &unsupportedErr) { + exitCode = pconstants.ExitCodeMigrationUnsupported + } else { + exitCode = 1 + } } return exitCode } diff --git a/go.mod b/go.mod index f3433e3f..0a633771 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.0 replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 -// github.com/turbot/pipe-fittings/v2 => ../pipe-fittings +//github.com/turbot/pipe-fittings/v2 => ../pipe-fittings //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core // github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) @@ -19,7 +19,7 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.11.0 github.com/turbot/go-kit v1.3.0 - github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 + github.com/turbot/pipe-fittings/v2 v2.7.0 github.com/turbot/tailpipe-plugin-sdk v0.9.3 github.com/zclconf/go-cty v1.16.3 golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 diff --git a/go.sum b/go.sum index 92a61b42..1640c3f0 100644 --- a/go.sum +++ b/go.sum @@ -1308,8 +1308,8 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 h1:FfKLkfbNmwxyPQIqDCd7m6o9bmtPB7D8a5txbVzjZp4= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= +github.com/turbot/pipe-fittings/v2 v2.7.0 h1:eCmpMNlVtV3AxOzsn8njE3O6aoHc74WVAHOntia2hqY= +github.com/turbot/pipe-fittings/v2 v2.7.0/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= diff --git a/internal/migration/errors.go b/internal/migration/errors.go new file mode 100644 index 00000000..231ba6f0 --- /dev/null +++ b/internal/migration/errors.go @@ -0,0 +1,27 @@ +package migration + +import "fmt" + +// UnsupportedError represents an error when migration is not supported +// due to specific command line arguments or configuration +type UnsupportedError struct { + Reason string +} + +func (e *UnsupportedError) Error() string { + msgFormat := "data must be migrated to Ducklake format - migration is not supported with '%s'.\n\nRun 'tailpipe query' to migrate your data to DuckLake format" + return fmt.Sprintf(msgFormat, e.Reason) +} + +func (e *UnsupportedError) Is(target error) bool { + _, ok := target.(*UnsupportedError) + return ok +} + +func (e *UnsupportedError) As(target interface{}) bool { + if t, ok := target.(**UnsupportedError); ok { + *t = e + return true + } + return false +} diff --git a/internal/migration/migration.go b/internal/migration/migration.go index f3879171..08893322 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -38,6 +38,7 @@ const ( // MigrateDataToDucklake performs migration of views from tailpipe.db and associated parquet files // into the new DuckLake metadata catalog func MigrateDataToDucklake(ctx context.Context) (err error) { + slog.Info("Starting data migration to DuckLake format") // define a status message var - this will be set when we encounter any issues - or when we are successful // this will be printed at the end of the function var statusMsg string @@ -82,11 +83,8 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { // if the output for this command is a machine readable format (csv/json) or progress is false, // it is possible/likely that tailpipe is being used in a non interactive way - in this case, // we should not prompt the user, instead return an error - msgFormat := "data must be migrated to Ducklake format - migration is not supported with '%s'.\n\nRun 'tailpipe query' to migrate your data to DuckLake format" - if error_helpers.IsMachineReadableOutput() { - return fmt.Errorf(msgFormat, "--output "+viper.GetString(constants.ArgOutput)) - } else if viper.IsSet(constants.ArgProgress) && !viper.GetBool(constants.ArgProgress) { - return fmt.Errorf(msgFormat, "--progress=false") + if err := checkMigrationSupported(); err != nil { + return err } // Prompt the user to confirm migration @@ -131,9 +129,6 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { return fmt.Errorf("failed to discover legacy tables: %w", err) } - slog.Info("Views: ", "views", views) - slog.Info("Schemas: ", "schemas", schemas) - // STEP 3: If this is the first time we are migrating(tables in ~/.tailpipe/data) then move the whole contents of data dir // into ~/.tailpipe/migration/migrating respecting the same folder structure. // We do this by simply renaming the directory. @@ -246,6 +241,24 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { return err } +// check if the data migration is supported, based on the current arguments +// if the output for this command is a machine readable format (csv/json) or progress is false, +// it is possible/likely that tailpipe is being used in a non interactive way - in this case, +// we should not prompt the user, instead return an error +// NOTE: set exit code to +func checkMigrationSupported() error { + if error_helpers.IsMachineReadableOutput() { + return &UnsupportedError{ + Reason: "--output " + viper.GetString(constants.ArgOutput), + } + } else if viper.IsSet(constants.ArgProgress) && !viper.GetBool(constants.ArgProgress) { + return &UnsupportedError{ + Reason: "--progress=false", + } + } + return nil +} + // moveDataToMigrating ensures the migration folder exists and handles any existing migrating folder func moveDataToMigrating(ctx context.Context, dataDefaultDir, migratingDefaultDir string) error { // Ensure the 'migrating' folder exists @@ -430,8 +443,6 @@ func migrateTableDirectory(ctx context.Context, db *database.DuckDb, tableName s func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus, parquetFiles []string) error { filesInLeaf := len(parquetFiles) - // Placeholder: validate schema (from 'ts') against parquet files if needed - slog.Info("Found leaf node with parquet files", "table", tableName, "dir", dirPath, "files", filesInLeaf) // Begin transaction tx, err := db.BeginTx(ctx, nil) @@ -469,7 +480,7 @@ func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName str slog.Warn("Cleanup: could not remove migrated leaf directory", "table", tableName, "dir", dirPath, "error", err) } status.OnFilesMigrated(filesInLeaf) - slog.Info("Migrated leaf node", "table", tableName, "source", dirPath) + slog.Debug("Migrated leaf node", "table", tableName, "source", dirPath) return nil } From bbb49b5843703be0f5dcfd5427914142e33bdefe Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Mon, 22 Sep 2025 10:03:31 +0100 Subject: [PATCH 55/61] Update changelog for v0.7.0 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03a15fd7..268942cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +## v0.7.0 [2025-09-22] + +### _Major Changes_ +* Replace native Parquet conversion with a **DuckLake database backend**. ([#546](https://github.com/turbot/tailpipe/issues/546)) + - DuckLake is DuckDB’s new lakehouse format: data remains in Parquet files, but metadata is efficiently tracked in a + separate DuckDB database. + - DuckLake supports function-based partitioning, which allows data to be partitioned by year and month. This enables + efficient file pruning on `tp_timestamp` without needing a separate `tp_date` filter. A `tp_date` column will still + be present for compatibility, but it is no longer required for efficient query filtering. + - Existing data will be **automatically migrated** the next time Tailpipe runs. Migration does **not** + occur if progress output is disabled (`--progress=false`) or when using machine-readable output (`json`, `line`, + `csv`). + +* The `connect` command now returns the path to an **initialisation SQL script** instead of the database path. ([#550](https://github.com/turbot/tailpipe/issues/550)) + - The script sets up DuckDB with required extensions, attaches the Tailpipe database, and defines views with optional + filters. + - You can pass the generated script to DuckDB using the `--init` argument to immediately configure the session. For + example: + ```sh + duckdb --init $(tailpipe connect) + ``` + +### _Bug Fixes_ +* Include partitions for local plugins in the `tailpipe plugin list` command. ([#538](https://github.com/turbot/tailpipe/issues/538)) + + ## v0.6.2 [2025-07-24] _Bug fixes_ * Fix issue where `--to` was not respected for zero granularity data. ([#483](https://github.com/turbot/tailpipe/issues/483)) From 4edc40c44d9558e2dd2b8a1950cb2d6269722a77 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Mon, 22 Sep 2025 16:20:45 +0530 Subject: [PATCH 56/61] Update changelog to include duckdb version --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 268942cf..074685a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ example: ```sh duckdb --init $(tailpipe connect) - ``` + ``` + **Note:** To ensure compatibility with DuckLake features, make sure you’re using DuckDB version 1.4.0 or later. ### _Bug Fixes_ * Include partitions for local plugins in the `tailpipe plugin list` command. ([#538](https://github.com/turbot/tailpipe/issues/538)) From 67d339a56523096fe6e63ee3cc30fde788badbe9 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Mon, 22 Sep 2025 17:35:32 +0530 Subject: [PATCH 57/61] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074685a6..fba086bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ occur if progress output is disabled (`--progress=false`) or when using machine-readable output (`json`, `line`, `csv`). + **Note:** For CentOS/RHEL users, the minimum supported version is now **CentOS Stream 10 / RHEL 10** due to `libstdc++` library compatibility. + * The `connect` command now returns the path to an **initialisation SQL script** instead of the database path. ([#550](https://github.com/turbot/tailpipe/issues/550)) - The script sets up DuckDB with required extensions, attaches the Tailpipe database, and defines views with optional filters. @@ -19,7 +21,7 @@ ```sh duckdb --init $(tailpipe connect) ``` - **Note:** To ensure compatibility with DuckLake features, make sure you’re using DuckDB version 1.4.0 or later. + **Note:** The minimum supported DuckDB version is 1.4.0. ### _Bug Fixes_ * Include partitions for local plugins in the `tailpipe plugin list` command. ([#538](https://github.com/turbot/tailpipe/issues/538)) From 566c9ff6662c315ca6534cd927e8cbe280325c34 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 6 Oct 2025 16:49:47 +0530 Subject: [PATCH 58/61] Update smoke test script to use init_script_path (#577) --- scripts/smoke_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index 5b283a48..424e0622 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -16,7 +16,7 @@ jq --version /usr/local/bin/tailpipe query "SELECT 1 as smoke_test" # verify basic query works # Test connect functionality -DB_FILE=$(/usr/local/bin/tailpipe connect --output json | jq -r '.database_filepath') +DB_FILE=$(/usr/local/bin/tailpipe connect --output json | jq -r '.init_script_path') # Verify the database file exists if [ -f "$DB_FILE" ]; then From e432ead7fce155b8004105d060f1808e104a8c5d Mon Sep 17 00:00:00 2001 From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com> Date: Tue, 7 Oct 2025 16:54:41 +0530 Subject: [PATCH 59/61] Update goreleaser-cross image to fix c++ errors on centos/ubi 9 (#579) --- .acceptance.goreleaser.yml | 5 ++++ .github/workflows/01-tailpipe-release.yaml | 16 ++++++++++++- .github/workflows/11-test-acceptance.yaml | 2 +- .goreleaser.yml | 28 +++++++++++++++------- Makefile | 4 +++- 5 files changed, 43 insertions(+), 12 deletions(-) diff --git a/.acceptance.goreleaser.yml b/.acceptance.goreleaser.yml index f96f03d5..b3e10084 100644 --- a/.acceptance.goreleaser.yml +++ b/.acceptance.goreleaser.yml @@ -12,10 +12,15 @@ builds: env: - CC=x86_64-linux-gnu-gcc - CXX=x86_64-linux-gnu-g++ + - CGO_ENABLED=1 + - GOFLAGS= + - CGO_LDFLAGS= ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + tags: [] + archives: - id: homebrew format: tar.gz diff --git a/.github/workflows/01-tailpipe-release.yaml b/.github/workflows/01-tailpipe-release.yaml index c2e8e2ea..4345a0fd 100644 --- a/.github/workflows/01-tailpipe-release.yaml +++ b/.github/workflows/01-tailpipe-release.yaml @@ -67,7 +67,7 @@ jobs: build_and_release: name: Build and Release Tailpipe needs: [ensure_branch_in_homebrew] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - name: validate if: github.ref == 'refs/heads/develop' @@ -110,6 +110,20 @@ jobs: token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Install Docker (if needed) + run: | + if ! command -v docker &> /dev/null; then + sudo apt-get update + sudo apt-get install -y docker.io + fi + + - name: Verify Docker installation + run: | + docker --version + - name: Calculate version id: calculate_version run: | diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index cc81641a..cbcbc901 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -16,7 +16,7 @@ env: jobs: goreleaser: name: Build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.goreleaser.yml b/.goreleaser.yml index d8e8a667..f8834287 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,40 +1,50 @@ version: 2 builds: - - id: tailpipe-linux-arm64 + - id: tailpipe-linux-amd64 binary: tailpipe goos: - linux goarch: - - arm64 + - amd64 env: - - CC=aarch64-linux-gnu-gcc - - CXX=aarch64-linux-gnu-g++ + - CC=x86_64-linux-gnu-gcc + - CXX=x86_64-linux-gnu-g++ + - CGO_ENABLED=1 + - GOFLAGS= + - CGO_LDFLAGS= # Custom ldflags. # # Default: '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser' # Templates: allowed ldflags: - # Go Releaser analyzes your Git repository and identifies the most recent Git tag (typically the highest version number) as the version for your release. + # Goreleaser analyzes your Git repository and identifies the most recent Git tag (typically the highest version number) as the version for your release. # This is how it determines the value of {{.Version}}. - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - - id: tailpipe-linux-amd64 + tags: [] + + - id: tailpipe-linux-arm64 binary: tailpipe goos: - linux goarch: - - amd64 + - arm64 env: - - CC=x86_64-linux-gnu-gcc - - CXX=x86_64-linux-gnu-g++ + - CC=gcc + - CXX=g++ + - CGO_ENABLED=1 + - GOFLAGS= + - CGO_LDFLAGS= ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + tags: [] + - id: tailpipe-darwin-arm64 binary: tailpipe goos: diff --git a/Makefile b/Makefile index 42138790..74f25773 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ OUTPUT_DIR?=/usr/local/bin PACKAGE_NAME := github.com/turbot/tailpipe -GOLANG_CROSS_VERSION ?= gcc13-osxcross-20250912194615 +GOLANG_CROSS_VERSION ?= gcc13-osxcross-20251006102018 # sed 's/[\/_]/-/g': Replaces both slashes (/) and underscores (_) with hyphens (-). # sed 's/[^a-zA-Z0-9.-]//g': Removes any character that isn’t alphanumeric, a dot (.), or a hyphen (-). @@ -30,6 +30,7 @@ release-dry-run: release-acceptance: @docker run \ --rm \ + --platform=linux/arm64 \ -e CGO_ENABLED=1 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/tailpipe \ @@ -48,6 +49,7 @@ release: fi docker run \ --rm \ + --platform=linux/arm64 \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ From 5817e03d9468343151f9f26a7c7c48defb74423e Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Tue, 7 Oct 2025 17:32:55 +0530 Subject: [PATCH 60/61] v0.7.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fba086bd..3508990a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.7.1 [2025-10-07] +_Bug Fixes_ +- Build: Restored CentOS/RHEL 9 compatibility by pinning the build image to an older libstdc++/GCC baseline. Previous build linked against newer GLIBCXX symbols, causing Tailpipe to fail on CentOS/RHEL 9. + ## v0.7.0 [2025-09-22] ### _Major Changes_ From 1675749b24fb4e3d978bfec3926c3877f425d191 Mon Sep 17 00:00:00 2001 From: Piotr Idzik <65706193+vil02@users.noreply.github.com> Date: Tue, 14 Oct 2025 08:11:45 +0200 Subject: [PATCH 61/61] Fix dead links to tailpipe hub --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2ed4cfd4..e8ae6611 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -[![plugins](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_plugins)](https://hub.tailpipe-io.vercel.app/)   -[![mods](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_mods)](https://hub.tailpipe-io.vercel.app/)   +[![plugins](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_plugins)](https://hub.tailpipe.io/)   +[![mods](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_mods)](https://hub.tailpipe.io/)   [![slack](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=slack)](https://turbot.com/community/join?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)   [![maintained by](https://img.shields.io/badge/maintained%20by-Turbot-blue)](https://turbot.com?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)