turbot · pskrbasu · Sep 22, 2025 · Sep 17, 2025 · Sep 17, 2025 · Sep 18, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,30 @@
+## v0.7.0 [2025-09-22]
+
+### _Major Changes_
+* Replace native Parquet conversion with a **DuckLake database backend**. ([#546](https://github.com/turbot/tailpipe/issues/546)) 
+  - DuckLake is DuckDB’s new lakehouse format: data remains in Parquet files, but metadata is efficiently tracked in a
+    separate DuckDB database.
+  - DuckLake supports function-based partitioning, which allows data to be partitioned by year and month. This enables
+    efficient file pruning on `tp_timestamp` without needing a separate `tp_date` filter. A `tp_date` column will still
+    be present for compatibility, but it is no longer required for efficient query filtering.
+  - Existing data will be **automatically migrated** the next time Tailpipe runs. Migration does **not**
+    occur if progress output is disabled (`--progress=false`) or when using machine-readable output (`json`, `line`,
+    `csv`).
+
+* The `connect` command now returns the path to an **initialisation SQL script** instead of the database path. ([#550](https://github.com/turbot/tailpipe/issues/550))
+  - The script sets up DuckDB with required extensions, attaches the Tailpipe database, and defines views with optional
+    filters.
+  - You can pass the generated script to DuckDB using the `--init` argument to immediately configure the session. For
+    example:
+    ```sh
+    duckdb --init $(tailpipe connect)
+    ```
+    **Note:** To ensure compatibility with DuckLake features, make sure you’re using DuckDB version 1.4.0 or later.
+
+### _Bug Fixes_
+* Include partitions for local plugins in the `tailpipe plugin list` command. ([#538](https://github.com/turbot/tailpipe/issues/538))
+
+
 ## v0.6.2 [2025-07-24]
 _Bug fixes_
 * Fix issue where `--to` was not respected for zero granularity data. ([#483](https://github.com/turbot/tailpipe/issues/483))

diff --git a/cmd/collect.go b/cmd/collect.go
@@ -124,6 +124,13 @@ func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) er
 		partitionNames = append(partitionNames, partition.FullName)
 	}
 	slog.Info("Starting collection", "partition(s)", partitionNames, "from", fromTime, "to", toTime)
+
+	// Create backup of metadata database before starting collection
+	if err := database.BackupDucklakeMetadata(); err != nil {
+		slog.Warn("Failed to backup metadata database", "error", err)
+		// Continue with collection - backup failure shouldn't block the operation
+	}
+
 	// now we have the partitions, we can start collecting
 
 	// start the plugin manager

diff --git a/cmd/compact.go b/cmd/compact.go
@@ -77,6 +77,12 @@ func runCompactCmd(cmd *cobra.Command, args []string) {
 	patterns, err := database.GetPartitionPatternsForArgs(maps.Keys(config.GlobalConfig.Partitions), args...)
 	error_helpers.FailOnErrorWithMessage(err, "failed to get partition patterns")
 
+	// Create backup of metadata database before starting compaction
+	if err := database.BackupDucklakeMetadata(); err != nil {
+		slog.Warn("Failed to backup metadata database", "error", err)
+		// Continue with compaction - backup failure shouldn't block the operation
+	}
+
 	// do the compaction
 
 	status, err := doCompaction(ctx, db, patterns)

diff --git a/cmd/partition.go b/cmd/partition.go
@@ -301,6 +301,13 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) {
 	error_helpers.FailOnError(err)
 	defer db.Close()
 
+	// Create backup before deletion
+	slog.Info("Creating backup before partition deletion", "partition", partitionName)
+	if err := database.BackupDucklakeMetadata(); err != nil {
+		slog.Warn("Failed to create backup before partition deletion", "error", err)
+		// Continue with deletion - backup failure should not prevent deletion
+	}
+
 	// show spinner while deleting the partition
 	spinner := statushooks.NewStatusSpinnerHook()
 	spinner.SetStatus(fmt.Sprintf("Deleting partition %s", partition.TableName))

diff --git a/cmd/root.go b/cmd/root.go
@@ -1,6 +1,9 @@
 package cmd
 
 import (
+	"errors"
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/turbot/pipe-fittings/v2/cmdconfig"
@@ -9,6 +12,7 @@ import (
 	"github.com/turbot/pipe-fittings/v2/filepaths"
 	"github.com/turbot/pipe-fittings/v2/utils"
 	"github.com/turbot/tailpipe/internal/constants"
+	"github.com/turbot/tailpipe/internal/migration"
 )
 
 var exitCode int
@@ -62,8 +66,18 @@ func Execute() int {
 	utils.LogTime("cmd.root.Execute start")
 	defer utils.LogTime("cmd.root.Execute end")
 	rootCmd := rootCommand()
+
+	// set the error output to stdout (as it;s common usage to redirect stderr to a file to capture logs
+	rootCmd.SetErr(os.Stdout)
+
+	// if the error is dues to unsupported migration, set a specific exit code - this will bve picked up by powerpipe
 	if err := rootCmd.Execute(); err != nil {
-		exitCode = -1
+		var unsupportedErr *migration.UnsupportedError
+		if errors.As(err, &unsupportedErr) {
+			exitCode = pconstants.ExitCodeMigrationUnsupported
+		} else {
+			exitCode = 1
+		}
 	}
 	return exitCode
 }
diff --git a/go.mod b/go.mod
@@ -6,7 +6,7 @@ toolchain go1.24.0
 
 replace (
 	github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50
-// github.com/turbot/pipe-fittings/v2 => ../pipe-fittings
+//github.com/turbot/pipe-fittings/v2 => ../pipe-fittings
 //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core
 // github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk
 )
@@ -17,9 +17,9 @@ require (
 	github.com/mattn/go-isatty v0.0.20
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.0
 	github.com/turbot/go-kit v1.3.0
-	github.com/turbot/pipe-fittings/v2 v2.7.0-rc.1
+	github.com/turbot/pipe-fittings/v2 v2.7.0
 	github.com/turbot/tailpipe-plugin-sdk v0.9.3
 	github.com/zclconf/go-cty v1.16.3
 	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792
@@ -39,39 +39,39 @@ require (
 	github.com/hashicorp/go-plugin v1.6.1
 	github.com/hashicorp/go-version v1.7.0
 	github.com/jedib0t/go-pretty/v6 v6.5.9
-	github.com/marcboeker/go-duckdb/v2 v2.3.5
+	github.com/marcboeker/go-duckdb/v2 v2.4.0
 	github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02
 	github.com/thediveo/enumflag/v2 v2.0.5
 	github.com/turbot/tailpipe-plugin-core v0.2.10
-	golang.org/x/text v0.27.0
-	google.golang.org/grpc v1.73.0
-	google.golang.org/protobuf v1.36.6
+	golang.org/x/text v0.28.0
+	google.golang.org/grpc v1.75.0
+	google.golang.org/protobuf v1.36.8
 )
 
 require (
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
-	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 )
 
 require (
-	cel.dev/expr v0.23.0 // indirect
+	cel.dev/expr v0.24.0 // indirect
 	cloud.google.com/go v0.121.0 // indirect
 	cloud.google.com/go/auth v0.16.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	cloud.google.com/go/compute/metadata v0.6.0 // indirect
+	cloud.google.com/go/compute/metadata v0.7.0 // indirect
 	cloud.google.com/go/iam v1.5.0 // indirect
 	cloud.google.com/go/monitoring v1.24.0 // indirect
 	cloud.google.com/go/storage v1.52.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
-	github.com/apache/arrow-go/v18 v18.4.0 // indirect
+	github.com/apache/arrow-go/v18 v18.4.1 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.44.183 // indirect
@@ -97,7 +97,7 @@ require (
 	github.com/charmbracelet/lipgloss v1.0.0 // indirect
 	github.com/charmbracelet/x/ansi v0.4.5 // indirect
 	github.com/charmbracelet/x/term v0.2.1 // indirect
-	github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
+	github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
 	github.com/containerd/containerd v1.7.27 // indirect
 	github.com/containerd/errdefs v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -106,12 +106,12 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgraph-io/ristretto v0.2.0 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/duckdb/duckdb-go-bindings v0.1.17 // indirect
-	github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.12 // indirect
-	github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.12 // indirect
-	github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.12 // indirect
-	github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.12 // indirect
-	github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.12 // indirect
+	github.com/duckdb/duckdb-go-bindings v0.1.19 // indirect
+	github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 // indirect
+	github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 // indirect
+	github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 // indirect
+	github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 // indirect
+	github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 // indirect
 	github.com/elastic/go-grok v0.3.1 // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
@@ -122,8 +122,8 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.0 // indirect
 	github.com/go-git/go-git/v5 v5.13.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
@@ -169,8 +169,8 @@ require (
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/marcboeker/go-duckdb/arrowmapping v0.0.10 // indirect
-	github.com/marcboeker/go-duckdb/mapping v0.0.11 // indirect
+	github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 // indirect
+	github.com/marcboeker/go-duckdb/mapping v0.0.19 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
@@ -220,30 +220,30 @@ require (
 	github.com/zclconf/go-cty-yaml v1.0.3 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
-	go.opentelemetry.io/otel v1.35.0 // indirect
-	go.opentelemetry.io/otel/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.40.0 // indirect
-	golang.org/x/mod v0.26.0 // indirect
-	golang.org/x/net v0.42.0 // indirect
-	golang.org/x/oauth2 v0.29.0 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/mod v0.27.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/sys v0.34.0 // indirect
-	golang.org/x/term v0.33.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/term v0.34.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
-	golang.org/x/tools v0.35.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.230.0 // indirect
 	google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect