From 19ee756af49191da9aedf634f03b46c12dcf7127 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Sep 2025 12:33:34 +0100 Subject: [PATCH 1/5] Set ExitCodeMigrationUnsupported exit code for migration unsupported --- cmd/root.go | 10 +++++++++- go.mod | 2 +- internal/migration/errors.go | 27 +++++++++++++++++++++++++++ internal/migration/migration.go | 33 ++++++++++++++++++++++----------- op.log | 21 +++++++++++++++++++++ 5 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 internal/migration/errors.go create mode 100644 op.log diff --git a/cmd/root.go b/cmd/root.go index 6b5669c7..402c3c0e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "os" "github.com/spf13/cobra" @@ -11,6 +12,7 @@ import ( "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/migration" ) var exitCode int @@ -68,8 +70,14 @@ func Execute() int { // set the error output to stdout (as it;s common usage to redirect stderr to a file to capture logs rootCmd.SetErr(os.Stdout) + // if the error is dues to unsupported migration, set a specific exit code - this will bve picked up by powerpipe if err := rootCmd.Execute(); err != nil { - exitCode = -1 + var unsupportedErr *migration.UnsupportedError + if errors.As(err, &unsupportedErr) { + exitCode = pconstants.ExitCodeMigrationUnsupported + } else { + exitCode = 1 + } } return exitCode } diff --git a/go.mod b/go.mod index f3433e3f..c2caf286 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.0 replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 -// github.com/turbot/pipe-fittings/v2 => ../pipe-fittings + //github.com/turbot/pipe-fittings/v2 => ../pipe-fittings //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core // github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) diff --git a/internal/migration/errors.go b/internal/migration/errors.go new file mode 100644 index 00000000..231ba6f0 --- /dev/null +++ b/internal/migration/errors.go @@ -0,0 +1,27 @@ +package migration + +import "fmt" + +// UnsupportedError represents an error when migration is not supported +// due to specific command line arguments or configuration +type UnsupportedError struct { + Reason string +} + +func (e *UnsupportedError) Error() string { + msgFormat := "data must be migrated to Ducklake format - migration is not supported with '%s'.\n\nRun 'tailpipe query' to migrate your data to DuckLake format" + return fmt.Sprintf(msgFormat, e.Reason) +} + +func (e *UnsupportedError) Is(target error) bool { + _, ok := target.(*UnsupportedError) + return ok +} + +func (e *UnsupportedError) As(target interface{}) bool { + if t, ok := target.(**UnsupportedError); ok { + *t = e + return true + } + return false +} diff --git a/internal/migration/migration.go b/internal/migration/migration.go index f3879171..08893322 100644 --- a/internal/migration/migration.go +++ b/internal/migration/migration.go @@ -38,6 +38,7 @@ const ( // MigrateDataToDucklake performs migration of views from tailpipe.db and associated parquet files // into the new DuckLake metadata catalog func MigrateDataToDucklake(ctx context.Context) (err error) { + slog.Info("Starting data migration to DuckLake format") // define a status message var - this will be set when we encounter any issues - or when we are successful // this will be printed at the end of the function var statusMsg string @@ -82,11 +83,8 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { // if the output for this command is a machine readable format (csv/json) or progress is false, // it is possible/likely that tailpipe is being used in a non interactive way - in this case, // we should not prompt the user, instead return an error - msgFormat := "data must be migrated to Ducklake format - migration is not supported with '%s'.\n\nRun 'tailpipe query' to migrate your data to DuckLake format" - if error_helpers.IsMachineReadableOutput() { - return fmt.Errorf(msgFormat, "--output "+viper.GetString(constants.ArgOutput)) - } else if viper.IsSet(constants.ArgProgress) && !viper.GetBool(constants.ArgProgress) { - return fmt.Errorf(msgFormat, "--progress=false") + if err := checkMigrationSupported(); err != nil { + return err } // Prompt the user to confirm migration @@ -131,9 +129,6 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { return fmt.Errorf("failed to discover legacy tables: %w", err) } - slog.Info("Views: ", "views", views) - slog.Info("Schemas: ", "schemas", schemas) - // STEP 3: If this is the first time we are migrating(tables in ~/.tailpipe/data) then move the whole contents of data dir // into ~/.tailpipe/migration/migrating respecting the same folder structure. // We do this by simply renaming the directory. @@ -246,6 +241,24 @@ func MigrateDataToDucklake(ctx context.Context) (err error) { return err } +// check if the data migration is supported, based on the current arguments +// if the output for this command is a machine readable format (csv/json) or progress is false, +// it is possible/likely that tailpipe is being used in a non interactive way - in this case, +// we should not prompt the user, instead return an error +// NOTE: set exit code to +func checkMigrationSupported() error { + if error_helpers.IsMachineReadableOutput() { + return &UnsupportedError{ + Reason: "--output " + viper.GetString(constants.ArgOutput), + } + } else if viper.IsSet(constants.ArgProgress) && !viper.GetBool(constants.ArgProgress) { + return &UnsupportedError{ + Reason: "--progress=false", + } + } + return nil +} + // moveDataToMigrating ensures the migration folder exists and handles any existing migrating folder func moveDataToMigrating(ctx context.Context, dataDefaultDir, migratingDefaultDir string) error { // Ensure the 'migrating' folder exists @@ -430,8 +443,6 @@ func migrateTableDirectory(ctx context.Context, db *database.DuckDb, tableName s func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus, parquetFiles []string) error { filesInLeaf := len(parquetFiles) - // Placeholder: validate schema (from 'ts') against parquet files if needed - slog.Info("Found leaf node with parquet files", "table", tableName, "dir", dirPath, "files", filesInLeaf) // Begin transaction tx, err := db.BeginTx(ctx, nil) @@ -469,7 +480,7 @@ func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName str slog.Warn("Cleanup: could not remove migrated leaf directory", "table", tableName, "dir", dirPath, "error", err) } status.OnFilesMigrated(filesInLeaf) - slog.Info("Migrated leaf node", "table", tableName, "source", dirPath) + slog.Debug("Migrated leaf node", "table", tableName, "source", dirPath) return nil } diff --git a/op.log b/op.log new file mode 100644 index 00000000..b17c4f7a --- /dev/null +++ b/op.log @@ -0,0 +1,21 @@ +{"time":"2025-09-19T11:39:31.813629+01:00","level":"INFO","msg":"Tailpipe CLI","source":"cli","app version":"0.0.0-dev-v0.7.x.20250919111157","log level":"info"} +{"time":"2025-09-19T11:39:31.813769+01:00","level":"INFO","msg":"Resource limits","source":"cli","max CLI memory (mb)":0,"max plugin memory (mb)":0,"max temp dir size (mb)":32768} +{"time":"2025-09-19T11:39:31.813885+01:00","level":"INFO","msg":"No migration needed - no tailpipe.db found in data or migrating directory","source":"cli"} +{"time":"2025-09-19T11:39:31.813937+01:00","level":"INFO","msg":"Compacting parquet files","source":"cli"} +{"time":"2025-09-19T11:39:31.81396+01:00","level":"INFO","msg":"Initializing DuckDB connection","source":"cli"} +{"time":"2025-09-19T11:39:31.819682+01:00","level":"INFO","msg":"install sqlite extension","source":"cli","command":"install sqlite"} +{"time":"2025-09-19T11:39:31.819889+01:00","level":"INFO","msg":"install ducklake extension","source":"cli","command":"install ducklake;"} +{"time":"2025-09-19T11:39:31.820009+01:00","level":"INFO","msg":"attach to ducklake database","source":"cli","command":"attach 'ducklake:sqlite:/Users/kai/.tailpipe/data/default/metadata.sqlite' AS tailpipe_ducklake (\n\tdata_path '/Users/kai/.tailpipe/data/default',\n\tmeta_journal_mode 'WAL')"} +{"time":"2025-09-19T11:39:31.87463+01:00","level":"INFO","msg":"set default catalog to ducklake","source":"cli","command":"use tailpipe_ducklake"} +{"time":"2025-09-19T11:39:31.885216+01:00","level":"INFO","msg":"Creating backup of DuckLake metadata database","source":"cli","source":"/Users/kai/.tailpipe/data/default/metadata.sqlite","backup":"/Users/kai/.tailpipe/data/default/metadata.sqlite.backup.20250919113931"} +{"time":"2025-09-19T11:39:31.889899+01:00","level":"INFO","msg":"Successfully created backup of DuckLake metadata database","source":"cli","backup":"/Users/kai/.tailpipe/data/default/metadata.sqlite.backup.20250919113931"} +{"time":"2025-09-19T11:39:31.890298+01:00","level":"INFO","msg":"Compacting DuckLake data files","source":"cli"} +{"time":"2025-09-19T11:39:31.903925+01:00","level":"INFO","msg":"Ordering DuckLake data files","source":"cli"} +{"time":"2025-09-19T11:52:21.872841+01:00","level":"INFO","msg":"Compacted and ordered all partition entries","source":"cli","tp_table":"aws_cost_and_usage_report","tp_partition":"cody_local_med","tp_index":"default","year":"2025","month":"6","input_files":899} +{"time":"2025-09-19T11:58:12.051011+01:00","level":"INFO","msg":"Compacted and ordered all partition entries","source":"cli","tp_table":"aws_cost_and_usage_report","tp_partition":"cody_local_med","tp_index":"default","year":"2025","month":"7","input_files":898} +{"time":"2025-09-19T11:58:12.053599+01:00","level":"INFO","msg":"Finished ordering DuckLake data file","source":"cli"} +{"time":"2025-09-19T11:58:12.053785+01:00","level":"INFO","msg":"Expiring old DuckLake snapshots","source":"cli"} +{"time":"2025-09-19T11:58:12.306431+01:00","level":"INFO","msg":"DuckLake snapshot expiration complete","source":"cli"} +{"time":"2025-09-19T11:58:34.441032+01:00","level":"INFO","msg":"Cleaning up expired files in DuckLake","source":"cli"} +{"time":"2025-09-19T11:58:34.95607+01:00","level":"INFO","msg":"DuckLake expired files cleanup complete","source":"cli"} +{"time":"2025-09-19T11:58:34.975409+01:00","level":"INFO","msg":"DuckLake compaction complete","source":"cli","source_file_count":3979,"destination_file_count":25} From 1ee4923539ccb496d2cc10f927e6a36be5079e6f Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Sep 2025 12:47:55 +0100 Subject: [PATCH 2/5] update pipe-fittings dependency to v2.7.0 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c2caf286..5380c357 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.11.0 github.com/turbot/go-kit v1.3.0 - github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 + github.com/turbot/pipe-fittings/v2 v2.7.0 github.com/turbot/tailpipe-plugin-sdk v0.9.3 github.com/zclconf/go-cty v1.16.3 golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 From 37e83b7ab57676fc9ee0af8231f4e151df9c5890 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Sep 2025 12:55:30 +0100 Subject: [PATCH 3/5] go.sum --- go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.sum b/go.sum index 92a61b42..96ecc2cc 100644 --- a/go.sum +++ b/go.sum @@ -1310,6 +1310,7 @@ github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1: github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 h1:FfKLkfbNmwxyPQIqDCd7m6o9bmtPB7D8a5txbVzjZp4= github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= +github.com/turbot/pipe-fittings/v2 v2.7.0/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= From 02819d7d39f994d1284276381f8324583d5a417a Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 19 Sep 2025 12:59:53 +0100 Subject: [PATCH 4/5] go.sum --- go.mod | 2 +- go.sum | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5380c357..0a633771 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.0 replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 - //github.com/turbot/pipe-fittings/v2 => ../pipe-fittings +//github.com/turbot/pipe-fittings/v2 => ../pipe-fittings //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core // github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) diff --git a/go.sum b/go.sum index 96ecc2cc..1640c3f0 100644 --- a/go.sum +++ b/go.sum @@ -1308,8 +1308,7 @@ github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2 h1:FfKLkfbNmwxyPQIqDCd7m6o9bmtPB7D8a5txbVzjZp4= -github.com/turbot/pipe-fittings/v2 v2.7.0-rc.2/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= +github.com/turbot/pipe-fittings/v2 v2.7.0 h1:eCmpMNlVtV3AxOzsn8njE3O6aoHc74WVAHOntia2hqY= github.com/turbot/pipe-fittings/v2 v2.7.0/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= From b03b76dfa7cab6463f002a29855841be765f3bb2 Mon Sep 17 00:00:00 2001 From: Puskar Basu Date: Fri, 19 Sep 2025 17:35:43 +0530 Subject: [PATCH 5/5] remove stray file --- op.log | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 op.log diff --git a/op.log b/op.log deleted file mode 100644 index b17c4f7a..00000000 --- a/op.log +++ /dev/null @@ -1,21 +0,0 @@ -{"time":"2025-09-19T11:39:31.813629+01:00","level":"INFO","msg":"Tailpipe CLI","source":"cli","app version":"0.0.0-dev-v0.7.x.20250919111157","log level":"info"} -{"time":"2025-09-19T11:39:31.813769+01:00","level":"INFO","msg":"Resource limits","source":"cli","max CLI memory (mb)":0,"max plugin memory (mb)":0,"max temp dir size (mb)":32768} -{"time":"2025-09-19T11:39:31.813885+01:00","level":"INFO","msg":"No migration needed - no tailpipe.db found in data or migrating directory","source":"cli"} -{"time":"2025-09-19T11:39:31.813937+01:00","level":"INFO","msg":"Compacting parquet files","source":"cli"} -{"time":"2025-09-19T11:39:31.81396+01:00","level":"INFO","msg":"Initializing DuckDB connection","source":"cli"} -{"time":"2025-09-19T11:39:31.819682+01:00","level":"INFO","msg":"install sqlite extension","source":"cli","command":"install sqlite"} -{"time":"2025-09-19T11:39:31.819889+01:00","level":"INFO","msg":"install ducklake extension","source":"cli","command":"install ducklake;"} -{"time":"2025-09-19T11:39:31.820009+01:00","level":"INFO","msg":"attach to ducklake database","source":"cli","command":"attach 'ducklake:sqlite:/Users/kai/.tailpipe/data/default/metadata.sqlite' AS tailpipe_ducklake (\n\tdata_path '/Users/kai/.tailpipe/data/default',\n\tmeta_journal_mode 'WAL')"} -{"time":"2025-09-19T11:39:31.87463+01:00","level":"INFO","msg":"set default catalog to ducklake","source":"cli","command":"use tailpipe_ducklake"} -{"time":"2025-09-19T11:39:31.885216+01:00","level":"INFO","msg":"Creating backup of DuckLake metadata database","source":"cli","source":"/Users/kai/.tailpipe/data/default/metadata.sqlite","backup":"/Users/kai/.tailpipe/data/default/metadata.sqlite.backup.20250919113931"} -{"time":"2025-09-19T11:39:31.889899+01:00","level":"INFO","msg":"Successfully created backup of DuckLake metadata database","source":"cli","backup":"/Users/kai/.tailpipe/data/default/metadata.sqlite.backup.20250919113931"} -{"time":"2025-09-19T11:39:31.890298+01:00","level":"INFO","msg":"Compacting DuckLake data files","source":"cli"} -{"time":"2025-09-19T11:39:31.903925+01:00","level":"INFO","msg":"Ordering DuckLake data files","source":"cli"} -{"time":"2025-09-19T11:52:21.872841+01:00","level":"INFO","msg":"Compacted and ordered all partition entries","source":"cli","tp_table":"aws_cost_and_usage_report","tp_partition":"cody_local_med","tp_index":"default","year":"2025","month":"6","input_files":899} -{"time":"2025-09-19T11:58:12.051011+01:00","level":"INFO","msg":"Compacted and ordered all partition entries","source":"cli","tp_table":"aws_cost_and_usage_report","tp_partition":"cody_local_med","tp_index":"default","year":"2025","month":"7","input_files":898} -{"time":"2025-09-19T11:58:12.053599+01:00","level":"INFO","msg":"Finished ordering DuckLake data file","source":"cli"} -{"time":"2025-09-19T11:58:12.053785+01:00","level":"INFO","msg":"Expiring old DuckLake snapshots","source":"cli"} -{"time":"2025-09-19T11:58:12.306431+01:00","level":"INFO","msg":"DuckLake snapshot expiration complete","source":"cli"} -{"time":"2025-09-19T11:58:34.441032+01:00","level":"INFO","msg":"Cleaning up expired files in DuckLake","source":"cli"} -{"time":"2025-09-19T11:58:34.95607+01:00","level":"INFO","msg":"DuckLake expired files cleanup complete","source":"cli"} -{"time":"2025-09-19T11:58:34.975409+01:00","level":"INFO","msg":"DuckLake compaction complete","source":"cli","source_file_count":3979,"destination_file_count":25}